0: ch1=8074 mean# 8074 dev# 0 part# 0
1: ch1=8074 ch2=8074 mean# 8074 dev# 0 part# 0
2: ch1=13338 ch2=10884 mean# 12111 dev# 1227 part# 0.1013128561
3: ch1=18148 ch2=14148 mean# 16148 dev# 2000 part# 0.1238543473
4: ch1=20188 ch2=20182 mean# 20185 dev# 3 part# 0.0001486252167
5: ch1=27764 ch2=20680 mean# 24222 dev# 3542 part# 0.1462306994
6: ch1=28222 ch2=28296 mean# 28259 dev# 37 part# 0.001309317386
7: ch1=34120 ch2=30472 mean# 32296 dev# 1824 part# 0.05647758236
8: ch1=37842 ch2=34824 mean# 36333 dev# 1509 part# 0.04153249112
9: ch1=40380 ch2=40360 mean# 40370 dev# 10 part# 0.0002477086946
10: ch1=47458 ch2=41356 mean# 44407 dev# 3051 part# 0.06870538429
11: ch1=49004 ch2=47884 mean# 48444 dev# 560 part# 0.01155973908
12: ch1=53814 ch2=51148 mean# 52481 dev# 1333 part# 0.02539966845
13: ch1=58624 ch2=54412 mean# 56518 dev# 2106 part# 0.03726246506
14: ch1=60572 ch2=60538 mean# 60555 dev# 17 part# 0.0002807365205
15: ch1=68240 ch2=60944 mean# 64592 dev# 3648 part# 0.05647758236
16: ch1=68698 ch2=68560 mean# 68629 dev# 69 part# 0.001005405878
17: ch1=74596 ch2=70736 mean# 72666 dev# 1930 part# 0.0265598767
18: ch1=78318 ch2=75088 mean# 76703 dev# 1615 part# 0.02105523904
19: ch1=80764 ch2=80716 mean# 80740 dev# 24 part# 0.0002972504335
20: ch1=87934 ch2=81620 mean# 84777 dev# 3157 part# 0.03723887375
21: ch1=89480 ch2=88148 mean# 88814 dev# 666 part# 0.007498817754
22: ch1=94290 ch2=91412 mean# 92851 dev# 1439 part# 0.01549794833
23: ch1=99100 ch2=94676 mean# 96888 dev# 2212 part# 0.02283048468
24: ch1=100956 ch2=100894 mean# 100925 dev# 31 part# 0.0003071587813
25: ch1=108716 ch2=101208 mean# 104962 dev# 3754 part# 0.03576532459
26: ch1=109174 ch2=108824 mean# 108999 dev# 175 part# 0.001605519317
27: ch1=113984 ch2=112088 mean# 113036 dev# 948 part# 0.008386708659
28: ch1=119882 ch2=114264 mean# 117073 dev# 2809 part# 0.02399357666
29: ch1=121148 ch2=121072 mean# 121110 dev# 38 part# 0.0003137643465
30: ch1=128410 ch2=121884 mean# 125147 dev# 3263 part# 0.02607333775
31: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
32: ch1=126692 ch2=123602 mean# 125147 dev# 1545 part# 0.01234548171
33: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
34: ch1=123192 ch2=127102 mean# 125147 dev# 1955 part# 0.01562162896
35: ch1=129004 ch2=121290 mean# 125147 dev# 3857 part# 0.03081975597
36: ch1=121428 ch2=128866 mean# 125147 dev# 3719 part# 0.02971705275
37: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
38: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
39: ch1=123686 ch2=126608 mean# 125147 dev# 1461 part# 0.01167427106
40: ch1=128502 ch2=121792 mean# 125147 dev# 3355 part# 0.02680847324
41: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
42: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
43: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
44: ch1=123096 ch2=127198 mean# 125147 dev# 2051 part# 0.01638872686
45: ch1=129092 ch2=121202 mean# 125147 dev# 3945 part# 0.03152292904
46: ch1=121424 ch2=128870 mean# 125147 dev# 3723 part# 0.02974901516
47: ch1=126230 ch2=124064 mean# 125147 dev# 1083 part# 0.008653823104
48: ch1=125142 ch2=125152 mean# 125147 dev# 5 part# 3.995301525e-05
49: ch1=123590 ch2=126704 mean# 125147 dev# 1557 part# 0.01244136895
50: ch1=128590 ch2=121704 mean# 125147 dev# 3443 part# 0.0275116463
51: ch1=121878 ch2=128416 mean# 125147 dev# 3269 part# 0.02612128137
52: ch1=126230 ch2=124064 mean# 125147 dev# 1083 part# 0.008653823104
53: ch1=125142 ch2=125152 mean# 125147 dev# 5 part# 3.995301525e-05
54: ch1=123000 ch2=127244 mean# 125122 dev# 2122 part# 0.01695944758
55: ch1=128046 ch2=122198 mean# 125122 dev# 2924 part# 0.02336919167
56: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
57: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
58: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
59: ch1=122410 ch2=127834 mean# 125122 dev# 2712 part# 0.02167484535
60: ch1=128636 ch2=121608 mean# 125122 dev# 3514 part# 0.02808458944
61: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
62: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
63: ch1=125096 ch2=125148 mean# 125122 dev# 26 part# 0.0002077971899
64: ch1=122908 ch2=127336 mean# 125122 dev# 2214 part# 0.01769472994
65: ch1=128138 ch2=122106 mean# 125122 dev# 3016 part# 0.02410447403
66: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
67: ch1=126642 ch2=123602 mean# 125122 dev# 1520 part# 0.01214814341
68: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
69: ch1=122322 ch2=127922 mean# 125122 dev# 2800 part# 0.02237815892
70: ch1=128732 ch2=121512 mean# 125122 dev# 3610 part# 0.0288518406
71: ch1=121836 ch2=128408 mean# 125122 dev# 3286 part# 0.02626236793
72: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
73: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
74: ch1=123908 ch2=126336 mean# 125122 dev# 1214 part# 0.00970253033
75: ch1=128234 ch2=122010 mean# 125122 dev# 3112 part# 0.0248717252
76: ch1=121836 ch2=128408 mean# 125122 dev# 3286 part# 0.02626236793
77: ch1=126646 ch2=123598 mean# 125122 dev# 1524 part# 0.01218011221
78: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
79: ch1=123322 ch2=126922 mean# 125122 dev# 1800 part# 0.0143859593
80: ch1=128828 ch2=121416 mean# 125122 dev# 3706 part# 0.02961909177
81: ch1=121840 ch2=128404 mean# 125122 dev# 3282 part# 0.02623039913
82: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
83: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
84: ch1=123820 ch2=126424 mean# 125122 dev# 1302 part# 0.0104058439
85: ch1=128330 ch2=121964 mean# 125147 dev# 3183 part# 0.02543408951
86: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
87: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
88: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
89: ch1=123276 ch2=127018 mean# 125147 dev# 1871 part# 0.01495041831
90: ch1=128920 ch2=121374 mean# 125147 dev# 3773 part# 0.03014854531
91: ch1=121428 ch2=128866 mean# 125147 dev# 3719 part# 0.02971705275
92: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
93: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
94: ch1=123770 ch2=126524 mean# 125147 dev# 1377 part# 0.0110030604
95: ch1=128418 ch2=121876 mean# 125147 dev# 3271 part# 0.02613726258
96: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
97: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
98: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
99: ch1=123180 ch2=127114 mean# 125147 dev# 1967 part# 0.0157175162
100: ch1=126832 ch2=122374 ch3=1088 mean# 83431.33333 dev# 58253.9661 part# 0.6982264788
101: ch1=119248 ch2=125606 ch3=5440 mean# 83431.33333 dev# 55209.25102 part# 0.6617328145
102: ch1=123198 ch2=119712 ch3=7384 mean# 83431.33333 dev# 53792.41415 part# 0.6447507429
103: ch1=118610 ch2=120800 ch3=10884 mean# 83431.33333 dev# 51306.5019 part# 0.6149548359
104: ch1=120786 ch2=118624 ch3=10884 mean# 83431.33333 dev# 51306.30397 part# 0.6149524636
105: ch1=119344 ch2=115714 ch3=15236 mean# 83431.33333 dev# 48244.14881 part# 0.578249764
106: ch1=112258 ch2=118624 ch3=19412 mean# 83431.33333 dev# 45343.04628 part# 0.5434774259
107: ch1=115342 ch2=114272 ch3=20680 mean# 83431.33333 dev# 44374.04347 part# 0.5318630507
108: ch1=112078 ch2=114272 ch3=23944 mean# 83431.33333 dev# 42073.43207 part# 0.504288142
109: ch1=112988 ch2=112096 ch3=25210 mean# 83431.33333 dev# 41170.31015 part# 0.4934634089
110: ch1=111048 ch2=109858 ch3=29388 mean# 83431.33333 dev# 38217.49541 part# 0.4580712531
111: ch1=107722 ch2=111008 ch3=31564 mean# 83431.33333 dev# 36700.26929 part# 0.4398859257
112: ch1=107264 ch2=107744 ch3=35286 mean# 83431.33333 dev# 34044.45566 part# 0.4080535968
113: ch1=108196 ch2=105568 ch3=36530 mean# 83431.33333 dev# 33181.6003 part# 0.3977114948
114: ch1=102298 ch2=107660 ch3=40336 mean# 83431.33333 dev# 30551.52563 part# 0.3661876708
115: ch1=106818 ch2=102108 ch3=41368 mean# 83431.33333 dev# 29805.35783 part# 0.3572441748
116: ch1=99640 ch2=104934 ch3=45720 mean# 83431.33333 dev# 26753.38116 part# 0.3206634737
117: ch1=103404 ch2=99078 ch3=47812 mean# 83431.33333 dev# 25248.5148 part# 0.3026262891
118: ch1=97914 ch2=101220 ch3=51160 mean# 83431.33333 dev# 22859.15754 part# 0.273987681
119: ch1=100090 ch2=97956 ch3=52248 mean# 83431.33333 dev# 22067.15051 part# 0.2644947603
120: ch1=98242 ch2=96082 ch3=55970 mean# 83431.33333 dev# 19438.10727 part# 0.2329832989
121: ch1=92474 ch2=99040 ch3=58780 mean# 83431.33333 dev# 17636.02875 part# 0.2113837578
122: ch1=96196 ch2=93142 ch3=60956 mean# 83431.33333 dev# 15941.29199 part# 0.1910708046
123: ch1=92478 ch2=93596 ch3=64220 mean# 83431.33333 dev# 13592.12952 part# 0.1629139674
124: ch1=91390 ch2=92508 ch3=66396 mean# 83431.33333 dev# 12054.44364 part# 0.1444834112
125: ch1=90132 ch2=90044 ch3=70118 mean# 83431.33333 dev# 9414.016831 part# 0.1128355074
126: ch1=87038 ch2=91416 ch3=71840 mean# 83431.33333 dev# 8388.920365 part# 0.100548799
127: ch1=88126 ch2=87064 ch3=75104 mean# 83431.33333 dev# 5904.253909 part# 0.07076782395
128: ch1=87330 ch2=85976 ch3=76988 mean# 83431.33333 dev# 4589.534278 part# 0.05500971991
129: ch1=83608 ch2=85976 ch3=80710 mean# 83431.33333 dev# 2153.461916 part# 0.02581118903
130: ch1=86820 ch2=81834 ch3=81640 mean# 83431.33333 dev# 2397.457728 part# 0.02873569955
131: ch1=81136 ch2=84254 ch3=84904 mean# 83431.33333 dev# 1644.595459 part# 0.01971196423
132: ch1=85172 ch2=82394 ch3=82728 mean# 83431.33333 dev# 1238.367025 part# 0.01484294899
133: ch1=81358 ch2=84888 ch3=84048 mean# 83431.33333 dev# 1505.641244 part# 0.01804647228
134: ch1=87034 ch2=81624 ch3=81636 mean# 83431.33333 dev# 2547.474741 part# 0.03053378915
135: ch1=78960 ch2=85346 ch3=85988 mean# 83431.33333 dev# 3172.554947 part# 0.03802594085
136: ch1=82564 ch2=83918 ch3=83812 mean# 83431.33333 dev# 614.8221062 part# 0.007369199096
137: ch1=84678 ch2=82716 ch3=82900 mean# 83431.33333 dev# 884.7211739 part# 0.01060418357
138: ch1=84858 ch2=82716 ch3=82720 mean# 83431.33333 dev# 1008.806996 part# 0.01209146439
139: ch1=83312 ch2=83174 ch3=83808 mean# 83431.33333 dev# 272.236825 part# 0.003263004607
140: ch1=80764 ch2=84812 ch3=84718 mean# 83431.33333 dev# 1886.479849 part# 0.02261116746
141: ch1=84402 ch2=83176 ch3=82716 mean# 83431.33333 dev# 711.5922678 part# 0.008529077019
142: ch1=81598 ch2=84892 ch3=83804 mean# 83431.33333 dev# 1370.345293 part# 0.01642482792
143: ch1=86408 ch2=81628 ch3=82258 mean# 83431.33333 dev# 2120.476886 part# 0.02541583361
144: ch1=80666 ch2=84892 ch3=84736 mean# 83431.33333 dev# 1956.422813 part# 0.02344949714
145: ch1=87018 ch2=81712 ch3=81564 mean# 83431.33333 dev# 2536.875944 part# 0.03040675299
146: ch1=81148 ch2=84262 ch3=84884 mean# 83431.33333 dev# 1634.406994 part# 0.01958984627
147: ch1=84236 ch2=83350 ch3=82708 mean# 83431.33333 dev# 626.4488983 part# 0.007508556717
148: ch1=81560 ch2=84854 ch3=83880 mean# 83431.33333 dev# 1381.686248 part# 0.01656075953
149: ch1=87050 ch2=81624 ch3=81620 mean# 83431.33333 dev# 2558.78426 part# 0.03066934397
150: ch1=80064 ch2=85346 ch3=84884 mean# 83431.33333 dev# 2388.522742 part# 0.02862860566
151: ch1=83478 ch2=83478 ch3=83338 mean# 83431.33333 dev# 65.99663291 part# 0.0007910293444
152: ch1=83782 ch2=82720 ch3=83792 mean# 83431.33333 dev# 503.005191 part# 0.006028972221
153: ch1=82694 ch2=83808 ch3=83792 mean# 83431.33333 dev# 521.414316 part# 0.006249622236
154: ch1=84870 ch2=82720 ch3=82704 mean# 83431.33333 dev# 1017.311926 part# 0.01219340367
155: ch1=80518 ch2=84896 ch3=84880 mean# 83431.33333 dev# 2060.048112 part# 0.02469154009
156: ch1=85328 ch2=82720 ch3=82246 mean# 83431.33333 dev# 1355.034399 part# 0.016241313
157: ch1=82698 ch2=83808 ch3=83788 mean# 83431.33333 dev# 518.6092513 part# 0.006216000999
158: ch1=84874 ch2=82720 ch3=82700 mean# 83431.33333 dev# 1020.152058 part# 0.01222744523
159: ch1=80780 ch2=84896 ch3=84618 mean# 83431.33333 dev# 1878.207893 part# 0.02251202058
160: ch1=87212 ch2=81632 ch3=81450 mean# 83431.33333 dev# 2674.367381 part# 0.03205471223
161: ch1=81156 ch2=84266 ch3=84872 mean# 83431.33333 dev# 1627.81353 part# 0.01951081764
162: ch1=84244 ch2=83354 ch3=82696 mean# 83431.33333 dev# 634.3297425 part# 0.007603015763
163: ch1=81754 ch2=84756 ch3=83784 mean# 83431.33333 dev# 1250.67484 part# 0.01499046929
164: ch1=87058 ch2=81628 ch3=81608 mean# 83431.33333 dev# 2564.453591 part# 0.03073729604
165: ch1=81160 ch2=84262 ch3=84872 mean# 83431.33333 dev# 1625.267431 part# 0.01948030034
166: ch1=84248 ch2=83350 ch3=82696 mean# 83431.33333 dev# 636.2061161 part# 0.007625505799
167: ch1=82190 ch2=84320 ch3=83784 mean# 83431.33333 dev# 904.6197482 part# 0.01084268598
168: ch1=85974 ch2=82712 ch3=81608 mean# 83431.33333 dev# 1853.567611 part# 0.02221668451
169: ch1=81164 ch2=84258 ch3=84872 mean# 83431.33333 dev# 1622.72391 part# 0.01944981394
170: ch1=83960 ch2=83346 ch3=82988 mean# 83431.33333 dev# 401.378735 part# 0.004810887216
171: ch1=84798 ch2=82796 ch3=82700 mean# 83431.33333 dev# 967.1736602 part# 0.0115924512
172: ch1=81160 ch2=84258 ch3=84876 mean# 83431.33333 dev# 1625.771065 part# 0.01948633685
173: ch1=85696 ch2=81898 ch3=82700 mean# 83431.33333 dev# 1634.490203 part# 0.0195908436
174: ch1=80886 ch2=84896 ch3=84512 mean# 83431.33333 dev# 1806.6369 part# 0.02165417749
175: ch1=87504 ch2=81632 ch3=81158 mean# 83431.33333 dev# 2886.304365 part# 0.0345949687
176: ch1=81152 ch2=84266 ch3=84876 mean# 83431.33333 dev# 1630.857716 part# 0.01954730496
177: ch1=84240 ch2=83354 ch3=82700 mean# 83431.33333 dev# 631.0759771 part# 0.007564016442
178: ch1=81610 ch2=83808 ch3=84876 mean# 83431.33333 dev# 1359.680681 part# 0.0162970029
179: ch1=88138 ch2=80544 ch3=81612 mean# 83431.33333 dev# 3356.554715 part# 0.04023134452
180: ch1=82240 ch2=83808 ch3=84246 mean# 83431.33333 dev# 861.1687149 part# 0.01032188604
181: ch1=84194 ch2=85572 ch3=80528 mean# 83431.33333 dev# 2128.650485 part# 0.02551380159
182: ch1=87808 ch2=84134 ch3=78352 mean# 83431.33333 dev# 3892.23923 part# 0.046652008
183: ch1=87050 ch2=88156 ch3=75088 mean# 83431.33333 dev# 5916.880785 part# 0.07091916848
184: ch1=89226 ch2=88156 ch3=72912 mean# 83431.33333 dev# 7451.107539 part# 0.08930826395
185: ch1=90314 ch2=89244 ch3=70736 mean# 83431.33333 dev# 8987.578144 part# 0.107724254
186: ch1=91854 ch2=92056 ch3=66384 mean# 83431.33333 dev# 12054.56708 part# 0.1444848908
187: ch1=94662 ch2=90336 ch3=65296 mean# 83431.33333 dev# 12944.65927 part# 0.1551534508
188: ch1=93574 ch2=94688 ch3=62032 mean# 83431.33333 dev# 15138.44663 part# 0.1814479767
189: ch1=95750 ch2=94688 ch3=59856 mean# 83431.33333 dev# 16675.91511 part# 0.1998759273
190: ch1=98464 ch2=95776 ch3=56054 mean# 83431.33333 dev# 19389.77602 part# 0.2324040052
191: ch1=94528 ch2=100266 ch3=55500 mean# 83431.33333 dev# 19888.86954 part# 0.2383860924
192: ch1=101648 ch2=97498 ch3=51148 mean# 83431.33333 dev# 22890.54875 part# 0.2743639331
193: ch1=100106 ch2=101216 ch3=48972 mean# 83431.33333 dev# 24370.6417 part# 0.2921041859
194: ch1=103370 ch2=101216 ch3=45708 mean# 83431.33333 dev# 26688.91577 part# 0.3198907976
195: ch1=102282 ch2=103392 ch3=44620 mean# 83431.33333 dev# 27447.49802 part# 0.3289830921
196: ch1=104188 ch2=105838 ch3=40268 mean# 83431.33333 dev# 30528.51818 part# 0.3659119058
197: ch1=108810 ch2=103392 ch3=38092 mean# 83431.33333 dev# 32135.96159 part# 0.3851785691
198: ch1=107722 ch2=107744 ch3=34828 mean# 83431.33333 dev# 34367.74776 part# 0.4119285452
199: ch1=106634 ch2=109920 ch3=33740 mean# 83431.33333 dev# 35162.67818 part# 0.4214565053
200: ch1=112532 ch2=108374 ch3=29388 mean# 83431.33333 dev# 38252.09057 part# 0.4584859074
201: ch1=111052 ch2=112322 ch3=26920 mean# 83431.33333 dev# 39962.91048 part# 0.4789916316
202: ch1=114254 ch2=111008 ch3=25032 mean# 83431.33333 dev# 41315.82206 part# 0.4952075008
203: ch1=114712 ch2=114902 ch3=20680 mean# 83431.33333 dev# 44371.96113 part# 0.5318380919
204: ch1=114528 ch2=116174 ch3=19592 mean# 83431.33333 dev# 45146.22678 part# 0.5411183662
205: ch1=119338 ch2=115352 ch3=15604 mean# 83431.33333 dev# 47988.76542 part# 0.575188763
206: ch1=114534 ch2=120524 ch3=15236 mean# 83431.33333 dev# 48283.34869 part# 0.57871961
207: ch1=122336 ch2=117074 ch3=10884 mean# 83431.33333 dev# 51343.67109 part# 0.6154003423
208: ch1=119248 ch2=122338 ch3=8708 mean# 83431.33333 dev# 52852.43251 part# 0.6334842127
209: ch1=124054 ch2=121884 ch3=4356 mean# 83431.33333 dev# 55921.72197 part# 0.6702724232
210: ch1=121878 ch2=124060 ch3=4356 mean# 83431.33333 dev# 55921.79979 part# 0.670273356
211: ch1=123102 ch2=126558 ch3=634 mean# 83431.33333 dev# 58563.55402 part# 0.7019371701
212: ch1=128452 ch2=121842 mean# 125147 dev# 3305 part# 0.02640894308
213: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
214: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
215: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
216: ch1=123146 ch2=127148 mean# 125147 dev# 2001 part# 0.0159891967
217: ch1=127954 ch2=122340 mean# 125147 dev# 2807 part# 0.02242962276
218: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
219: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
220: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
221: ch1=123644 ch2=126650 mean# 125147 dev# 1503 part# 0.01200987639
222: ch1=128544 ch2=121750 mean# 125147 dev# 3397 part# 0.02714407856
223: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
224: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
225: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
226: ch1=123054 ch2=127240 mean# 125147 dev# 2093 part# 0.01672433219
227: ch1=128050 ch2=122244 mean# 125147 dev# 2903 part# 0.02319672066
228: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
229: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
230: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
231: ch1=122464 ch2=127830 mean# 125147 dev# 2683 part# 0.02143878799
232: ch1=128640 ch2=121654 mean# 125147 dev# 3493 part# 0.02791117646
233: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
234: ch1=127326 ch2=122968 mean# 125147 dev# 2179 part# 0.01741152405
235: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
236: ch1=122962 ch2=127332 mean# 125147 dev# 2185 part# 0.01745946767
237: ch1=128142 ch2=122152 mean# 125147 dev# 2995 part# 0.02393185614
238: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
239: ch1=126696 ch2=123598 mean# 125147 dev# 1549 part# 0.01237744413
240: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
241: ch1=122376 ch2=127918 mean# 125147 dev# 2771 part# 0.02214196105
242: ch1=128736 ch2=121558 mean# 125147 dev# 3589 part# 0.02867827435
243: ch1=121890 ch2=128404 mean# 125147 dev# 3257 part# 0.02602539414
244: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
245: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
246: ch1=123962 ch2=126332 mean# 125147 dev# 1185 part# 0.009468864615
247: ch1=128238 ch2=122056 mean# 125147 dev# 3091 part# 0.02469895403
248: ch1=122978 ch2=127316 mean# 125147 dev# 2169 part# 0.01733161802
249: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
250: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
251: ch1=123372 ch2=126922 mean# 125147 dev# 1775 part# 0.01418332042
252: ch1=128828 ch2=121466 mean# 125147 dev# 3681 part# 0.02941340983
253: ch1=121890 ch2=128404 mean# 125147 dev# 3257 part# 0.02602539414
254: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
255: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
256: ch1=123870 ch2=126424 mean# 125147 dev# 1277 part# 0.0102040001
257: ch1=128330 ch2=121964 mean# 125147 dev# 3183 part# 0.02543408951
258: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
259: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
260: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
261: ch1=123276 ch2=127018 mean# 125147 dev# 1871 part# 0.01495041831
262: ch1=127832 ch2=122462 mean# 125147 dev# 2685 part# 0.02145476919
263: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
264: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
265: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
266: ch1=123774 ch2=126520 mean# 125147 dev# 1373 part# 0.01097109799
267: ch1=128422 ch2=121872 mean# 125147 dev# 3275 part# 0.02616922499
268: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
269: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
270: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
271: ch1=123184 ch2=127110 mean# 125147 dev# 1963 part# 0.01568555379
272: ch1=127924 ch2=122370 mean# 125147 dev# 2777 part# 0.02218990467
273: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
274: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
275: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
276: ch1=122594 ch2=127700 mean# 125147 dev# 2553 part# 0.02040000959
277: ch1=121152 ch2=129142 mean# 125147 dev# 3995 part# 0.03192245919
278: ch1=114066 ch2=136228 mean# 125147 dev# 11081 part# 0.08854387241
279: ch1=112520 ch2=137774 mean# 125147 dev# 12627 part# 0.1008973447
280: ch1=107710 ch2=142584 mean# 125147 dev# 17437 part# 0.1393321454
281: ch1=102900 ch2=147394 mean# 125147 dev# 22247 part# 0.1777669461
282: ch1=100960 ch2=149334 mean# 125147 dev# 24187 part# 0.193268716
283: ch1=93284 ch2=157010 mean# 125147 dev# 31863 part# 0.254604585
284: ch1=92826 ch2=157468 mean# 125147 dev# 32321 part# 0.2582642812
285: ch1=88016 ch2=162278 mean# 125147 dev# 37131 part# 0.2966990819
286: ch1=82118 ch2=166520 mean# 124319 dev# 42201 part# 0.3394573637
2025-11-19T13:58:41.208502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435621765195653:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:41.275885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435624968971209:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:41.307686Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435621765195663:2057] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:41.308145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574435621765195653:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:41.313950Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435624968971220:2057] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:41.348138Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435621765195663:2057] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:41.353461Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435624968971220:2057] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:41.354858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435624968971209:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T13:58:41.354889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:41.354899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435624968971209:2048] [node 1] ICP22 created new session: [2:7574435624968971223:2048]
2025-11-19T13:58:41.354908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435624968971223:2048] [node 1] ICS09 handshake done sender: [2:7574435624968971220:2057] self: [2:7574435624968971219:0] peer: [1:7574435621765195664:0] socket: 16 qp: 77970
2025-11-19T13:58:41.354925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435624968971223:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:41.354957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:41.355627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574435621765195653:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:41.355651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:41.355657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435621765195653:2048] [node 2] ICP22 created new session: [1:7574435621765195666:2048]
2025-11-19T13:58:41.355663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435621765195666:2048] [node 2] ICS09 handshake done sender: [1:7574435621765195663:2057] self: [1:7574435621765195664:0] peer: [2:7574435624968971219:0] socket: 17 qp: 77966
2025-11-19T13:58:41.355669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435621765195666:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:41.355691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:41.376042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:41.376136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435624968971223:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-19T13:58:41.376176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435624968971209:2048] [node 1] ICP30 unregister session Session# [2:7574435624968971223:2048] VirtualId# [2:7574435624968971219:0]
2025-11-19T13:58:41.376183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435624968971223:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:41.376241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:41.376262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:41.377374Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435624968971209:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# ::1(::1:5266) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:41.377386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574435624968971209:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# ::1(::1:5266) Couldn't connect to any resolved address LastSessionDieTime# 2025-11-19T13:58:41.373979Z
2025-11-19T13:58:41.377389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# ::1(::1:5266) Couldn't connect to any resolved address LastSessionDieTime# 2025-11-19T13:58:41.373979Z
2025-11-19T13:58:41.555482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435623920104100:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:41.647895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435621467635222:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:41.675909Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435623920104110:2057] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:41.676426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574435623920104100:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:41.676568Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1579: Handshake [1:7574435623920104110:2057] [node 2] ICH03 handshake failed, explanation# incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected
2025-11-19T13:58:41.678151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:404: Proxy [1:7574435623920104100:2048] [node 2] ICP24 incoming handshake failed, temporary: 1 explanation: incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected outgoing: [0:0:0]
2025-11-19T13:58:41.678163Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [1:7574435623920104100:2048] [node 2] ICP32 transit to hold-by-error state Explanation# incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:41.678167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:41.681619Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1579: Handshake [2:7574435621467635233:2057] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:3855) error from peer: connection checker refused to proceed with incoming connection: rejected
2025-11-19T13:58:41.682358Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435621467635222:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# ::1(::1:3855) error from peer: connection checker refused to proceed with incoming connection: rejected incoming: [0:0:0] held: no
2025-11-19T13:58:41.682377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574435621467635222:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# ::1(::1:3855) error from peer: connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:41.682380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# ::1(::1:3855) error from peer: connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:41.759896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435621512545055:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:41.759914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435621512545056:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:41.835915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435623671206906:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:41.835942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435623671206907:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:41.923888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435623402209921:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:41.923913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435623402209922:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:41.967890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435624274461770:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:41.967908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435624274461771:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:41.967914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435624274461772:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:42.076420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435627989276743:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:42.076445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435627989276744:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:42.076450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435627989276745:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:42.195931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435628575480636:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:42.195954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435628575480637:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:42.195959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435628575480638:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:42.258964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435628575480649:2059] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.259679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435627989276796:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50033
2025-11-19T13:58:42.260015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435628569429125:2105] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.260637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435627989276797:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50033
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.260730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435627966174262:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27212
2025-11-19T13:58:42.261784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435628575480655:2062] [node 4] ICP01 ready to work
2025-11-19T13:58:42.281606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435625807512417:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:42.291777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435625807512418:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:42.291884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435625807512419:2110] [node 5] ICP01 ready to work
2025-11-19T13:58:42.291774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435627966174266:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:42.291886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435627966174267:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:42.291716Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435628575480655:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(29855449ns) {Configure:104(29854386ns) {ProcessConfigured:135(10282125ns) {StartInitialHandshake:157(10280561ns) {DropHandshakes:469(1921ns) {DropIncomingHandshake:441(404ns) DropOutgoingHandshake:456(213ns)} GenerateSessionVirtualId:988(286ns) PrepareNewSessionHandshake:476(2064ns) {SwitchToState:178(1020ns)}}}}}
2025-11-19T13:58:42.312044Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435627989276744:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(44082018ns) {Configure:104(44080565ns) {ProcessConfigured:135(20161762ns) {StartInitialHandshake:157(20160483ns) {DropHandshakes:469(1541ns) {DropIncomingHandshake:441(242ns) DropOutgoingHandshake:456(287ns)} GenerateSessionVirtualId:988(284ns) PrepareNewSessionHandshake:476(2552ns) {SwitchToState:178(392ns)}}}}}
2025-11-19T13:58:42.312483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435628569429133:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:42.316426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435627966174268:2110] [node 6] ICP01 ready to work
2025-11-19T13:58:42.323814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435627697177281:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:42.326566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435627989276810:2105] [node 6] ICP01 ready to work
2025-11-19T13:58:42.336054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435627989276811:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:42.337489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435628575480636:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(45595186ns) {Configure:104(45594278ns) {ProcessConfigured:135(16520ns) {StartInitialHandshake:157(16127ns) {DropHandshakes:469(392ns) {DropIncomingHandshake:441(33ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(132ns) PrepareNewSessionHandshake:476(329ns) {SwitchToState:178(234ns)}}}}}
2025-11-19T13:58:42.337346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435627697177282:2108] [node 5] ICP01 ready to work
2025-11-19T13:58:42.339180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435625807512417:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(47225947ns) {Configure:104(47225495ns) {ProcessConfigured:135(27079ns) {StartInitialHandshake:157(25905ns) {DropHandshakes:469(512ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(127ns) PrepareNewSessionHandshake:476(359ns) {SwitchToState:178(72ns)}}}}}
2025-11-19T13:58:42.341087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435627697177283:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:42.344009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435623671206907:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(27522293ns) {Configure:104(27521915ns) {ProcessConfigured:135(23116ns) {StartInitialHandshake:157(22799ns) {DropHandshakes:469(389ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(172ns) PrepareNewSessionHandshake:476(396ns) {SwitchToState:178(71ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.345361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435624274461772:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(32814917ns) {Configure:104(32813313ns) {ProcessConfigured:135(17594ns) {StartInitialHandshake:157(16554ns) {DropHandshakes:469(478ns) {DropIncomingHandshake:441(27ns) DropOutgoingHandshake:456(19ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:476(372ns) {SwitchToState:178(56ns)}}}}}
2025-11-19T13:58:42.346231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435628569429142:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50189
2025-11-19T13:58:42.346242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435628569429143:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50189
2025-11-19T13:58:42.345002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435627966174279:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27212
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.352300Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435627989276797:2101] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.352476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435627989276811:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.352502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435627989276820:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50033
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.355170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435628575480713:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60261
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.359691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435627966174283:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27212
2025-11-19T13:58:42.359999Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435627966174283:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.377951Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435628569429127:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.377970Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435628569429127:2106] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:42.408856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435627989276796:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.412206Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435628575480713:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.412255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435628575480649:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.412398Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435627989276813:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.412407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435627989276813:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:42.423947Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435628569429143:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.421703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435625807512428:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29219
2025-11-19T13:58:42.421748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435625807512429:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29219
2025-11-19T13:58:42.421778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435625807512430:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29219
2025-11-19T13:58:42.421809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435625807512431:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29219
2025-11-19T13:58:42.425542Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435627989276813:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.425709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435628569429146:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50189
2025-11-19T13:58:42.425748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435628569429147:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50189
2025-11-19T13:58:42.425771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435624274461771:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.426084Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435628569429127:2106] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:42.426170Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435625807512431:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.426201Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435625807512430:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.427942Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435627966174272:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.427956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435627966174272:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.432809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435627989276810:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.432839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:42.432847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435627989276810:2105] [node 6] ICP22 created new session: [5:7574435627989276828:2105]
2025-11-19T13:58:42.432854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435627989276828:2105] [node 6] ICS09 handshake done sender: [5:7574435627989276813:2113] self: [5:7574435627989276812:0] peer: [6:7574435628575480715:0] socket: 55 qp: -1
2025-11-19T13:58:42.432861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435627989276828:2105] [node 6] ICS10 traffic start
2025-11-19T13:58:42.432873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:42.433186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435628569429125:2105] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.433203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:42.433207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435628569429125:2105] [node 5] ICP22 created new session: [4:7574435628569429150:2105]
2025-11-19T13:58:42.433211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435628569429150:2105] [node 5] ICS09 handshake done sender: [4:7574435628569429127:2106] self: [4:7574435628569429126:0] peer: [5:7574435627989276824:0] socket: 39 qp: -1
2025-11-19T13:58:42.433216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435628569429150:2105] [node 5] ICS10 traffic start
2025-11-19T13:58:42.433222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:42.433481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435628569429152:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50189
2025-11-19T13:58:42.434098Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435627966174272:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.434279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435627966174266:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.434291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:42.434298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435627966174266:2108] [node 4] ICP22 created new session: [2:7574435627966174287:2108]
2025-11-19T13:58:42.434304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435627966174287:2108] [node 4] ICS09 handshake done sender: [2:7574435627966174272:2112] self: [2:7574435627966174271:0] peer: [4:7574435628569429149:0] socket: 47 qp: -1
2025-11-19T13:58:42.434309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435627966174287:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:42.434319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:42.434505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435627966174286:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27212
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.435122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435627989276827:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50033
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.463247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435627697177297:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49232
2025-11-19T13:58:42.463201Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435627989276797:2101] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:42.466396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435627697177298:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49232
2025-11-19T13:58:42.463907Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435628569429143:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:42.465001Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435627966174279:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.465079Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435627966174286:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.465126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435627966174266:2108] [node 4] ICP18 handshake (actor [2:7574435627966174286:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.467496Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435628569429142:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.467607Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435628569429146:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.467614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435624274461771:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.467632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:42.467640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435624274461771:2049] [node 2] ICP22 created new session: [4:7574435628569429154:2049]
2025-11-19T13:58:42.467646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435628569429154:2049] [node 2] ICS09 handshake done sender: [4:7574435628569429143:2116] self: [4:7574435628569429149:0] peer: [2:7574435627966174271:0] socket: 50 qp: -1
2025-11-19T13:58:42.467654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435628569429154:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:42.467663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:42.467610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435628575480649:2059] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435628575480713:2115] held: no
2025-11-19T13:58:42.467699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435628575480716:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60261
2025-11-19T13:58:42.467712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435628575480717:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60261
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.468098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435627697177297:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.468118Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435627697177298:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.468219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435623402209922:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.468263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435627697177299:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49232
2025-11-19T13:58:42.467875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435627989276830:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50033
2025-11-19T13:58:42.467937Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435627989276820:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.467945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435627989276811:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.467959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:42.467965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435627989276811:2108] [node 4] ICP22 created new session: [5:7574435627989276832:2108]
2025-11-19T13:58:42.467969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435627989276832:2108] [node 4] ICS09 handshake done sender: [5:7574435627989276797:2101] self: [5:7574435627989276824:0] peer: [4:7574435628569429126:0] socket: 40 qp: -1
2025-11-19T13:58:42.467973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435627989276832:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:42.467979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:42.469293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435627989276745:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.471886Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435627697177287:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.471893Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435627697177287:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:42.472687Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435627697177287:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:42.472839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435627697177282:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.472867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:42.472874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435627697177282:2108] [node 5] ICP22 created new session: [3:7574435627697177303:2108]
2025-11-19T13:58:42.472880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435627697177303:2108] [node 5] ICS09 handshake done sender: [3:7574435627697177287:2112] self: [3:7574435627697177286:0] peer: [5:7574435627989276834:0] socket: 43 qp: -1
2025-11-19T13:58:42.472889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435627697177303:2108] [node 5] ICS10 traffic start
2025-11-19T13:58:42.472900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:42.476042Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435628575480713:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:42.476267Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435625807512429:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.486494Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435627966174270:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.486514Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435627966174270:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:42.487518Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435627966174270:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:42.487601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435623671206907:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.487634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:42.487643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435623671206907:2049] [node 3] ICP22 created new session: [2:7574435627966174292:2049]
2025-11-19T13:58:42.487649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435627966174292:2049] [node 3] ICS09 handshake done sender: [2:7574435627966174270:2111] self: [2:7574435627966174269:0] peer: [3:7574435627697177300:0] socket: 46 qp: -1
2025-11-19T13:58:42.487657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435627966174292:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:42.487664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.492019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435628575480649:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.493723Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435625807512428:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.493922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435625807512436:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29219
2025-11-19T13:58:42.495843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435624274461771:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(24476421ns) {HandleHandshakeStatus:329(24475646ns) {DropIncomingHandshake:441(439ns) DropOutgoingHandshake:456(1407ns) SwitchToState:178(279ns) ProcessPendingSessionEvents:495(18419835ns) {ForwardSessionEventToSession:612(18406496ns) ForwardSessionEventToSession:612(2413ns) ForwardSessionEventToSession:612(607ns) ForwardSessionEventToSession:612(1328ns) ForwardSessionEventToSession:612(881ns) ForwardSessionEventToSession:612(623ns) ForwardSessionEventToSession:612(907ns) ForwardSessionEventToSession:612(576ns) ForwardSessionEventToSession:612(524ns) ForwardSessionEventToSession:612(668ns)}}}
2025-11-19T13:58:42.495548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:42.495562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435628575480649:2059] [node 5] ICP22 created new session: [6:7574435628575480719:2059]
2025-11-19T13:58:42.495569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435628575480719:2059] [node 5] ICS09 handshake done sender: [6:7574435628575480713:2115] self: [6:7574435628575480715:0] peer: [5:7574435627989276812:0] socket: 58 qp: -1
2025-11-19T13:58:42.495577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435628575480719:2059] [node 5] ICS10 traffic start
2025-11-19T13:58:42.495587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:42.495781Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435628575480716:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.496032Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435627966174262:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.497151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435623671206906:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.498363Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435628569429147:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.498507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435624274461772:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.498569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435624274461770:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.498799Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435625807512412:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.498812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435625807512412:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:42.499350Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435625807512421:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.499353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435625807512421:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.499986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435627966174296:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27212
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.500325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435628575480636:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.500357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435628575480721:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60261
2025-11-19T13:58:42.500425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435628575480722:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60261
2025-11-19T13:58:42.500610Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435628575480721:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.500627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435628575480637:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.500703Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435627966174278:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.500709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435627966174278:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:42.501421Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435627966174278:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:42.501515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435627966174268:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.501528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:42.501536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435627966174268:2110] [node 6] ICP22 created new session: [2:7574435627966174298:2110]
2025-11-19T13:58:42.501541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435627966174298:2110] [node 6] ICS09 handshake done sender: [2:7574435627966174278:2115] self: [2:7574435627966174277:0] peer: [6:7574435628575480724:0] socket: 85 qp: -1
2025-11-19T13:58:42.501548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435627966174298:2110] [node 6] ICS10 traffic start
2025-11-19T13:58:42.501563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.502130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435628575480725:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60261
2025-11-19T13:58:42.502301Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435628575480721:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:42.502365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435628575480637:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.502384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:42.502389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435628575480637:2049] [node 2] ICP22 created new session: [6:7574435628575480726:2049]
2025-11-19T13:58:42.502404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435628575480726:2049] [node 2] ICS09 handshake done sender: [6:7574435628575480721:2118] self: [6:7574435628575480724:0] peer: [2:7574435627966174277:0] socket: 86 qp: -1
2025-11-19T13:58:42.502409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435628575480726:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:42.502414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:42.503833Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435625807512423:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.503843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435625807512423:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:42.504570Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435625807512412:2107] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.505111Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435628575480722:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.505162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435628575480728:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60261
2025-11-19T13:58:42.505178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435628575480655:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.505500Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435625807512421:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.505610Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435628569429141:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.505621Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435628569429141:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:42.506191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435628569429160:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50189
2025-11-19T13:58:42.506398Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435625807512423:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:42.506486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435621512545055:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.506523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:42.506528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435621512545055:2048] [node 2] ICP22 created new session: [1:7574435625807512442:2048]
2025-11-19T13:58:42.506534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435625807512442:2048] [node 2] ICS09 handshake done sender: [1:7574435625807512412:2107] self: [1:7574435625807512411:0] peer: [2:7574435627966174295:0] socket: 36 qp: -1
2025-11-19T13:58:42.506539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435625807512442:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:42.506547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:42.506692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435625807512417:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.506702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:42.506706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435625807512417:2108] [node 4] ICP22 created new session: [1:7574435625807512444:2108]
2025-11-19T13:58:42.506710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435625807512444:2108] [node 4] ICS09 handshake done sender: [1:7574435625807512421:2111] self: [1:7574435625807512420:0] peer: [4:7574435628569429159:0] socket: 65 qp: -1
2025-11-19T13:58:42.506713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435625807512444:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:42.506716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:42.506785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435625807512418:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.506793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:42.506807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435625807512418:2109] [node 6] ICP22 created new session: [1:7574435625807512446:2109]
2025-11-19T13:58:42.506816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435625807512446:2109] [node 6] ICS09 handshake done sender: [1:7574435625807512423:2112] self: [1:7574435625807512422:0] peer: [6:7574435628575480723:0] socket: 66 qp: -1
2025-11-19T13:58:42.506819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435625807512446:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:42.506827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:42.505384Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435627966174262:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:42.507199Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435627697177285:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.507210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435627697177285:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:42.507714Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435623402209922:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19562) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435627697177297:2116] held: no
2025-11-19T13:58:42.508218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435623671206906:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.508233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:42.508238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435623671206906:2048] [node 1] ICP22 created new session: [2:7574435627966174300:2048]
2025-11-19T13:58:42.508242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435627966174300:2048] [node 1] ICS09 handshake done sender: [2:7574435627966174262:2107] self: [2:7574435627966174295:0] peer: [1:7574435625807512411:0] socket: 41 qp: -1
2025-11-19T13:58:42.508247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435627966174300:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:42.508253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:42.510206Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435628575480716:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.510603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435628575480730:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60261
2025-11-19T13:58:42.510617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435628575480636:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.510627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:42.510632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435628575480636:2048] [node 1] ICP22 created new session: [6:7574435628575480731:2048]
2025-11-19T13:58:42.510638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435628575480731:2048] [node 1] ICS09 handshake done sender: [6:7574435628575480716:2116] self: [6:7574435628575480723:0] peer: [1:7574435625807512422:0] socket: 73 qp: -1
2025-11-19T13:58:42.510643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435628575480731:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:42.510658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.514421Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435627989276830:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.514555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435627989276835:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50033
2025-11-19T13:58:42.514861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435627989276836:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50033
2025-11-19T13:58:42.514911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435627989276744:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.514955Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435627989276820:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:42.515895Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435627697177299:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.515967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435627697177306:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49232
2025-11-19T13:58:42.516002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435627697177307:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49232
2025-11-19T13:58:42.516085Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435627966174276:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.516093Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435627966174276:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:42.517981Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435627697177297:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.518048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435627697177282:2108] [node 5] ICP18 handshake (actor [3:7574435627697177299:2118]) is too old
2025-11-19T13:58:42.518071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435627697177309:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49232
2025-11-19T13:58:42.518138Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435627697177285:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:42.518213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435623402209922:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.518234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:42.518240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435623402209922:2049] [node 2] ICP22 created new session: [3:7574435627697177311:2049]
2025-11-19T13:58:42.518245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435627697177311:2049] [node 2] ICS09 handshake done sender: [3:7574435627697177297:2116] self: [3:7574435627697177300:0] peer: [2:7574435627966174269:0] socket: 68 qp: -1
2025-11-19T13:58:42.518250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435627697177311:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:42.518256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:42.519775Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435628569429141:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:42.518502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435627697177281:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.518514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:42.518517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435627697177281:2107] [node 4] ICP22 created new session: [3:7574435627697177313:2107]
2025-11-19T13:58:42.518521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435627697177313:2107] [node 4] ICS09 handshake done sender: [3:7574435627697177285:2111] self: [3:7574435627697177284:0] peer: [4:7574435628569429156:0] socket: 42 qp: -1
2025-11-19T13:58:42.518530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435627697177313:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:42.518538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:42.519916Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435627966174276:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:42.520105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435627966174267:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.520126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:42.520132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435627966174267:2109] [node 5] ICP22 created new session: [2:7574435627966174303:2109]
2025-11-19T13:58:42.520138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435627966174303:2109] [node 5] ICS09 handshake done sender: [2:7574435627966174276:2114] self: [2:7574435627966174275:0] peer: [5:7574435627989276837:0] socket: 79 qp: -1
2025-11-19T13:58:42.520142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435627966174303:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:42.519930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435627989276745:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.519950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:42.519956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435627989276745:2050] [node 3] ICP22 created new session: [5:7574435627989276838:2050]
2025-11-19T13:58:42.519961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435627989276838:2050] [node 3] ICS09 handshake done sender: [5:7574435627989276820:2117] self: [5:7574435627989276834:0] peer: [3:7574435627697177286:0] socket: 53 qp: -1
2025-11-19T13:58:42.519967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435627989276838:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:42.519984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:42.520236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:42.520405Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435628569429147:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:42.520514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435624274461770:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9074) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435628569429147:2118] held: no
2025-11-19T13:58:42.520531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435624274461770:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.520548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:42.520553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435624274461770:2048] [node 1] ICP22 created new session: [4:7574435628569429162:2048]
2025-11-19T13:58:42.520558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435628569429162:2048] [node 1] ICS09 handshake done sender: [4:7574435628569429147:2118] self: [4:7574435628569429159:0] peer: [1:7574435625807512420:0] socket: 71 qp: -1
2025-11-19T13:58:42.520563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435628569429162:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:42.520569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:42.521794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435627989276745:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.526640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435628569429133:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.526662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:42.526667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435628569429133:2109] [node 6] ICP22 created new session: [4:7574435628569429165:2109]
2025-11-19T13:58:42.526672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435628569429165:2109] [node 6] ICS09 handshake done sender: [4:7574435628569429141:2114] self: [4:7574435628569429140:0] peer: [6:7574435628575480729:0] socket: 77 qp: -1
2025-11-19T13:58:42.526677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435628569429165:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:42.526684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:42.526860Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435624274461772:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20672) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435628569429142:2115] held: no
2025-11-19T13:58:42.527328Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435627697177306:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.527393Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435627697177309:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.527430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435623402209921:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.527581Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435625807512436:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.527613Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435625807512427:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.527616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435625807512427:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:42.527261Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435627989276836:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.527285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435627989276744:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19562) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435627989276830:2119] held: no
2025-11-19T13:58:42.527321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435627989276840:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50033
2025-11-19T13:58:42.532297Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435628575480722:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.532773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435627989276743:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.532790Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435627989276830:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:42.532892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435627989276744:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.532905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:42.532910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435627989276744:2049] [node 2] ICP22 created new session: [5:7574435627989276842:2049]
2025-11-19T13:58:42.532915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435627989276842:2049] [node 2] ICS09 handshake done sender: [5:7574435627989276830:2119] self: [5:7574435627989276837:0] peer: [2:7574435627966174275:0] socket: 38 qp: -1
2025-11-19T13:58:42.532922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435627989276842:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:42.532928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:42.532950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435627697177317:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49232
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.533214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435628575480733:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60261
2025-11-19T13:58:42.533253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435628575480655:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.533261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:42.533265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435628575480655:2062] [node 4] ICP22 created new session: [6:7574435628575480734:2062]
2025-11-19T13:58:42.533270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435628575480734:2062] [node 4] ICS09 handshake done sender: [6:7574435628575480722:2119] self: [6:7574435628575480729:0] peer: [4:7574435628569429140:0] socket: 89 qp: -1
2025-11-19T13:58:42.533274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435628575480734:2062] [node 4] ICS10 traffic start
2025-11-19T13:58:42.533279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:42.534094Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435625807512427:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:42.534249Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435625807512425:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.534257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435625807512425:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:42.534678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435621512545056:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.534690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:42.534695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435621512545056:2049] [node 3] ICP22 created new session: [1:7574435625807512452:2049]
2025-11-19T13:58:42.534699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435625807512452:2049] [node 3] ICS09 handshake done sender: [1:7574435625807512427:2114] self: [1:7574435625807512426:0] peer: [3:7574435627697177316:0] socket: 62 qp: -1
2025-11-19T13:58:42.534704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435625807512452:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:42.534711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.535268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435627989276844:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50033
2025-11-19T13:58:42.535357Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435625807512425:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:42.535494Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435627989276836:2121] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:42.535543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435627989276743:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.535553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:42.535557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435627989276743:2048] [node 1] ICP22 created new session: [5:7574435627989276845:2048]
2025-11-19T13:58:42.535561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435627989276845:2048] [node 1] ICS09 handshake done sender: [5:7574435627989276836:2121] self: [5:7574435627989276841:0] peer: [1:7574435625807512424:0] socket: 64 qp: -1
2025-11-19T13:58:42.535565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435627989276845:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:42.535580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:42.536655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435625807512419:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.536671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:42.536676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435625807512419:2110] [node 5] ICP22 created new session: [1:7574435625807512455:2110]
2025-11-19T13:58:42.536680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435625807512455:2110] [node 5] ICS09 handshake done sender: [1:7574435625807512425:2113] self: [1:7574435625807512424:0] peer: [5:7574435627989276841:0] socket: 48 qp: -1
2025-11-19T13:58:42.536684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435625807512455:2110] [node 5] ICS10 traffic start
2025-11-19T13:58:42.536699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.538145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435628569429164:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50189
2025-11-19T13:58:42.542141Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435628575480733:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.542228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435628575480638:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:42.543517Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435628569429142:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:42.543596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435624274461772:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.543625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:42.543632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435624274461772:2050] [node 3] ICP22 created new session: [4:7574435628569429167:2050]
2025-11-19T13:58:42.543638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435628569429167:2050] [node 3] ICS09 handshake done sender: [4:7574435628569429142:2115] self: [4:7574435628569429156:0] peer: [3:7574435627697177284:0] socket: 49 qp: -1
2025-11-19T13:58:42.543646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435628569429167:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:42.543653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:42.544937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435623402209921:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9074) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435627697177309:2121] held: no
2025-11-19T13:58:42.545016Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435627697177293:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:42.545021Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435627697177293:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:42.545756Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435627697177309:2121] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:42.550198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435628575480737:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60261
2025-11-19T13:58:42.552029Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435628575480733:2123] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:42.552124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435628575480638:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.552142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:42.552150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435628575480638:2050] [node 3] ICP22 created new session: [6:7574435628575480738:2050]
2025-11-19T13:58:42.552156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435628575480738:2050] [node 3] ICS09 handshake done sender: [6:7574435628575480733:2123] self: [6:7574435628575480736:0] peer: [3:7574435627697177292:0] socket: 56 qp: -1
2025-11-19T13:58:42.552163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435628575480738:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:42.552184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:42.549584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435623402209921:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:42.549618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:42.549628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435623402209921:2048] [node 1] ICP22 created new session: [3:7574435627697177319:2048]
2025-11-19T13:58:42.549636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435627697177319:2048] [node 1] ICS09 handshake done sender: [3:7574435627697177309:2121] self: [3:7574435627697177316:0] peer: [1:7574435625807512426:0] socket: 99 qp: -1
2025-11-19T13:58:42.549645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435627697177319:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:42.549656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:42.551742Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435627697177293:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:42.556064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435627697177283:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:42.556088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:42.556095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435627697177283:2109] [node 6] ICP22 created new session: [3:7574435627697177321:2109]
2025-11-19T13:58:42.556101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435627697177321:2109] [node 6] ICS09 handshake done sender: [3:7574435627697177293:2115] self: [3:7574435627697177292:0] peer: [6:7574435628575480736:0] socket: 101 qp: -1
2025-11-19T13:58:42.556107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435627697177321:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:42.556139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:42.560805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:42.560874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435627697177319:2048] [node 1] ICS01 socket: 99 reason# EndOfStream
2025-11-19T13:58:42.560879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435623402209921:2048] [node 1] ICP30 unregister session Session# [3:7574435627697177319:2048] VirtualId# [3:7574435627697177316:0]
2025-11-19T13:58:42.560884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435627697177319:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:42.560930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:42.560950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:42.563046Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:42.563119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435628569429162:2048] [node 1] ICS01 socket: 71 reason# EndOfStream
2025-11-19T13:58:42.563126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435624274461770:2048] [node 1] ICP30 unregister session Session# [4:7574435628569429162:2048] VirtualId# [4:7574435628569429159:0]
2025-11-19T13:58:42.563131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435628569429162:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:42.563180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:42.563205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:42.560238Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:42.560334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435627966174300:2048] [node 1] ICS01 socket: 41 reason# EndOfStream
2025-11-19T13:58:42.560344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435623671206906:2048] [node 1] ICP30 unregister session Session# [2:7574435627966174300:2048] VirtualId# [2:7574435627966174295:0]
2025-11-19T13:58:42.560350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435627966174300:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:42.560396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:42.560418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:42.565353Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:42.565427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435628575480731:2048] [node 1] ICS01 socket: 73 reason# EndOfStream
2025-11-19T13:58:42.565434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435628575480636:2048] [node 1] ICP30 unregister session Session# [6:7574435628575480731:2048] VirtualId# [6:7574435628575480723:0]
2025-11-19T13:58:42.565440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435628575480731:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:42.565481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:42.565503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:42.567794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:42.567874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435627989276845:2048] [node 1] ICS01 socket: 64 reason# EndOfStream
2025-11-19T13:58:42.567880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435627989276743:2048] [node 1] ICP30 unregister session Session# [5:7574435627989276845:2048] VirtualId# [5:7574435627989276841:0]
2025-11-19T13:58:42.567885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435627989276845:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:42.567931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:42.567955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:42.575556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:42.575666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435627697177311:2049] [node 2] ICS01 socket: 68 reason# EndOfStream
2025-11-19T13:58:42.575676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435623402209922:2049] [node 2] ICP30 unregister session Session# [3:7574435627697177311:2049] VirtualId# [3:7574435627697177300:0]
2025-11-19T13:58:42.575684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435627697177311:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:42.575729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:42.575767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:42.590586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:42.590692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435628575480726:2049] [node 2] ICS01 socket: 86 reason# EndOfStream
2025-11-19T13:58:42.590703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435628575480637:2049] [node 2] ICP30 unregister session Session# [6:7574435628575480726:2049] VirtualId# [6:7574435628575480724:0]
2025-11-19T13:58:42.590711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435628575480726:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:42.590759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:42.590781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:42.590868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:42.590925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435627989276842:2049] [node 2] ICS01 socket: 38 reason# EndOfStream
2025-11-19T13:58:42.590929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435627989276744:2049] [node 2] ICP30 unregister session Session# [5:7574435627989276842:2049] VirtualId# [5:7574435627989276837:0]
2025-11-19T13:58:42.590933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435627989276842:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:42.590957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:42.590977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:42.592062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:42.592161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435628569429154:2049] [node 2] ICS07 socket disconnect 50 reason# EndOfStream
2025-11-19T13:58:42.592167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435628569429154:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:42.592211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:42.592232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435628569429154:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:42.595475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435624274461771:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19562) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:42.595494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435628569429154:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:42.595499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435624274461771:2049] [node 2] ICP30 unregister session Session# [4:7574435628569429154:2049] VirtualId# [4:7574435628569429149:0]
2025-11-19T13:58:42.595517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:42.595523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435624274461771:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19562) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:42.595526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19562) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:42.604228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435624274461771:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:58:42.610600Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:42.610661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435628575480738:2050] [node 3] ICS01 socket: 56 reason# EndOfStream
2025-11-19T13:58:42.610665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435628575480638:2050] [node 3] ICP30 unregister session Session# [6:7574435628575480738:2050] VirtualId# [6:7574435628575480736:0]
2025-11-19T13:58:42.610670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435628575480738:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:42.610701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:42.610721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:42.610087Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:42.610146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435627989276838:2050] [node 3] ICS01 socket: 53 reason# EndOfStream
2025-11-19T13:58:42.610150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435627989276745:2050] [node 3] ICP30 unregister session Session# [5:7574435627989276838:2050] VirtualId# [5:7574435627989276834:0]
2025-11-19T13:58:42.610153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435627989276838:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:42.610176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:42.610196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:42.628755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:42.628877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435628575480734:2062] [node 4] ICS01 socket: 89 reason# EndOfStream
2025-11-19T13:58:42.628889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435628575480655:2062] [node 4] ICP30 unregister session Session# [6:7574435628575480734:2062] VirtualId# [6:7574435628575480729:0]
2025-11-19T13:58:42.628898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435628575480734:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:42.628951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:42.628977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:42.827869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435628148407774:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:42.827893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435628148407775:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:42.895941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435628229999099:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:42.895956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435628229999100:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:43.023295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435631281714476:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:43.023311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435631281714477:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:43.056067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435632424097818:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:43.056101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435632424097819:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:43.056105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435632424097820:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:43.079955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435631741602637:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:43.079975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435631741602638:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:43.079979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435631741602639:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:43.135860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435633478791526:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:43.135884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435633478791527:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:43.135889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435633478791528:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:43.196132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435633478791557:2073] [node 5] ICP01 ready to work
2025-11-19T13:58:43.196191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435633478791558:2077] [node 4] ICP01 ready to work
2025-11-19T13:58:43.196145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435632424097873:2101] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.200204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435632424097877:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.205417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435631741602696:2095] [node 4] ICP01 ready to work
2025-11-19T13:58:43.206141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435631741602694:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64776
2025-11-19T13:58:43.206196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435631741602695:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64776
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.207985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435632524966455:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10831
2025-11-19T13:58:43.211370Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435631741602694:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.211882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435632524966456:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10831
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.215718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435631281714541:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:43.215816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435631281714542:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:43.216831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435631281714539:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20720
2025-11-19T13:58:43.216847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435631281714540:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20720
2025-11-19T13:58:43.214495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435632443375138:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:43.214605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435632443375139:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.222321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435632424097884:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
2025-11-19T13:58:43.222349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435632424097885:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
2025-11-19T13:58:43.223185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435632424097883:2105] [node 6] ICP01 ready to work
2025-11-19T13:58:43.224015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435631741602704:2091] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.232128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435632443375144:2111] [node 4] ICP01 ready to work
2025-11-19T13:58:43.234608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435632443375145:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43300
2025-11-19T13:58:43.235002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435631741602637:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23568597ns) {Configure:104(23567014ns) {ProcessConfigured:135(23284307ns) {StartInitialHandshake:157(23283624ns) {DropHandshakes:469(1049ns) {DropIncomingHandshake:441(227ns) DropOutgoingHandshake:456(163ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:476(1645ns) {SwitchToState:178(1151ns)}}}}}
2025-11-19T13:58:43.235413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435631741602696:2095] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.236241Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435632443375143:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.236251Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435632443375143:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:43.237081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435632443375149:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43300
2025-11-19T13:58:43.237260Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435632443375143:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:43.235866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435631741602711:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64776
2025-11-19T13:58:43.235908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435631741602712:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64776
2025-11-19T13:58:43.236097Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435631741602712:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.236126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435631741602637:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.236706Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435631741602711:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.237355Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435631281714540:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.237385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435631281714476:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.239867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435631281714554:2107] [node 6] ICP01 ready to work
2025-11-19T13:58:43.243444Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435632424097875:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.243462Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435632424097875:2102] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:43.245671Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435632424097875:2102] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:43.245795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435632424097873:2101] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.245813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:43.245834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435632424097873:2101] [node 5] ICP22 created new session: [4:7574435632424097899:2101]
2025-11-19T13:58:43.245840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435632424097899:2101] [node 5] ICS09 handshake done sender: [4:7574435632424097875:2102] self: [4:7574435632424097874:0] peer: [5:7574435631741602715:0] socket: 42 qp: -1
2025-11-19T13:58:43.245848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435632424097899:2101] [node 5] ICS10 traffic start
2025-11-19T13:58:43.245857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.248015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435631741602639:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.248279Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435632424097877:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.252301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435632443375139:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.252328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:43.252337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435632443375139:2110] [node 5] ICP22 created new session: [1:7574435632443375152:2110]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.254211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435632524966460:2110] [node 5] ICP01 ready to work
2025-11-19T13:58:43.252466Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435632424097885:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.252343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435632443375152:2110] [node 5] ICS09 handshake done sender: [1:7574435632443375143:2112] self: [1:7574435632443375142:0] peer: [5:7574435631741602719:0] socket: 64 qp: -1
2025-11-19T13:58:43.252352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435632443375152:2110] [node 5] ICS10 traffic start
2025-11-19T13:58:43.252365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:43.252365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435632443375138:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(33366826ns) {Configure:104(33365790ns) {ProcessConfigured:135(33091017ns) {StartInitialHandshake:157(33089974ns) {DropHandshakes:469(1257ns) {DropIncomingHandshake:441(154ns) DropOutgoingHandshake:456(178ns)} GenerateSessionVirtualId:988(91ns) PrepareNewSessionHandshake:476(1068ns) {SwitchToState:178(459ns)}}}}}
2025-11-19T13:58:43.259481Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435632443375133:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.259492Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435632443375133:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:43.264972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435632424097820:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.271863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435632424097883:2105] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(48586583ns) {Configure:104(48584854ns) {ProcessConfigured:135(34729286ns) {StartInitialHandshake:157(34727984ns) {DropHandshakes:469(673ns) {DropIncomingHandshake:441(250ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(511ns) PrepareNewSessionHandshake:476(1056ns) {SwitchToState:178(184ns)}}}}}
2025-11-19T13:58:43.258704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435631281714557:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20720
2025-11-19T13:58:43.259982Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435631281714539:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.252395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435631741602721:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64776
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.283090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435632443375139:2110] [node 5] ICP35 event processing took too much time PendingConnection:278(30737603ns) {HandleHandshakeStatus:329(30736690ns) {DropIncomingHandshake:441(78ns) DropOutgoingHandshake:456(554ns) SwitchToState:178(261ns) ProcessPendingSessionEvents:495(23671556ns) {ForwardSessionEventToSession:612(23654840ns) ForwardSessionEventToSession:612(3169ns) ForwardSessionEventToSession:612(890ns) ForwardSessionEventToSession:612(663ns) ForwardSessionEventToSession:612(598ns) ForwardSessionEventToSession:612(1752ns) ForwardSessionEventToSession:612(844ns) ForwardSessionEventToSession:612(512ns) ForwardSessionEventToSession:612(685ns)}}}
2025-11-19T13:58:43.287792Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435631281714546:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.287818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435631281714546:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:43.288217Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435631281714557:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.288865Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435631281714546:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:43.288958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435631281714541:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.288976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:43.288984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435631281714541:2108] [node 4] ICP22 created new session: [3:7574435631281714560:2108]
2025-11-19T13:58:43.288991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435631281714560:2108] [node 4] ICS09 handshake done sender: [3:7574435631281714546:2113] self: [3:7574435631281714545:0] peer: [4:7574435632424097902:0] socket: 59 qp: -1
2025-11-19T13:58:43.288999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435631281714560:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:43.289009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:43.290581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435632424097820:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61520) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435632424097885:2110] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.290846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435633478791604:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.292036Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435631741602695:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.294347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435631741602722:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64776
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.294574Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435632443375145:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.294799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435632443375155:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43300
2025-11-19T13:58:43.298342Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435632443375133:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:43.298439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435628148407775:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.298454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:43.298461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435628148407775:2049] [node 3] ICP22 created new session: [1:7574435632443375158:2049]
2025-11-19T13:58:43.298468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435632443375158:2049] [node 3] ICS09 handshake done sender: [1:7574435632443375133:2108] self: [1:7574435632443375132:0] peer: [3:7574435631281714555:0] socket: 47 qp: -1
2025-11-19T13:58:43.298477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435632443375158:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:43.298497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:43.298007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435631281714554:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(58031682ns) {Configure:104(58030386ns) {ProcessConfigured:135(43857404ns) {StartInitialHandshake:157(43856145ns) {DropHandshakes:469(1087ns) {DropIncomingHandshake:441(155ns) DropOutgoingHandshake:456(146ns)} GenerateSessionVirtualId:988(702ns) PrepareNewSessionHandshake:476(977ns) {SwitchToState:178(263ns)}}}}}
2025-11-19T13:58:43.298071Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435631281714548:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.298074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435631281714548:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:43.298587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435631281714564:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20720
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.300078Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435631281714548:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:43.300329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435631281714542:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.300343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:43.300349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435631281714542:2109] [node 5] ICP22 created new session: [3:7574435631281714567:2109]
2025-11-19T13:58:43.300355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435631281714567:2109] [node 5] ICS09 handshake done sender: [3:7574435631281714548:2114] self: [3:7574435631281714547:0] peer: [5:7574435631741602723:0] socket: 60 qp: -1
2025-11-19T13:58:43.300360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435631281714567:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:43.300378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:43.300717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435632524966461:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:43.297919Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435631741602712:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:43.300895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435632524966462:2111] [node 4] ICP01 ready to work
2025-11-19T13:58:43.300635Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435631281714540:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:43.301214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435631281714476:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.301225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:43.301230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435631281714476:2048] [node 1] ICP22 created new session: [3:7574435631281714569:2048]
2025-11-19T13:58:43.301237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435631281714569:2048] [node 1] ICS09 handshake done sender: [3:7574435631281714540:2111] self: [3:7574435631281714555:0] peer: [1:7574435632443375132:0] socket: 57 qp: -1
2025-11-19T13:58:43.301241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435631281714569:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:43.301249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:43.303090Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435632443375155:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.303121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435628148407775:2049] [node 3] ICP18 handshake (actor [1:7574435632443375155:2117]) is too old
2025-11-19T13:58:43.306934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435631741602637:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.306969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:43.306976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435631741602637:2048] [node 1] ICP22 created new session: [5:7574435631741602724:2048]
2025-11-19T13:58:43.306984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435631741602724:2048] [node 1] ICS09 handshake done sender: [5:7574435631741602712:2115] self: [5:7574435631741602719:0] peer: [1:7574435632443375142:0] socket: 66 qp: -1
2025-11-19T13:58:43.306998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435631741602724:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:43.307019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:43.299954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435632424097906:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
2025-11-19T13:58:43.301035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435632424097907:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
2025-11-19T13:58:43.308180Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435632424097906:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.308255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435632424097818:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.308392Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435633478791604:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.308421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435633478791558:2077] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.310052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435632443375147:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.310064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435632443375147:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:43.310894Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435632443375147:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:43.311352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435632443375144:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.311370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:43.311377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435632443375144:2111] [node 4] ICP22 created new session: [1:7574435632443375162:2111]
2025-11-19T13:58:43.311381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435632443375162:2111] [node 4] ICS09 handshake done sender: [1:7574435632443375147:2114] self: [1:7574435632443375146:0] peer: [4:7574435632424097908:0] socket: 76 qp: -1
2025-11-19T13:58:43.311386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435632443375162:2111] [node 4] ICS10 traffic start
2025-11-19T13:58:43.311392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:43.307128Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435631741602694:2104] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.308899Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435632424097885:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:43.309004Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435632424097884:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.309042Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435632424097903:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.309046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435632424097903:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:43.309529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435632424097820:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.309539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:43.309543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435632424097820:2050] [node 3] ICP22 created new session: [4:7574435632424097909:2050]
2025-11-19T13:58:43.309548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435632424097909:2050] [node 3] ICS09 handshake done sender: [4:7574435632424097885:2110] self: [4:7574435632424097902:0] peer: [3:7574435631281714545:0] socket: 61 qp: -1
2025-11-19T13:58:43.309554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435632424097909:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:43.309560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:43.309792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435632424097873:2101] [node 5] ICP18 handshake (actor [4:7574435632424097884:2109]) is too old
2025-11-19T13:58:43.310824Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435632424097903:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:43.311035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435632424097883:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.311046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:43.311052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435632424097883:2105] [node 6] ICP22 created new session: [4:7574435632424097913:2105]
2025-11-19T13:58:43.311057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435632424097913:2105] [node 6] ICS09 handshake done sender: [4:7574435632424097903:2117] self: [4:7574435632424097889:0] peer: [6:7574435633478791605:0] socket: 73 qp: -1
2025-11-19T13:58:43.311061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435632424097913:2105] [node 6] ICS10 traffic start
2025-11-19T13:58:43.311067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:43.311900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435632424097912:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
2025-11-19T13:58:43.314594Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435632443375139:2110] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:43.313827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435631741602696:2095] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.313846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:43.313853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435631741602696:2095] [node 4] ICP22 created new session: [5:7574435631741602728:2095]
2025-11-19T13:58:43.313859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435631741602728:2095] [node 4] ICS09 handshake done sender: [5:7574435631741602694:2104] self: [5:7574435631741602715:0] peer: [4:7574435632424097874:0] socket: 48 qp: -1
2025-11-19T13:58:43.313864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435631741602728:2095] [node 4] ICS10 traffic start
2025-11-19T13:58:43.313870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:43.316299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435633478791558:2077] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20966) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435633478791604:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.317579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435632443375164:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43300
2025-11-19T13:58:43.319953Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435632424097906:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:43.320049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435632424097818:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435632424097906:2118] held: no
2025-11-19T13:58:43.320070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435632424097818:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.320082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:43.320087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435632424097818:2048] [node 1] ICP22 created new session: [4:7574435632424097915:2048]
2025-11-19T13:58:43.320092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435632424097915:2048] [node 1] ICS09 handshake done sender: [4:7574435632424097906:2118] self: [4:7574435632424097908:0] peer: [1:7574435632443375146:0] socket: 77 qp: -1
2025-11-19T13:58:43.320098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435632424097915:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:43.320107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.321110Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435628229999099:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(66810202ns) {Configure:104(66809185ns) {ProcessConfigured:135(13837014ns) {StartInitialHandshake:157(13835342ns) {DropHandshakes:469(543ns) {DropIncomingHandshake:441(31ns) DropOutgoingHandshake:456(14ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:476(802ns) {SwitchToState:178(236ns)}}}}}
2025-11-19T13:58:43.322519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435632524966476:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10831
2025-11-19T13:58:43.322568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435632524966477:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10831
2025-11-19T13:58:43.322782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435631741602696:2095] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:43.328514Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435632443375164:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.331890Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435632524966455:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.331926Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435632524966476:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.331944Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435632524966477:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.331984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435628229999099:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.340429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435632443375131:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.340447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435632443375131:2107] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.343413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435631741602726:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64776
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.343526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435632443375166:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43300
2025-11-19T13:58:43.343941Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435632524966456:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.349496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435631281714572:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20720
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.349776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435633478791607:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.348244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435631741602730:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64776
2025-11-19T13:58:43.349841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435633478791608:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-19T13:58:43.349891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435633478791609:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-19T13:58:43.349932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435633478791610:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-19T13:58:43.349945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435633478791611:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-19T13:58:43.350497Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435633478791607:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.350530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435633478791526:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.358771Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435632443375131:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:43.358880Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435633478791609:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.358923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435633478791528:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.359029Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435631281714563:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.359038Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435631281714563:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:43.359780Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435632443375151:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.359788Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435632443375151:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:43.360831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435628148407774:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.360865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:43.360873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435628148407774:2048] [node 2] ICP22 created new session: [1:7574435632443375168:2048]
2025-11-19T13:58:43.360880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435632443375168:2048] [node 2] ICS09 handshake done sender: [1:7574435632443375131:2107] self: [1:7574435632443375130:0] peer: [2:7574435632524966478:0] socket: 46 qp: -1
2025-11-19T13:58:43.360888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435632443375168:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:43.360899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:43.361447Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435633478791604:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.361512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435633478791614:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-19T13:58:43.361525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435633478791615:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-19T13:58:43.361614Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435632443375151:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:43.361748Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435633478791607:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:43.361805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435633478791558:2077] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.361817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:43.361821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435633478791558:2077] [node 4] ICP22 created new session: [6:7574435633478791616:2077]
2025-11-19T13:58:43.361826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435633478791616:2077] [node 4] ICS09 handshake done sender: [6:7574435633478791604:2115] self: [6:7574435633478791605:0] peer: [4:7574435632424097889:0] socket: 79 qp: -1
2025-11-19T13:58:43.361830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435633478791616:2077] [node 4] ICS10 traffic start
2025-11-19T13:58:43.361835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:43.362138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435632443375138:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.362152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:43.362158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435632443375138:2109] [node 6] ICP22 created new session: [1:7574435632443375171:2109]
2025-11-19T13:58:43.362162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435632443375171:2109] [node 6] ICS09 handshake done sender: [1:7574435632443375151:2116] self: [1:7574435632443375140:0] peer: [6:7574435633478791612:0] socket: 84 qp: -1
2025-11-19T13:58:43.362166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435632443375171:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:43.362172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:43.362424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435633478791526:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.362436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:43.362440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435633478791526:2048] [node 1] ICP22 created new session: [6:7574435633478791618:2048]
2025-11-19T13:58:43.362445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435633478791618:2048] [node 1] ICS09 handshake done sender: [6:7574435633478791607:2116] self: [6:7574435633478791612:0] peer: [1:7574435632443375140:0] socket: 86 qp: -1
2025-11-19T13:58:43.362448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435633478791618:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:43.362452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:43.363596Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435631281714563:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:43.364697Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435633478791609:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:43.364827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435631281714554:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.364846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:43.364853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435631281714554:2107] [node 6] ICP22 created new session: [3:7574435631281714574:2107]
2025-11-19T13:58:43.364858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435631281714574:2107] [node 6] ICS09 handshake done sender: [3:7574435631281714563:2117] self: [3:7574435631281714556:0] peer: [6:7574435633478791613:0] socket: 75 qp: -1
2025-11-19T13:58:43.364863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435631281714574:2107] [node 6] ICS10 traffic start
2025-11-19T13:58:43.364870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:43.369032Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435633478791528:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61520) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435633478791609:2118] held: no
2025-11-19T13:58:43.369082Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435633478791611:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.369116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435633478791557:2073] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.369255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435633478791528:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.369268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:43.369272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435633478791528:2050] [node 3] ICP22 created new session: [6:7574435633478791622:2050]
2025-11-19T13:58:43.369277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435633478791622:2050] [node 3] ICS09 handshake done sender: [6:7574435633478791609:2118] self: [6:7574435633478791613:0] peer: [3:7574435631281714556:0] socket: 95 qp: -1
2025-11-19T13:58:43.369282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435633478791622:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:43.369287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:43.369606Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435631741602714:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.369616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435631741602714:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:43.370142Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435631281714572:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.370170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435631281714542:2109] [node 5] ICP18 handshake (actor [3:7574435631281714572:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.370466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435633478791624:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
2025-11-19T13:58:43.370572Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435631741602714:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:43.370711Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435633478791611:2120] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:43.370784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435633478791557:2073] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.370796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:43.370801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435633478791557:2073] [node 5] ICP22 created new session: [6:7574435633478791625:2073]
2025-11-19T13:58:43.370806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435633478791625:2073] [node 5] ICS09 handshake done sender: [6:7574435633478791611:2120] self: [6:7574435633478791621:0] peer: [5:7574435631741602713:0] socket: 97 qp: -1
2025-11-19T13:58:43.370810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435633478791625:2073] [node 5] ICS10 traffic start
2025-11-19T13:58:43.370815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:43.372630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435631741602704:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.372650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:43.372655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435631741602704:2091] [node 6] ICP22 created new session: [5:7574435631741602734:2091]
2025-11-19T13:58:43.372660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435631741602734:2091] [node 6] ICS09 handshake done sender: [5:7574435631741602714:2116] self: [5:7574435631741602713:0] peer: [6:7574435633478791621:0] socket: 91 qp: -1
2025-11-19T13:58:43.372665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435631741602734:2091] [node 6] ICS10 traffic start
2025-11-19T13:58:43.372671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:43.372803Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435631741602639:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61520) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435631741602711:2114] held: no
2025-11-19T13:58:43.388015Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435632443375166:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.388074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435628148407774:2048] [node 2] ICP18 handshake (actor [1:7574435632443375166:2119]) is too old
2025-11-19T13:58:43.401516Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435631741602711:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:43.401912Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435631741602730:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.401945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435631741602638:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.402112Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435633478791610:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.402143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435633478791527:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.402329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435631741602639:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.402349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:43.402357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435631741602639:2050] [node 3] ICP22 created new session: [5:7574435631741602737:2050]
2025-11-19T13:58:43.402364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435631741602737:2050] [node 3] ICS09 handshake done sender: [5:7574435631741602711:2114] self: [5:7574435631741602723:0] peer: [3:7574435631281714547:0] socket: 65 qp: -1
2025-11-19T13:58:43.402374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435631741602737:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:43.402382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:43.403009Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435632524966466:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.403022Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435632524966466:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:43.403561Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435632524966468:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.403565Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435632524966468:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.405977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435628229999099:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435632524966455:2107] held: no
2025-11-19T13:58:43.406167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435632524966482:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10831
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.406715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435632524966483:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10831
2025-11-19T13:58:43.406798Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435632524966455:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:43.406989Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435632524966466:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:43.408326Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435632524966468:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:43.408402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435628229999099:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.408416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:43.408422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435628229999099:2048] [node 1] ICP22 created new session: [2:7574435632524966487:2048]
2025-11-19T13:58:43.408427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435632524966487:2048] [node 1] ICS09 handshake done sender: [2:7574435632524966455:2107] self: [2:7574435632524966478:0] peer: [1:7574435632443375130:0] socket: 49 qp: -1
2025-11-19T13:58:43.408432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435632524966487:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:43.408436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:43.408540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435632524966460:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.408550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:43.408553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435632524966460:2110] [node 5] ICP22 created new session: [2:7574435632524966489:2110]
2025-11-19T13:58:43.408557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435632524966489:2110] [node 5] ICS09 handshake done sender: [2:7574435632524966466:2113] self: [2:7574435632524966465:0] peer: [5:7574435631741602736:0] socket: 63 qp: -1
2025-11-19T13:58:43.408560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435632524966489:2110] [node 5] ICS10 traffic start
2025-11-19T13:58:43.408564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:43.408655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435632524966461:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.408660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:43.408668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435632524966461:2109] [node 6] ICP22 created new session: [2:7574435632524966491:2109]
2025-11-19T13:58:43.408671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435632524966491:2109] [node 6] ICS09 handshake done sender: [2:7574435632524966468:2114] self: [2:7574435632524966467:0] peer: [6:7574435633478791627:0] socket: 67 qp: -1
2025-11-19T13:58:43.408674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435632524966491:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:43.408679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.410539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435631281714576:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20720
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.406499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435631741602739:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64776
2025-11-19T13:58:43.407147Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435631741602730:2122] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:43.407204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435631741602638:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.407218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:43.407224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435631741602638:2049] [node 2] ICP22 created new session: [5:7574435631741602740:2049]
2025-11-19T13:58:43.407230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435631741602740:2049] [node 2] ICS09 handshake done sender: [5:7574435631741602730:2122] self: [5:7574435631741602736:0] peer: [2:7574435632524966465:0] socket: 54 qp: -1
2025-11-19T13:58:43.407236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435631741602740:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:43.407245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:43.411995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435633478791628:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13138
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.412231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435632424097917:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
2025-11-19T13:58:43.415405Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435633478791610:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:43.416244Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435632424097917:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.416292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435632424097819:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.416130Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435632524966483:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.416506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435633478791527:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.416528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:43.416535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435633478791527:2049] [node 2] ICP22 created new session: [6:7574435633478791629:2049]
2025-11-19T13:58:43.416541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435633478791629:2049] [node 2] ICS09 handshake done sender: [6:7574435633478791610:2119] self: [6:7574435633478791627:0] peer: [2:7574435632524966467:0] socket: 96 qp: -1
2025-11-19T13:58:43.416549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435633478791629:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:43.416558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:43.417700Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435632524966470:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.417712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435632524966470:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:43.418247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435632524966461:2109] [node 6] ICP18 handshake (actor [2:7574435632524966483:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.424216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435632424097919:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58961
2025-11-19T13:58:43.424354Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435632524966470:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:43.424530Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435632424097917:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:43.424613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435632424097819:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.424633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:43.424639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435632424097819:2049] [node 2] ICP22 created new session: [4:7574435632424097920:2049]
2025-11-19T13:58:43.424646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435632424097920:2049] [node 2] ICS09 handshake done sender: [4:7574435632424097917:2121] self: [4:7574435632424097918:0] peer: [2:7574435632524966469:0] socket: 93 qp: -1
2025-11-19T13:58:43.424652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435632424097920:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:43.424658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:43.429560Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435631281714576:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.429607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435631281714477:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:43.429753Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435632524966472:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:43.429755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435632524966462:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.429763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435632524966472:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:43.429770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:43.429777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435632524966462:2111] [node 4] ICP22 created new session: [2:7574435632524966495:2111]
2025-11-19T13:58:43.429783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435632524966495:2111] [node 4] ICS09 handshake done sender: [2:7574435632524966470:2115] self: [2:7574435632524966469:0] peer: [4:7574435632424097918:0] socket: 90 qp: -1
2025-11-19T13:58:43.429791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435632524966495:2111] [node 4] ICS10 traffic start
2025-11-19T13:58:43.429800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:43.430898Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435632524966472:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:43.430918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435631281714578:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20720
2025-11-19T13:58:43.431101Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435631281714576:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:43.430981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435628229999100:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:43.431004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:43.431008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435628229999100:2049] [node 3] ICP22 created new session: [2:7574435632524966498:2049]
2025-11-19T13:58:43.431013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435632524966498:2049] [node 3] ICS09 handshake done sender: [2:7574435632524966472:2116] self: [2:7574435632524966471:0] peer: [3:7574435631281714577:0] socket: 53 qp: -1
2025-11-19T13:58:43.431018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435632524966498:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:43.431024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:43.431224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435631281714477:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:43.431237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:43.431242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435631281714477:2049] [node 2] ICP22 created new session: [3:7574435631281714579:2049]
2025-11-19T13:58:43.431247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435631281714579:2049] [node 2] ICS09 handshake done sender: [3:7574435631281714576:2120] self: [3:7574435631281714577:0] peer: [2:7574435632524966471:0] socket: 55 qp: -1
2025-11-19T13:58:43.431252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435631281714579:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:43.431265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:43.460282Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:43.460390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435632524966487:2048] [node 1] ICS01 socket: 49 reason# EndOfStream
2025-11-19T13:58:43.460402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435628229999099:2048] [node 1] ICP30 unregister session Session# [2:7574435632524966487:2048] VirtualId# [2:7574435632524966478:0]
2025-11-19T13:58:43.460410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435632524966487:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:43.460452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:43.460476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:43.460953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:43.461024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435631281714569:2048] [node 1] ICS01 socket: 57 reason# EndOfStream
2025-11-19T13:58:43.461029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435631281714476:2048] [node 1] ICP30 unregister session Session# [3:7574435631281714569:2048] VirtualId# [3:7574435631281714555:0]
2025-11-19T13:58:43.461034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435631281714569:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:43.461062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:43.461078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:43.475958Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:43.476070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435633478791618:2048] [node 1] ICS01 socket: 86 reason# EndOfStream
2025-11-19T13:58:43.476082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435633478791526:2048] [node 1] ICP30 unregister session Session# [6:7574435633478791618:2048] VirtualId# [6:7574435633478791612:0]
2025-11-19T13:58:43.476091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435633478791618:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:43.476133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:43.476158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:43.476196Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:43.476253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435632424097915:2048] [node 1] ICS01 socket: 77 reason# EndOfStream
2025-11-19T13:58:43.476258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435632424097818:2048] [node 1] ICP30 unregister session Session# [4:7574435632424097915:2048] VirtualId# [4:7574435632424097908:0]
2025-11-19T13:58:43.476261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435632424097915:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:43.476290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:43.476307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:43.476268Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:43.476328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435631741602724:2048] [node 1] ICS01 socket: 66 reason# EndOfStream
2025-11-19T13:58:43.476332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435631741602637:2048] [node 1] ICP30 unregister session Session# [5:7574435631741602724:2048] VirtualId# [5:7574435631741602719:0]
2025-11-19T13:58:43.476336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435631741602724:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:43.476368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:43.476386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:43.491519Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:43.491624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435631281714579:2049] [node 2] ICS07 socket disconnect 55 reason# EndOfStream
2025-11-19T13:58:43.491630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435631281714579:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:43.491673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:43.491689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435631281714579:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:43.493772Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435631281714477:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20266) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:58:43.493787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435631281714579:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:43.494091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435631281714477:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20266) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:43.494104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435631281714579:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:43.494109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435631281714477:2049] [node 2] ICP30 unregister session Session# [3:7574435631281714579:2049] VirtualId# [3:7574435631281714577:0]
2025-11-19T13:58:43.494129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:43.494135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435631281714477:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20266) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:43.494138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20266) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:43.496816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:43.496889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435632424097920:2049] [node 2] ICS01 socket: 93 reason# EndOfStream
2025-11-19T13:58:43.496896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435632424097819:2049] [node 2] ICP30 unregister session Session# [4:7574435632424097920:2049] VirtualId# [4:7574435632424097918:0]
2025-11-19T13:58:43.496901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435632424097920:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:43.496938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:43.496958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:43.497025Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:43.497092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435633478791629:2049] [node 2] ICS01 socket: 96 reason# EndOfStream
2025-11-19T13:58:43.497096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435633478791527:2049] [node 2] ICP30 unregister session Session# [6:7574435633478791629:2049] VirtualId# [6:7574435633478791627:0]
2025-11-19T13:58:43.497099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435633478791629:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:43.497120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:43.497136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:43.497989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:43.498059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435631741602740:2049] [node 2] ICS01 socket: 54 reason# EndOfStream
2025-11-19T13:58:43.498064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435631741602638:2049] [node 2] ICP30 unregister session Session# [5:7574435631741602740:2049] VirtualId# [5:7574435631741602736:0]
2025-11-19T13:58:43.498068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435631741602740:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:43.498104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:43.498125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:43.537591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:43.537649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435633478791622:2050] [node 3] ICS01 socket: 95 reason# EndOfStream
2025-11-19T13:58:43.537653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435633478791528:2050] [node 3] ICP30 unregister session Session# [6:7574435633478791622:2050] VirtualId# [6:7574435633478791613:0]
2025-11-19T13:58:43.537657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435633478791622:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:43.537681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:43.537698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:43.537340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:43.537445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435631741602737:2050] [node 3] ICS01 socket: 65 reason# EndOfStream
2025-11-19T13:58:43.537455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435631741602639:2050] [node 3] ICP30 unregister session Session# [5:7574435631741602737:2050] VirtualId# [5:7574435631741602723:0]
2025-11-19T13:58:43.537464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435631741602737:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:43.537512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:43.537535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:43.552392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:43.552503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435633478791616:2077] [node 4] ICS01 socket: 79 reason# EndOfStream
2025-11-19T13:58:43.552514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435633478791558:2077] [node 4] ICP30 unregister session Session# [6:7574435633478791616:2077] VirtualId# [6:7574435633478791605:0]
2025-11-19T13:58:43.552523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435633478791616:2077] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:43.552579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:43.552601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:43.692008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435629863378712:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:43.692031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435629863378713:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:43.816322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435630517841436:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:43.816341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435630517841437:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:43.920603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435631163566503:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:43.920617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435631163566504:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:44.088349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435635459929733:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:44.088370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435635459929734:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:44.088376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435635459929735:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:44.183960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435638120610373:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:44.183980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435638120610374:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:44.183985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435638120610375:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:44.275976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435636149487117:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:44.275996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435636149487118:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:44.276001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435636149487119:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:44.320492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435635459929790:2101] [node 6] ICP01 ready to work
2025-11-19T13:58:44.320963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435634812808792:2106] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.321701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435634158346068:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8984
2025-11-19T13:58:44.321723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435634158346069:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8984
2025-11-19T13:58:44.321831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435634158346073:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:44.321899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435634158346074:2110] [node 4] ICP01 ready to work
2025-11-19T13:58:44.321959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435634158346075:2111] [node 6] ICP01 ready to work
2025-11-19T13:58:44.321780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435635458533859:2104] [node 4] ICP01 ready to work
2025-11-19T13:58:44.323617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435635458533865:2107] [node 5] ICP01 ready to work
2025-11-19T13:58:44.320096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435638120610425:2089] [node 6] ICP01 ready to work
2025-11-19T13:58:44.335858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435636149487119:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19034100ns) {Configure:104(19032772ns) {ProcessConfigured:135(18488983ns) {StartInitialHandshake:157(18487925ns) {DropHandshakes:469(1313ns) {DropIncomingHandshake:441(138ns) DropOutgoingHandshake:456(147ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:476(1605ns) {SwitchToState:178(1228ns)}}}}}
2025-11-19T13:58:44.336027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435636149487164:2081] [node 5] ICP01 ready to work
2025-11-19T13:58:44.336119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435636149487165:2089] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.338149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435635458533867:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19759
2025-11-19T13:58:44.338168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435635458533868:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.336670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435638120610441:2106] [node 4] ICP01 ready to work
2025-11-19T13:58:44.339688Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435638120610374:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16979650ns) {Configure:104(16979175ns) {ProcessConfigured:135(16772682ns) {StartInitialHandshake:157(16772322ns) {DropHandshakes:469(406ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(152ns) PrepareNewSessionHandshake:476(898ns) {SwitchToState:178(594ns)}}}}}
2025-11-19T13:58:44.339808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435638120610448:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.343990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435634158346074:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(21886558ns) {Configure:104(21883822ns) {ProcessConfigured:135(20576455ns) {StartInitialHandshake:157(20575767ns) {DropHandshakes:469(640ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(166ns) PrepareNewSessionHandshake:476(1065ns) {SwitchToState:178(514ns)}}}}}
2025-11-19T13:58:44.344627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435634158346082:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8984
2025-11-19T13:58:44.344895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435634158346073:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(22906437ns) {Configure:104(22892765ns) {ProcessConfigured:135(21346203ns) {StartInitialHandshake:157(21345852ns) {DropHandshakes:469(267ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(60ns) PrepareNewSessionHandshake:476(739ns) {SwitchToState:178(567ns)}}}}}
2025-11-19T13:58:44.345622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435630517841436:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21664661ns) {Configure:104(21663827ns) {ProcessConfigured:135(21473409ns) {StartInitialHandshake:157(21473162ns) {DropHandshakes:469(375ns) {DropIncomingHandshake:441(15ns) DropOutgoingHandshake:456(41ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:476(1037ns) {SwitchToState:178(339ns)}}}}}
2025-11-19T13:58:44.346718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435634812808801:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:44.345013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435635458533865:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21363347ns) {Configure:104(21362990ns) {ProcessConfigured:135(21168204ns) {StartInitialHandshake:157(21167860ns) {DropHandshakes:469(315ns) {DropIncomingHandshake:441(34ns) DropOutgoingHandshake:456(42ns)} GenerateSessionVirtualId:988(63ns) PrepareNewSessionHandshake:476(859ns) {SwitchToState:178(57ns)}}}}}
2025-11-19T13:58:44.345166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435635458533871:2110] [node 6] ICP01 ready to work
2025-11-19T13:58:44.339967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435636149487118:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16936432ns) {Configure:104(16935395ns) {ProcessConfigured:135(16706665ns) {StartInitialHandshake:157(16706446ns) {DropHandshakes:469(429ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(43ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:476(659ns) {SwitchToState:178(494ns)}}}}}
2025-11-19T13:58:44.348495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435635459929805:2108] [node 5] ICP01 ready to work
2025-11-19T13:58:44.350432Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435638120610448:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.354136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435636149487192:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8946
2025-11-19T13:58:44.354156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435636149487193:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8946
2025-11-19T13:58:44.354167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435636149487194:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8946
2025-11-19T13:58:44.354227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435636149487195:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8946
2025-11-19T13:58:44.356151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435635459929810:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20750
2025-11-19T13:58:44.356171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435635459929811:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20750
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.363971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435634812808798:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:44.364793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435630517841437:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17045145ns) {Configure:104(17044470ns) {ProcessConfigured:135(16723760ns) {StartInitialHandshake:157(16723120ns) {DropHandshakes:469(620ns) {DropIncomingHandshake:441(226ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:476(1698ns) {SwitchToState:178(760ns)}}}}}
2025-11-19T13:58:44.365344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435634812808804:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
2025-11-19T13:58:44.365610Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435635458533867:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.370006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435638120610453:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.370240Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435636149487194:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.367934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435635459929805:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19356749ns) {Configure:104(19355161ns) {ProcessConfigured:135(19049634ns) {StartInitialHandshake:157(19048719ns) {DropHandshakes:469(974ns) {DropIncomingHandshake:441(235ns) DropOutgoingHandshake:456(131ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(1975ns) {SwitchToState:178(1407ns)}}}}}
2025-11-19T13:58:44.370904Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435636149487192:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.370945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435636149487164:2081] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:44.370136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435635458533880:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19759
2025-11-19T13:58:44.370157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435635458533881:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19759
2025-11-19T13:58:44.372044Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435635459929810:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.372644Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435638120610453:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.372673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435638120610375:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.375923Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435638120610428:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.375945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435638120610428:2100] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.374710Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435629863378712:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(53037902ns) {Configure:104(53035109ns) {ProcessConfigured:135(51357025ns) {StartInitialHandshake:157(51355447ns) {DropHandshakes:469(320ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(19ns)} GenerateSessionVirtualId:988(178ns) PrepareNewSessionHandshake:476(837ns) {SwitchToState:178(627ns)}}}}}
2025-11-19T13:58:44.374846Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435634158346069:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.375044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435634158346089:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8984
2025-11-19T13:58:44.379623Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435635459929811:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.379771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435635459929733:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.383349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435636149487193:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.383394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435636149487165:2089] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:44.383491Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435634158346068:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.383559Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435634158346089:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.382167Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435634812808794:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.382185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435634812808794:2107] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.382967Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435634812808804:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.383315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435634812808811:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
2025-11-19T13:58:44.383803Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435634812808794:2107] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:44.383937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435634812808792:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.383958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:44.383967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435634812808792:2106] [node 6] ICP22 created new session: [2:7574435634812808814:2106]
2025-11-19T13:58:44.383974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435634812808814:2106] [node 6] ICS09 handshake done sender: [2:7574435634812808794:2107] self: [2:7574435634812808793:0] peer: [6:7574435636149487199:0] socket: 54 qp: -1
2025-11-19T13:58:44.383987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435634812808814:2106] [node 6] ICS10 traffic start
2025-11-19T13:58:44.384008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:44.384593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435634812808801:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(20544449ns) {Configure:104(20542293ns) {ProcessConfigured:135(20279671ns) {StartInitialHandshake:157(20278798ns) {DropHandshakes:469(736ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(181ns)} GenerateSessionVirtualId:988(314ns) PrepareNewSessionHandshake:476(1395ns) {SwitchToState:178(751ns)}}}}}
2025-11-19T13:58:44.384696Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435634812808811:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.387132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435635459929735:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:44.388414Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435635459929792:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.388430Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435635459929792:2102] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.387235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435636149487118:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:44.387279Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435636149487118:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16953211ns) {IncomingHandshake:261(16952711ns) {DropIncomingHandshake:441(334ns) PrepareNewSessionHandshake:476(697ns) {SwitchToState:178(331ns)} GenerateSessionVirtualId:988(529ns) IssueIncomingHandshakeReply:193(16945320ns) {SwitchToState:178(264ns)}}}
2025-11-19T13:58:44.390013Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435635458533861:2105] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.390039Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435635458533861:2105] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.390165Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435635458533870:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.390172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435635458533870:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.390199Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435634158346079:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.390216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435634158346079:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.399878Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435638120610428:2100] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:44.401186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435638120610425:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.401217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:44.401227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435638120610425:2089] [node 6] ICP22 created new session: [5:7574435638120610456:2089]
2025-11-19T13:58:44.401233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435638120610456:2089] [node 6] ICS09 handshake done sender: [5:7574435638120610428:2100] self: [5:7574435638120610427:0] peer: [6:7574435636149487200:0] socket: 51 qp: -1
2025-11-19T13:58:44.401242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435638120610456:2089] [node 6] ICS10 traffic start
2025-11-19T13:58:44.401253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:44.404192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435634812808792:2106] [node 6] ICP35 event processing took too much time PendingConnection:278(20223537ns) {HandleHandshakeStatus:329(20222176ns) {DropIncomingHandshake:441(109ns) DropOutgoingHandshake:456(291ns) SwitchToState:178(229ns) ProcessPendingSessionEvents:495(15993398ns) {ForwardSessionEventToSession:612(15980881ns) ForwardSessionEventToSession:612(2552ns) ForwardSessionEventToSession:612(999ns) ForwardSessionEventToSession:612(578ns) ForwardSessionEventToSession:612(583ns) ForwardSessionEventToSession:612(1296ns) ForwardSessionEventToSession:612(772ns) ForwardSessionEventToSession:612(766ns) ForwardSessionEventToSession:612(558ns) ForwardSessionEventToSession:612(781ns)}}}
2025-11-19T13:58:44.405469Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435631163566503:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(56117898ns) {Configure:104(56115663ns) {ProcessConfigured:135(44927326ns) {StartInitialHandshake:157(44926226ns) {DropHandshakes:469(1859ns) {DropIncomingHandshake:441(452ns) DropOutgoingHandshake:456(173ns)} GenerateSessionVirtualId:988(289ns) PrepareNewSessionHandshake:476(1051ns) {SwitchToState:178(708ns)}}}}}
2025-11-19T13:58:44.405582Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435635458533881:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.405656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435631163566504:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:44.406795Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435634812808806:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.406807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435634812808806:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.407324Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435634812808806:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:44.407453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435630517841437:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.407479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:44.407486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435630517841437:2049] [node 3] ICP22 created new session: [2:7574435634812808820:2049]
2025-11-19T13:58:44.407492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435634812808820:2049] [node 3] ICS09 handshake done sender: [2:7574435634812808806:2113] self: [2:7574435634812808802:0] peer: [3:7574435635458533883:0] socket: 81 qp: -1
2025-11-19T13:58:44.407500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435634812808820:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:44.407519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:44.407977Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435635458533861:2105] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.408141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435635458533859:2104] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.408156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:44.408163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435635458533859:2104] [node 4] ICP22 created new session: [3:7574435635458533886:2104]
2025-11-19T13:58:44.408168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435635458533886:2104] [node 4] ICS09 handshake done sender: [3:7574435635458533861:2105] self: [3:7574435635458533860:0] peer: [4:7574435635459929813:0] socket: 57 qp: -1
2025-11-19T13:58:44.408175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435635458533886:2104] [node 4] ICS10 traffic start
2025-11-19T13:58:44.408204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:44.408371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435635458533885:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19759
2025-11-19T13:58:44.410167Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435635459929792:2102] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:44.410339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435635459929790:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.410353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:44.410360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435635459929790:2101] [node 6] ICP22 created new session: [4:7574435635459929816:2101]
2025-11-19T13:58:44.410367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435635459929816:2101] [node 6] ICS09 handshake done sender: [4:7574435635459929792:2102] self: [4:7574435635459929791:0] peer: [6:7574435636149487201:0] socket: 52 qp: -1
2025-11-19T13:58:44.410374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435635459929816:2101] [node 6] ICS10 traffic start
2025-11-19T13:58:44.410392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.410869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435636149487202:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8946
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.415111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435638120610458:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
2025-11-19T13:58:44.414691Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435634158346079:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:44.414842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435636149487204:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8946
2025-11-19T13:58:44.415962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435631163566504:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14241) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435635458533881:2117] held: no
2025-11-19T13:58:44.416143Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435638120610458:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.417620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435635459929819:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20750
2025-11-19T13:58:44.417682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435635459929820:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20750
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.419463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435635459929821:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20750
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.419605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435635459929822:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20750
2025-11-19T13:58:44.419646Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435635459929819:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.419662Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435635459929810:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:44.419827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435635459929735:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.419844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:44.419850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435635459929735:2050] [node 3] ICP22 created new session: [4:7574435635459929823:2050]
2025-11-19T13:58:44.419856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435635459929823:2050] [node 3] ICS09 handshake done sender: [4:7574435635459929810:2114] self: [4:7574435635459929813:0] peer: [3:7574435635458533860:0] socket: 58 qp: -1
2025-11-19T13:58:44.419862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435635459929823:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:44.419870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:44.420044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435635459929734:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:44.417382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435636149487203:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8946
2025-11-19T13:58:44.418323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435636149487205:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8946
2025-11-19T13:58:44.419197Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435636149487164:2081] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3029) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435636149487192:2112] held: no
2025-11-19T13:58:44.419226Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435636149487194:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:44.419563Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435636149487192:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:44.419762Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435636149487193:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:44.420274Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435634812808809:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.420286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435634812808809:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.417866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435634158346074:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.417893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:44.417900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435634158346074:2110] [node 4] ICP22 created new session: [1:7574435634158346095:2110]
2025-11-19T13:58:44.417917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435634158346095:2110] [node 4] ICS09 handshake done sender: [1:7574435634158346079:2112] self: [1:7574435634158346077:0] peer: [4:7574435635459929814:0] socket: 69 qp: -1
2025-11-19T13:58:44.417926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435634158346095:2110] [node 4] ICS10 traffic start
2025-11-19T13:58:44.417935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:44.420948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435638120610374:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:44.421162Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435634812808809:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:44.421202Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435634812808816:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.421210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435634812808816:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.421141Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435635459929811:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:44.421954Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435635458533868:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.422010Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435635458533881:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:44.428109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435634812808798:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.428142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:44.428162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435634812808798:2108] [node 4] ICP22 created new session: [2:7574435634812808824:2108]
2025-11-19T13:58:44.428169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435634812808824:2108] [node 4] ICS09 handshake done sender: [2:7574435634812808809:2114] self: [2:7574435634812808808:0] peer: [4:7574435635459929825:0] socket: 86 qp: -1
2025-11-19T13:58:44.428177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435634812808824:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:44.428187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.428808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435631163566504:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.428828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:44.428835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435631163566504:2049] [node 2] ICP22 created new session: [3:7574435635458533891:2049]
2025-11-19T13:58:44.428840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435635458533891:2049] [node 2] ICS09 handshake done sender: [3:7574435635458533881:2117] self: [3:7574435635458533883:0] peer: [2:7574435634812808802:0] socket: 82 qp: -1
2025-11-19T13:58:44.428845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435635458533891:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:44.428855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:44.429018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435635458533890:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.428378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435636149487118:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.428402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:44.428407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435636149487118:2049] [node 2] ICP22 created new session: [6:7574435636149487206:2049]
2025-11-19T13:58:44.428443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435636149487206:2049] [node 2] ICS09 handshake done sender: [6:7574435636149487194:2114] self: [6:7574435636149487199:0] peer: [2:7574435634812808793:0] socket: 76 qp: -1
2025-11-19T13:58:44.428449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435636149487206:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:44.428468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:44.428614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435636149487164:2081] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.428621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:44.428625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435636149487164:2081] [node 5] ICP22 created new session: [6:7574435636149487208:2081]
2025-11-19T13:58:44.428629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435636149487208:2081] [node 5] ICS09 handshake done sender: [6:7574435636149487192:2112] self: [6:7574435636149487200:0] peer: [5:7574435638120610427:0] socket: 59 qp: -1
2025-11-19T13:58:44.428633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435636149487208:2081] [node 5] ICS10 traffic start
2025-11-19T13:58:44.428637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:44.428708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435636149487165:2089] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.428714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:44.428717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435636149487165:2089] [node 4] ICP22 created new session: [6:7574435636149487210:2089]
2025-11-19T13:58:44.428722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435636149487210:2089] [node 4] ICS09 handshake done sender: [6:7574435636149487193:2113] self: [6:7574435636149487201:0] peer: [4:7574435635459929791:0] socket: 75 qp: -1
2025-11-19T13:58:44.428727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435636149487210:2089] [node 4] ICS10 traffic start
2025-11-19T13:58:44.428732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:44.428406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435638120610425:2089] [node 6] ICP35 event processing took too much time PendingConnection:278(27175050ns) {HandleHandshakeStatus:329(27173837ns) {DropIncomingHandshake:441(7199ns) DropOutgoingHandshake:456(463ns) SwitchToState:178(247ns) ProcessPendingSessionEvents:495(12419088ns) {ForwardSessionEventToSession:612(12403672ns) ForwardSessionEventToSession:612(1879ns) ForwardSessionEventToSession:612(1277ns) ForwardSessionEventToSession:612(776ns) ForwardSessionEventToSession:612(911ns) ForwardSessionEventToSession:612(1453ns) ForwardSessionEventToSession:612(785ns) ForwardSessionEventToSession:612(693ns) ForwardSessionEventToSession:612(703ns) ForwardSessionEventToSession:612(1076ns)}}}
2025-11-19T13:58:44.429353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435638120610461:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
2025-11-19T13:58:44.431375Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435635458533870:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:44.431607Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435638120610453:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:44.431727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435638120610375:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.431763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:44.431769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435638120610375:2050] [node 3] ICP22 created new session: [5:7574435638120610462:2050]
2025-11-19T13:58:44.431776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435638120610462:2050] [node 3] ICS09 handshake done sender: [5:7574435638120610453:2116] self: [5:7574435638120610454:0] peer: [3:7574435635458533866:0] socket: 73 qp: -1
2025-11-19T13:58:44.431784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435638120610462:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:44.431793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:44.434066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435635458533865:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.434091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:44.434098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435635458533865:2107] [node 5] ICP22 created new session: [3:7574435635458533894:2107]
2025-11-19T13:58:44.434104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435635458533894:2107] [node 5] ICS09 handshake done sender: [3:7574435635458533870:2111] self: [3:7574435635458533866:0] peer: [5:7574435638120610454:0] socket: 72 qp: -1
2025-11-19T13:58:44.434110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435635458533894:2107] [node 5] ICS10 traffic start
2025-11-19T13:58:44.434117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:44.436557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435635459929733:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.436583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:44.436592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435635459929733:2048] [node 1] ICP22 created new session: [4:7574435635459929827:2048]
2025-11-19T13:58:44.436599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435635459929827:2048] [node 1] ICS09 handshake done sender: [4:7574435635459929811:2115] self: [4:7574435635459929814:0] peer: [1:7574435634158346077:0] socket: 78 qp: -1
2025-11-19T13:58:44.436605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435635459929827:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:44.436612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.437692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435634812808826:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
2025-11-19T13:58:44.438057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435634158346097:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8984
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.440546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435638120610465:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
2025-11-19T13:58:44.440565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435638120610466:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
2025-11-19T13:58:44.441999Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435636149487205:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.442041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435636149487119:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:44.443949Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435634812808816:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:44.444097Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435634812808826:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.445249Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435635459929733:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:44.446224Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435638120610466:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.446456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435635459929826:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20750
2025-11-19T13:58:44.446674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435634812808801:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.446695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:44.446701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435634812808801:2109] [node 5] ICP22 created new session: [2:7574435634812808829:2109]
2025-11-19T13:58:44.446706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435634812808829:2109] [node 5] ICS09 handshake done sender: [2:7574435634812808816:2116] self: [2:7574435634812808805:0] peer: [5:7574435638120610460:0] socket: 49 qp: -1
2025-11-19T13:58:44.446712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435634812808829:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:44.446731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:44.444490Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435634158346097:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.447923Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435638120610458:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:44.453651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435634812808798:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(25505864ns) {HandleHandshakeStatus:329(25503890ns) {DropIncomingHandshake:441(6073ns) DropOutgoingHandshake:456(169ns) SwitchToState:178(134ns) ProcessPendingSessionEvents:495(25371501ns) {ForwardSessionEventToSession:612(25350997ns) ForwardSessionEventToSession:612(3319ns) ForwardSessionEventToSession:612(1133ns) ForwardSessionEventToSession:612(2126ns) ForwardSessionEventToSession:612(1185ns) ForwardSessionEventToSession:612(774ns) ForwardSessionEventToSession:612(571ns) ForwardSessionEventToSession:612(693ns) ForwardSessionEventToSession:612(696ns) ForwardSessionEventToSession:612(1108ns)}}}
2025-11-19T13:58:44.455421Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435634158346082:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.453845Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435635458533873:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.453866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435635458533873:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.454614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435635458533859:2104] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:44.455584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435638120610373:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:44.455607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435638120610374:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.455622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:44.455630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435638120610374:2049] [node 2] ICP22 created new session: [5:7574435638120610468:2049]
2025-11-19T13:58:44.455636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435638120610468:2049] [node 2] ICS09 handshake done sender: [5:7574435638120610458:2117] self: [5:7574435638120610460:0] peer: [2:7574435634812808805:0] socket: 98 qp: -1
2025-11-19T13:58:44.455644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435638120610468:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:44.455663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:44.457933Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435635459929734:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14241) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435635459929819:2117] held: no
2025-11-19T13:58:44.458029Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435635459929819:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:44.458156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435635459929734:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.458171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:44.458178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435635459929734:2049] [node 2] ICP22 created new session: [4:7574435635459929830:2049]
2025-11-19T13:58:44.458183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435635459929830:2049] [node 2] ICS09 handshake done sender: [4:7574435635459929819:2117] self: [4:7574435635459929825:0] peer: [2:7574435634812808808:0] socket: 87 qp: -1
2025-11-19T13:58:44.458190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435635459929830:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:44.458196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.463447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435636149487213:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8946
2025-11-19T13:58:44.463613Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435635458533873:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:44.460976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435638120610470:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
2025-11-19T13:58:44.462886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435638120610471:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
2025-11-19T13:58:44.461450Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435634158346088:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.461467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435634158346088:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.463975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435635458533871:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.464010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:44.464019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435635458533871:2110] [node 6] ICP22 created new session: [3:7574435635458533898:2110]
2025-11-19T13:58:44.464026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435635458533898:2110] [node 6] ICS09 handshake done sender: [3:7574435635458533873:2112] self: [3:7574435635458533872:0] peer: [6:7574435636149487212:0] socket: 93 qp: -1
2025-11-19T13:58:44.464032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435635458533898:2110] [node 6] ICS10 traffic start
2025-11-19T13:58:44.464042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:44.464629Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435636149487205:2122] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:44.464620Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435638120610470:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.464661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435638120610441:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:44.464761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435636149487119:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.464779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:44.464786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435636149487119:2050] [node 3] ICP22 created new session: [6:7574435636149487214:2050]
2025-11-19T13:58:44.464792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435636149487214:2050] [node 3] ICS09 handshake done sender: [6:7574435636149487205:2122] self: [6:7574435636149487212:0] peer: [3:7574435635458533872:0] socket: 97 qp: -1
2025-11-19T13:58:44.464798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435636149487214:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:44.464806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:44.465244Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435634158346088:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:44.464803Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435635459929812:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.464815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435635459929812:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.465421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435634158346073:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.465444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:44.465450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435634158346073:2109] [node 5] ICP22 created new session: [1:7574435634158346102:2109]
2025-11-19T13:58:44.465456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435634158346102:2109] [node 5] ICS09 handshake done sender: [1:7574435634158346088:2116] self: [1:7574435634158346078:0] peer: [5:7574435638120610467:0] socket: 109 qp: -1
2025-11-19T13:58:44.465462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435634158346102:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:44.465471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:44.466206Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435635459929812:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:44.466458Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435638120610466:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.467262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435638120610473:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.468256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435634812808831:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
2025-11-19T13:58:44.468733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435635459929805:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.468750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:44.468760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435635459929805:2108] [node 5] ICP22 created new session: [4:7574435635459929833:2108]
2025-11-19T13:58:44.468766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435635459929833:2108] [node 5] ICS09 handshake done sender: [4:7574435635459929812:2116] self: [4:7574435635459929807:0] peer: [5:7574435638120610472:0] socket: 48 qp: -1
2025-11-19T13:58:44.468774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435635459929833:2108] [node 5] ICS10 traffic start
2025-11-19T13:58:44.468780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:44.469324Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435634158346090:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.469335Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435634158346090:2118] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.469182Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435634812808831:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.469222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435630517841436:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:44.469803Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435638120610373:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6179) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435638120610466:2120] held: no
2025-11-19T13:58:44.469829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435638120610373:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.469838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:44.469843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435638120610373:2048] [node 1] ICP22 created new session: [5:7574435638120610474:2048]
2025-11-19T13:58:44.469851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435638120610474:2048] [node 1] ICS09 handshake done sender: [5:7574435638120610466:2120] self: [5:7574435638120610467:0] peer: [1:7574435634158346078:0] socket: 111 qp: -1
2025-11-19T13:58:44.469857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435638120610474:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:44.469866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:44.469966Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435638120610470:2121] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.478571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435634812808833:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
2025-11-19T13:58:44.478916Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435634158346090:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:44.479128Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435634812808831:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:44.479242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435630517841436:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.479261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:44.479268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435630517841436:2048] [node 1] ICP22 created new session: [2:7574435634812808834:2048]
2025-11-19T13:58:44.479274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435634812808834:2048] [node 1] ICS09 handshake done sender: [2:7574435634812808831:2118] self: [2:7574435634812808832:0] peer: [1:7574435634158346076:0] socket: 106 qp: -1
2025-11-19T13:58:44.479280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435634812808834:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:44.479288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:44.480530Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435636149487195:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.480592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435636149487117:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:44.480300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435629863378712:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.480325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:44.480332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435629863378712:2048] [node 2] ICP22 created new session: [1:7574435634158346106:2048]
2025-11-19T13:58:44.480339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435634158346106:2048] [node 2] ICS09 handshake done sender: [1:7574435634158346090:2118] self: [1:7574435634158346076:0] peer: [2:7574435634812808832:0] socket: 56 qp: -1
2025-11-19T13:58:44.480346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435634158346106:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:44.480360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:44.487332Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435634158346081:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.487349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435634158346081:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:44.491674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435638120610441:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.491707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:44.491717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435638120610441:2106] [node 4] ICP22 created new session: [5:7574435638120610476:2106]
2025-11-19T13:58:44.491724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435638120610476:2106] [node 4] ICS09 handshake done sender: [5:7574435638120610470:2121] self: [5:7574435638120610472:0] peer: [4:7574435635459929807:0] socket: 63 qp: -1
2025-11-19T13:58:44.491732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435638120610476:2106] [node 4] ICS10 traffic start
2025-11-19T13:58:44.491740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:44.496263Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435634158346081:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:44.500932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435634158346075:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.500970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:44.500984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435634158346075:2111] [node 6] ICP22 created new session: [1:7574435634158346109:2111]
2025-11-19T13:58:44.500991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435634158346109:2111] [node 6] ICS09 handshake done sender: [1:7574435634158346081:2113] self: [1:7574435634158346080:0] peer: [6:7574435636149487216:0] socket: 70 qp: -1
2025-11-19T13:58:44.501001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435634158346109:2111] [node 6] ICS10 traffic start
2025-11-19T13:58:44.501011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.501644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435636149487217:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8946
2025-11-19T13:58:44.502385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435636149487117:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6179) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435636149487195:2115] held: no
2025-11-19T13:58:44.502499Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435636149487195:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:44.507568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435636149487117:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.507602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:44.507613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435636149487117:2048] [node 1] ICP22 created new session: [6:7574435636149487218:2048]
2025-11-19T13:58:44.507622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435636149487218:2048] [node 1] ICS09 handshake done sender: [6:7574435636149487195:2115] self: [6:7574435636149487216:0] peer: [1:7574435634158346080:0] socket: 77 qp: -1
2025-11-19T13:58:44.507633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435636149487218:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:44.507646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:44.512278Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435635458533880:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.512331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435631163566503:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:44.512526Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435634158346084:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.512538Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435634158346084:2115] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:44.513602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435635458533901:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19759
2025-11-19T13:58:44.513765Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435634158346084:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:44.513987Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435635458533880:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:44.514111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435631163566503:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:44.514137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:44.514143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435631163566503:2048] [node 1] ICP22 created new session: [3:7574435635458533902:2048]
2025-11-19T13:58:44.514149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435635458533902:2048] [node 1] ICS09 handshake done sender: [3:7574435635458533880:2116] self: [3:7574435635458533900:0] peer: [1:7574435634158346083:0] socket: 74 qp: -1
2025-11-19T13:58:44.514155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435635458533902:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:44.514171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:44.515339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435629863378713:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:44.515359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:44.515365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435629863378713:2049] [node 3] ICP22 created new session: [1:7574435634158346112:2049]
2025-11-19T13:58:44.515371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435634158346112:2049] [node 3] ICS09 handshake done sender: [1:7574435634158346084:2115] self: [1:7574435634158346083:0] peer: [3:7574435635458533900:0] socket: 71 qp: -1
2025-11-19T13:58:44.515376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435634158346112:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:44.515382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:44.526497Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435635459929822:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:44.526555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435635459929805:2108] [node 5] ICP18 handshake (actor [4:7574435635459929822:2120]) is too old
2025-11-19T13:58:44.540062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:44.540162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435634812808834:2048] [node 1] ICS01 socket: 106 reason# EndOfStream
2025-11-19T13:58:44.540172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435630517841436:2048] [node 1] ICP30 unregister session Session# [2:7574435634812808834:2048] VirtualId# [2:7574435634812808832:0]
2025-11-19T13:58:44.540180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435634812808834:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:44.540226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:44.540250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:44.540801Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:44.540876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435635458533902:2048] [node 1] ICS01 socket: 74 reason# EndOfStream
2025-11-19T13:58:44.540882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435631163566503:2048] [node 1] ICP30 unregister session Session# [3:7574435635458533902:2048] VirtualId# [3:7574435635458533900:0]
2025-11-19T13:58:44.540887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435635458533902:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:44.540921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:44.540941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:44.545015Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:44.545098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435638120610474:2048] [node 1] ICS01 socket: 111 reason# EndOfStream
2025-11-19T13:58:44.545105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435638120610373:2048] [node 1] ICP30 unregister session Session# [5:7574435638120610474:2048] VirtualId# [5:7574435638120610467:0]
2025-11-19T13:58:44.545111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435638120610474:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:44.545150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:44.545185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:44.545281Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:44.545345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435635459929827:2048] [node 1] ICS01 socket: 78 reason# EndOfStream
2025-11-19T13:58:44.545350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435635459929733:2048] [node 1] ICP30 unregister session Session# [4:7574435635459929827:2048] VirtualId# [4:7574435635459929814:0]
2025-11-19T13:58:44.545355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435635459929827:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:44.545383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:44.545410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:44.545487Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:44.545545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435636149487218:2048] [node 1] ICS01 socket: 77 reason# EndOfStream
2025-11-19T13:58:44.545549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435636149487117:2048] [node 1] ICP30 unregister session Session# [6:7574435636149487218:2048] VirtualId# [6:7574435636149487216:0]
2025-11-19T13:58:44.545553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435636149487218:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:44.545583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:44.545602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:44.559884Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:44.559972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435635458533891:2049] [node 2] ICS01 socket: 82 reason# EndOfStream
2025-11-19T13:58:44.559980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435631163566504:2049] [node 2] ICP30 unregister session Session# [3:7574435635458533891:2049] VirtualId# [3:7574435635458533883:0]
2025-11-19T13:58:44.559988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435635458533891:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:44.560043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:44.560063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:44.562984Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:44.563098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435636149487206:2049] [node 2] ICS01 socket: 76 reason# EndOfStream
2025-11-19T13:58:44.563106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435636149487118:2049] [node 2] ICP30 unregister session Session# [6:7574435636149487206:2049] VirtualId# [6:7574435636149487199:0]
2025-11-19T13:58:44.563114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435636149487206:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:44.563173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:44.563201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:44.563240Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:44.563307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435635459929830:2049] [node 2] ICS01 socket: 87 reason# EndOfStream
2025-11-19T13:58:44.563313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435635459929734:2049] [node 2] ICP30 unregister session Session# [4:7574435635459929830:2049] VirtualId# [4:7574435635459929825:0]
2025-11-19T13:58:44.563331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435635459929830:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:44.563360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:44.563383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:44.563419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:44.563482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435638120610468:2049] [node 2] ICS01 socket: 98 reason# EndOfStream
2025-11-19T13:58:44.563487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435638120610374:2049] [node 2] ICP30 unregister session Session# [5:7574435638120610468:2049] VirtualId# [5:7574435638120610460:0]
2025-11-19T13:58:44.563493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435638120610468:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:44.563521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:44.563542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:44.588983Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:44.589085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435638120610462:2050] [node 3] ICS01 socket: 73 reason# EndOfStream
2025-11-19T13:58:44.589097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435638120610375:2050] [node 3] ICP30 unregister session Session# [5:7574435638120610462:2050] VirtualId# [5:7574435638120610454:0]
2025-11-19T13:58:44.589105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435638120610462:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:44.589162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:44.589187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:44.589231Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:44.589296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435636149487214:2050] [node 3] ICS07 socket disconnect 97 reason# EndOfStream
2025-11-19T13:58:44.589302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435636149487214:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:44.589327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:44.589350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435636149487214:2050] [node 3] ICS15 start handshake
2025-11-19T13:58:44.590135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435636149487119:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12109) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:44.590146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435636149487214:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:44.590151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435636149487119:2050] [node 3] ICP30 unregister session Session# [6:7574435636149487214:2050] VirtualId# [6:7574435636149487212:0]
2025-11-19T13:58:44.590163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:44.590168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435636149487119:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12109) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:44.590171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12109) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:44.600972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435636149487119:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:58:44.610302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:44.610394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435636149487210:2089] [node 4] ICS01 socket: 75 reason# EndOfStream
2025-11-19T13:58:44.610406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435636149487165:2089] [node 4] ICP30 unregister session Session# [6:7574435636149487210:2089] VirtualId# [6:7574435636149487201:0]
2025-11-19T13:58:44.610413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435636149487210:2089] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:44.610476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:44.610501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:44.850191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435635389514288:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:44.850210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435635389514289:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:44.915925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435637733821933:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:44.915945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435637733821934:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:44.979921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435637857723811:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:44.979933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435637857723812:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:45.042803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435638413751216:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:45.042816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435638413751218:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:45.042833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435638413751217:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:45.152798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435639833500984:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:45.152815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435639833500985:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:45.152820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435639833500986:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:45.212129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435639713950499:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:45.212146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435639713950500:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:45.212150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435639713950501:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:45.234499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435642028789285:2102] [node 5] ICP01 ready to work
2025-11-19T13:58:45.235288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435639713950525:2070] [node 5] ICP01 ready to work
2025-11-19T13:58:45.234835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435642152691154:2093] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.235599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435642028789293:2103] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.235539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435638413751261:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13308
2025-11-19T13:58:45.235165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435639684481634:2096] [node 4] ICP01 ready to work
2025-11-19T13:58:45.235965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435639684481635:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11359
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.236565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435639684481644:2100] [node 5] ICP01 ready to work
2025-11-19T13:58:45.236294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435639833501023:2084] [node 4] ICP01 ready to work
2025-11-19T13:58:45.236325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435639833501018:2081] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21604
2025-11-19T13:58:45.236341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435639833501019:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21604
2025-11-19T13:58:45.236795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435639684481645:2101] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.237588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435642152691159:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36702
2025-11-19T13:58:45.237607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435642152691160:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36702
2025-11-19T13:58:45.236502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435642028789299:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-11-19T13:58:45.237323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435642028789301:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:45.236868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435639833501027:2077] [node 6] ICP01 ready to work
2025-11-19T13:58:45.238387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435638413751268:2097] [node 5] ICP01 ready to work
2025-11-19T13:58:45.253046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435642152691172:2099] [node 5] ICP01 ready to work
2025-11-19T13:58:45.257613Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435639833501023:2084] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(21224126ns) {Configure:104(21223400ns) {ProcessConfigured:135(20851538ns) {StartInitialHandshake:157(20850507ns) {DropHandshakes:469(499ns) {DropIncomingHandshake:441(35ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(274ns) PrepareNewSessionHandshake:476(989ns) {SwitchToState:178(333ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.258829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435642028789306:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-11-19T13:58:45.270764Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435639684481644:2100] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(33914742ns) {Configure:104(33914012ns) {ProcessConfigured:135(33331889ns) {StartInitialHandshake:157(33331223ns) {DropHandshakes:469(473ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(1083ns) {SwitchToState:178(675ns)}}}}}
2025-11-19T13:58:45.272029Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642028789299:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.272082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435637733821933:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.271056Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435637857723811:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(34343214ns) {Configure:104(34342222ns) {ProcessConfigured:135(34012308ns) {StartInitialHandshake:157(34012028ns) {DropHandshakes:469(520ns) {DropIncomingHandshake:441(88ns) DropOutgoingHandshake:456(77ns)} GenerateSessionVirtualId:988(264ns) PrepareNewSessionHandshake:476(1130ns) {SwitchToState:178(976ns)}}}}}
2025-11-19T13:58:45.272836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435642152691183:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36702
2025-11-19T13:58:45.272850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435642152691184:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36702
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.273098Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435642028789301:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(35731537ns) {Configure:104(35731036ns) {ProcessConfigured:135(35510461ns) {StartInitialHandshake:157(35510051ns) {DropHandshakes:469(420ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(24ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:476(867ns) {SwitchToState:178(631ns)}}}}}
2025-11-19T13:58:45.273498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435639713950501:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(22605931ns) {Configure:104(22604741ns) {ProcessConfigured:135(22206332ns) {StartInitialHandshake:157(22205901ns) {DropHandshakes:469(973ns) {DropIncomingHandshake:441(123ns) DropOutgoingHandshake:456(123ns)} GenerateSessionVirtualId:988(240ns) PrepareNewSessionHandshake:476(837ns) {SwitchToState:178(346ns)}}}}}
2025-11-19T13:58:45.273286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435639684481664:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11359
2025-11-19T13:58:45.273301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435639684481666:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11359
::ffff:127.1.0.0
2025-11-19T13:58:45.269347Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435638413751217:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(32647774ns) {Configure:104(32646706ns) {ProcessConfigured:135(32228141ns) {StartInitialHandshake:157(32227517ns) {DropHandshakes:469(560ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(85ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:476(1347ns) {SwitchToState:178(882ns)}}}}}
2025-11-19T13:58:45.270937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435638413751268:2097] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(32482199ns) {Configure:104(32481436ns) {ProcessConfigured:135(32177988ns) {StartInitialHandshake:157(32177696ns) {DropHandshakes:469(508ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(563ns) {SwitchToState:178(291ns)}}}}}
2025-11-19T13:58:45.275396Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435639684481637:2098] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.275408Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435639684481637:2098] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.275540Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435639833501027:2077] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(38564633ns) {Configure:104(38563363ns) {ProcessConfigured:135(38226999ns) {StartInitialHandshake:157(38226357ns) {DropHandshakes:469(1225ns) {DropIncomingHandshake:441(161ns) DropOutgoingHandshake:456(138ns)} GenerateSessionVirtualId:988(171ns) PrepareNewSessionHandshake:476(1666ns) {SwitchToState:178(840ns)}}}}}
2025-11-19T13:58:45.275663Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435639833501019:2082] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.275703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435639833500985:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.275953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435639833500984:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(37595811ns) {Configure:104(37595110ns) {ProcessConfigured:135(37322000ns) {StartInitialHandshake:157(37321586ns) {DropHandshakes:469(453ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(64ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:476(425ns) {SwitchToState:178(224ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.276180Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642028789288:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.276191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435642028789288:2104] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.276496Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435639684481637:2098] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:45.277001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435642028789311:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.277160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435639684481665:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11359
2025-11-19T13:58:45.277275Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435639684481664:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.277629Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642028789311:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.277875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435639833501064:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21604
2025-11-19T13:58:45.280153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435635389514288:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.280179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:45.280187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435635389514288:2048] [node 2] ICP22 created new session: [1:7574435639684481669:2048]
2025-11-19T13:58:45.280194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435639684481669:2048] [node 2] ICS09 handshake done sender: [1:7574435639684481637:2098] self: [1:7574435639684481636:0] peer: [2:7574435642028789309:0] socket: 62 qp: -1
2025-11-19T13:58:45.280202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435639684481669:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:45.280224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:45.284116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435639713950500:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(31516414ns) {Configure:104(31515281ns) {ProcessConfigured:135(16283440ns) {StartInitialHandshake:157(16281907ns) {DropHandshakes:469(1192ns) {DropIncomingHandshake:441(188ns) DropOutgoingHandshake:456(126ns)} GenerateSessionVirtualId:988(459ns) PrepareNewSessionHandshake:476(1990ns) {SwitchToState:178(1334ns)}}}}}
2025-11-19T13:58:45.284100Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435639684481665:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.287875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435638413751293:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13308
2025-11-19T13:58:45.287822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435642152691178:2098] [node 6] ICP01 ready to work
2025-11-19T13:58:45.288002Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435642152691159:2095] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.290219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435637733821933:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24364) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435642028789299:2108] held: no
2025-11-19T13:58:45.290303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435642028789313:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-11-19T13:58:45.291366Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435642028789299:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:45.291691Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435639833501064:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.291732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435639833500986:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.289527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435642152691188:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36702
2025-11-19T13:58:45.291543Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435642152691183:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.291622Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435642152691184:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.290845Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435638413751261:2092] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.290900Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435638413751293:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.290935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435638413751218:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.291945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435638413751292:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13308
2025-11-19T13:58:45.292412Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435638413751292:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.292460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435638413751216:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.292599Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435642152691156:2094] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.292608Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435642152691156:2094] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.299908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435637857723812:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(46785807ns) {Configure:104(46784828ns) {ProcessConfigured:135(46508339ns) {StartInitialHandshake:157(46507273ns) {DropHandshakes:469(858ns) {DropIncomingHandshake:441(133ns) DropOutgoingHandshake:456(148ns)} GenerateSessionVirtualId:988(103ns) PrepareNewSessionHandshake:476(1156ns) {SwitchToState:178(463ns)}}}}}
2025-11-19T13:58:45.300189Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642028789306:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.300414Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642028789292:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.300423Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435642028789292:2106] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.301838Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435639684481666:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.302290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435635389514288:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(22100153ns) {HandleHandshakeStatus:329(22099379ns) {DropIncomingHandshake:441(4854ns) DropOutgoingHandshake:456(167ns) SwitchToState:178(63ns) ProcessPendingSessionEvents:495(16651448ns) {ForwardSessionEventToSession:612(16622609ns) ForwardSessionEventToSession:612(2081ns) ForwardSessionEventToSession:612(1087ns) ForwardSessionEventToSession:612(738ns) ForwardSessionEventToSession:612(678ns) ForwardSessionEventToSession:612(1089ns) ForwardSessionEventToSession:612(681ns) ForwardSessionEventToSession:612(645ns) ForwardSessionEventToSession:612(586ns) ForwardSessionEventToSession:612(16153ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.303482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435639833501067:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21604
2025-11-19T13:58:45.299979Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435642152691160:2096] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.300134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435637857723812:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.300217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435637857723811:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.304046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435642152691195:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36702
2025-11-19T13:58:45.304000Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435639684481640:2099] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.304019Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435639684481640:2099] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.307891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639713950575:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50703
2025-11-19T13:58:45.309539Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435642028789288:2104] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:45.309823Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435639833501019:2082] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:45.309912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435639833500985:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.309935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:45.309944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435639833500985:2049] [node 2] ICP22 created new session: [5:7574435639833501068:2049]
2025-11-19T13:58:45.309953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435639833501068:2049] [node 2] ICS09 handshake done sender: [5:7574435639833501019:2082] self: [5:7574435639833501062:0] peer: [2:7574435642028789287:0] socket: 64 qp: -1
2025-11-19T13:58:45.309961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435639833501068:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:45.309972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:45.312370Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435642028789292:2106] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:45.312687Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435639684481648:2102] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.312707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435639684481648:2102] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.312555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435637733821933:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.312586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:45.312598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435637733821933:2048] [node 1] ICP22 created new session: [2:7574435642028789317:2048]
2025-11-19T13:58:45.312605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435642028789317:2048] [node 1] ICS09 handshake done sender: [2:7574435642028789299:2108] self: [2:7574435642028789309:0] peer: [1:7574435639684481636:0] socket: 66 qp: -1
2025-11-19T13:58:45.312614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435642028789317:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:45.312651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:45.314343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435642028789285:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.314386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:45.314396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435642028789285:2102] [node 5] ICP22 created new session: [2:7574435642028789319:2102]
2025-11-19T13:58:45.314404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435642028789319:2102] [node 5] ICS09 handshake done sender: [2:7574435642028789288:2104] self: [2:7574435642028789287:0] peer: [5:7574435639833501062:0] socket: 59 qp: -1
2025-11-19T13:58:45.314412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435642028789319:2102] [node 5] ICS10 traffic start
2025-11-19T13:58:45.314438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:45.314662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435637733821934:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.314668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:45.314675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435637733821934:2049] [node 3] ICP22 created new session: [2:7574435642028789321:2049]
2025-11-19T13:58:45.314679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435642028789321:2049] [node 3] ICS09 handshake done sender: [2:7574435642028789292:2106] self: [2:7574435642028789291:0] peer: [3:7574435642152691190:0] socket: 61 qp: -1
2025-11-19T13:58:45.314684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435642028789321:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:45.314690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:45.313417Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435642152691182:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.313449Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435642152691182:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.315840Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435639713950575:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.315923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435639713950499:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.316259Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435642152691182:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:45.316485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435639684481650:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.316499Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435639684481650:2103] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.316890Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435639684481640:2099] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:45.316789Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435642152691156:2094] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:45.316829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435642152691172:2099] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.316848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:45.316856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435642152691172:2099] [node 5] ICP22 created new session: [3:7574435642152691199:2099]
2025-11-19T13:58:45.316862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435642152691199:2099] [node 5] ICS09 handshake done sender: [3:7574435642152691182:2114] self: [3:7574435642152691181:0] peer: [5:7574435639833501066:0] socket: 82 qp: -1
2025-11-19T13:58:45.316880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435642152691199:2099] [node 5] ICS10 traffic start
2025-11-19T13:58:45.316889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.317160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435642152691198:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36702
2025-11-19T13:58:45.317049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435635389514289:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.317083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:45.317093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435635389514289:2049] [node 3] ICP22 created new session: [1:7574435639684481675:2049]
2025-11-19T13:58:45.317099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435639684481675:2049] [node 3] ICS09 handshake done sender: [1:7574435639684481640:2099] self: [1:7574435639684481639:0] peer: [3:7574435642152691193:0] socket: 65 qp: -1
2025-11-19T13:58:45.317107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435639684481675:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:45.317114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:45.319960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435637733821933:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(28438754ns) {HandleHandshakeStatus:329(28437767ns) {DropIncomingHandshake:441(900ns) DropOutgoingHandshake:456(75ns) SwitchToState:178(167ns) ProcessPendingSessionEvents:495(653ns)}}
2025-11-19T13:58:45.317296Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435642152691159:2095] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:45.317498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435642152691154:2093] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.317524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:45.317530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435642152691154:2093] [node 4] ICP22 created new session: [3:7574435642152691202:2093]
2025-11-19T13:58:45.317536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435642152691202:2093] [node 4] ICS09 handshake done sender: [3:7574435642152691156:2094] self: [3:7574435642152691155:0] peer: [4:7574435638413751294:0] socket: 56 qp: -1
2025-11-19T13:58:45.317541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435642152691202:2093] [node 4] ICS10 traffic start
2025-11-19T13:58:45.317547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:45.317924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435637857723812:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.317934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:45.317938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435637857723812:2049] [node 2] ICP22 created new session: [3:7574435642152691204:2049]
2025-11-19T13:58:45.317944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435642152691204:2049] [node 2] ICS09 handshake done sender: [3:7574435642152691159:2095] self: [3:7574435642152691190:0] peer: [2:7574435642028789291:0] socket: 68 qp: -1
2025-11-19T13:58:45.317948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435642152691204:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:45.317955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:45.324158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435639833500986:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24207) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435639833501064:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.326510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435639684481678:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11359
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.326840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435638413751298:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13308
2025-11-19T13:58:45.326992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435637857723811:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24364) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435642152691160:2096] held: no
2025-11-19T13:58:45.327169Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435642152691160:2096] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:45.327330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435637857723811:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.327346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:45.327355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435637857723811:2048] [node 1] ICP22 created new session: [3:7574435642152691207:2048]
2025-11-19T13:58:45.327361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435642152691207:2048] [node 1] ICS09 handshake done sender: [3:7574435642152691160:2096] self: [3:7574435642152691193:0] peer: [1:7574435639684481639:0] socket: 69 qp: -1
2025-11-19T13:58:45.327368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435642152691207:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: 2025-11-19T13:58:45.327374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.327586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435638413751299:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13308
2025-11-19T13:58:45.327653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435638413751300:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13308
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.328053Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435638413751299:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.328089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435638413751217:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.328116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639713950577:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50703
2025-11-19T13:58:45.328152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639713950578:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50703
2025-11-19T13:58:45.328163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639713950579:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50703
2025-11-19T13:58:45.328243Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642028789298:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.328256Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435642028789298:2107] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.329117Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435639684481648:2102] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.329873Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435639684481650:2103] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:45.330327Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435639684481678:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.330345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435639684481634:2096] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.330363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:45.330372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435639684481634:2096] [node 4] ICP22 created new session: [1:7574435639684481681:2096]
2025-11-19T13:58:45.330378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435639684481681:2096] [node 4] ICS09 handshake done sender: [1:7574435639684481648:2102] self: [1:7574435639684481647:0] peer: [4:7574435638413751297:0] socket: 70 qp: -1
2025-11-19T13:58:45.330384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435639684481681:2096] [node 4] ICS10 traffic start
2025-11-19T13:58:45.330407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:45.330530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435639684481645:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.330536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:45.330539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435639684481645:2101] [node 6] ICP22 created new session: [1:7574435639684481683:2101]
2025-11-19T13:58:45.330543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435639684481683:2101] [node 6] ICS09 handshake done sender: [1:7574435639684481650:2103] self: [1:7574435639684481649:0] peer: [6:7574435639713950576:0] socket: 92 qp: -1
2025-11-19T13:58:45.330548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435639684481683:2101] [node 6] ICS10 traffic start
2025-11-19T13:58:45.330553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.331363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639713950580:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50703
2025-11-19T13:58:45.331924Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435639713950575:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:45.333843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435639713950499:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.333865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:45.333873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435639713950499:2048] [node 1] ICP22 created new session: [6:7574435639713950581:2048]
2025-11-19T13:58:45.333880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435639713950581:2048] [node 1] ICS09 handshake done sender: [6:7574435639713950575:2114] self: [6:7574435639713950576:0] peer: [1:7574435639684481649:0] socket: 67 qp: -1
2025-11-19T13:58:45.333888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435639713950581:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:45.333915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:45.364607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435639833501072:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21604
2025-11-19T13:58:45.368270Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435639833501064:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.376405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435639833500986:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.376433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:45.376443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435639833500986:2050] [node 3] ICP22 created new session: [5:7574435639833501073:2050]
2025-11-19T13:58:45.376450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435639833501073:2050] [node 3] ICS09 handshake done sender: [5:7574435639833501064:2116] self: [5:7574435639833501066:0] peer: [3:7574435642152691181:0] socket: 87 qp: -1
2025-11-19T13:58:45.376462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435639833501073:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:45.376475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:45.376009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435642028789325:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-11-19T13:58:45.377934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435639713950499:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(44061110ns) {HandleHandshakeStatus:329(44060062ns) {DropIncomingHandshake:441(236ns) DropOutgoingHandshake:456(1749ns) SwitchToState:178(189ns) ProcessPendingSessionEvents:495(43917729ns) {ForwardSessionEventToSession:612(43901923ns) ForwardSessionEventToSession:612(2918ns) ForwardSessionEventToSession:612(1335ns) ForwardSessionEventToSession:612(620ns) ForwardSessionEventToSession:612(603ns) ForwardSessionEventToSession:612(1246ns) ForwardSessionEventToSession:612(808ns) ForwardSessionEventToSession:612(642ns) ForwardSessionEventToSession:612(942ns) ForwardSessionEventToSession:612(943ns)}}}
2025-11-19T13:58:45.378307Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435642028789298:2107] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:45.378027Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435639713950577:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.378049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435639713950579:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.378216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435639713950525:2070] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.378227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435639713950501:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.378564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435642028789293:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.378586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:45.378594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435642028789293:2103] [node 4] ICP22 created new session: [2:7574435642028789327:2103]
2025-11-19T13:58:45.378797Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435642152691194:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.378809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435642152691194:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.380357Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435639833501061:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.380369Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435639833501061:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.381171Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435639833501061:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:45.381396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435639833501027:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.381413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:45.381420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435639833501027:2077] [node 6] ICP22 created new session: [5:7574435639833501076:2077]
2025-11-19T13:58:45.381427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435639833501076:2077] [node 6] ICS09 handshake done sender: [5:7574435639833501061:2114] self: [5:7574435639833501028:0] peer: [6:7574435639713950583:0] socket: 97 qp: -1
2025-11-19T13:58:45.381433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435639833501076:2077] [node 6] ICS10 traffic start
2025-11-19T13:58:45.381453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.381851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639713950585:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50703
2025-11-19T13:58:45.381871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639713950586:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50703
2025-11-19T13:58:45.382200Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435639713950578:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.382224Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435639713950577:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:45.382321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435639713950525:2070] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.382335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:45.382342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435639713950525:2070] [node 5] ICP22 created new session: [6:7574435639713950587:2070]
2025-11-19T13:58:45.382347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435639713950587:2070] [node 5] ICS09 handshake done sender: [6:7574435639713950577:2115] self: [6:7574435639713950583:0] peer: [5:7574435639833501028:0] socket: 99 qp: -1
2025-11-19T13:58:45.382353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435639713950587:2070] [node 5] ICS10 traffic start
2025-11-19T13:58:45.382361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:45.382484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435639713950500:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.383957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435639713950574:2082] [node 4] ICP01 ready to work
2025-11-19T13:58:45.378599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435642028789327:2103] [node 4] ICS09 handshake done sender: [2:7574435642028789298:2107] self: [2:7574435642028789297:0] peer: [4:7574435638413751301:0] socket: 74 qp: -1
2025-11-19T13:58:45.378605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435642028789327:2103] [node 4] ICS10 traffic start
2025-11-19T13:58:45.378614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:45.383009Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642028789310:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.383014Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435642028789310:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.383723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435642028789329:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-11-19T13:58:45.383873Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435642028789310:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:45.384100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435642028789301:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.384109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:45.384115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435642028789301:2109] [node 6] ICP22 created new session: [2:7574435642028789331:2109]
2025-11-19T13:58:45.384121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435642028789331:2109] [node 6] ICS09 handshake done sender: [2:7574435642028789310:2116] self: [2:7574435642028789302:0] peer: [6:7574435639713950589:0] socket: 79 qp: -1
2025-11-19T13:58:45.383894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435638413751296:2098] [node 6] ICP01 ready to work
2025-11-19T13:58:45.384329Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435642152691194:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:45.384414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435642152691178:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.384424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:45.384429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435642152691178:2098] [node 6] ICP22 created new session: [3:7574435642152691210:2098]
2025-11-19T13:58:45.384433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435642152691210:2098] [node 6] ICS09 handshake done sender: [3:7574435642152691194:2119] self: [3:7574435642152691187:0] peer: [6:7574435639713950584:0] socket: 81 qp: -1
2025-11-19T13:58:45.384437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435642152691210:2098] [node 6] ICS10 traffic start
2025-11-19T13:58:45.384444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.387254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435639833501078:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21604
2025-11-19T13:58:45.388099Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435639833501078:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.388640Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435639684481663:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.388659Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435639684481663:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.389427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435639833500984:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.390041Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435639713950579:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.390590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639713950592:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50703
2025-11-19T13:58:45.390759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435639713950501:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.390776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:45.390782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435639713950501:2050] [node 3] ICP22 created new session: [6:7574435639713950593:2050]
2025-11-19T13:58:45.390789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435639713950593:2050] [node 3] ICS09 handshake done sender: [6:7574435639713950579:2117] self: [6:7574435639713950584:0] peer: [3:7574435642152691187:0] socket: 109 qp: -1
2025-11-19T13:58:45.390796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435639713950593:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:45.390818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:45.389220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435642028789331:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:45.389261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.391339Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435639713950578:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.399035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435639833501080:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21604
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.399275Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435639684481663:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:45.399418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435639684481644:2100] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.399437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:45.399445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435639684481644:2100] [node 5] ICP22 created new session: [1:7574435639684481687:2100]
2025-11-19T13:58:45.399452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435639684481687:2100] [node 5] ICS09 handshake done sender: [1:7574435639684481663:2115] self: [1:7574435639684481646:0] peer: [5:7574435639833501079:0] socket: 104 qp: -1
2025-11-19T13:58:45.399460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435639684481687:2100] [node 5] ICS10 traffic start
2025-11-19T13:58:45.399471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:45.397751Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435638413751292:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:45.398217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435638413751216:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24364) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435638413751292:2115] held: no
2025-11-19T13:58:45.398277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435638413751217:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24550) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435638413751299:2118] held: no
2025-11-19T13:58:45.398432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435638413751302:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13308
2025-11-19T13:58:45.399279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435638413751218:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24207) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435638413751261:2092] held: no
2025-11-19T13:58:45.399367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435638413751218:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(79293250ns) {HandleHandshakeStatus:395(79292599ns) {LogHandshakeFail:472(3869ns) {UpdateErrorStateLog:544(1703ns)} DropOutgoingHandshake:456(448ns) ProcessPendingSessionEvents:495(26935ns) {DropSessionEvent:512(13769ns) DropSessionEvent:512(1309ns) DropSessionEvent:512(1042ns) DropSessionEvent:512(2635ns) DropSessionEvent:512(1179ns) DropSessionEvent:512(1099ns) DropSessionEvent:512(1616ns) DropSessionEvent:512(1162ns) DropSessionEvent:512(1103ns) DropSessionEvent:512(845ns)}}}
2025-11-19T13:58:45.399553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435638413751305:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13308
2025-11-19T13:58:45.399569Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435638413751261:2092] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.400349Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435639833501078:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:45.400399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435639833501081:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21604
2025-11-19T13:58:45.400584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435639833500984:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.400604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:45.400615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435639833500984:2048] [node 1] ICP22 created new session: [5:7574435639833501082:2048]
2025-11-19T13:58:45.400621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435639833501082:2048] [node 1] ICS09 handshake done sender: [5:7574435639833501078:2120] self: [5:7574435639833501079:0] peer: [1:7574435639684481646:0] socket: 110 qp: -1
2025-11-19T13:58:45.400628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435639833501082:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:45.400635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:45.402611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435639713950500:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.402628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:45.402634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435639713950500:2049] [node 2] ICP22 created new session: [6:7574435639713950598:2049]
2025-11-19T13:58:45.402640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435639713950598:2049] [node 2] ICS09 handshake done sender: [6:7574435639713950578:2116] self: [6:7574435639713950589:0] peer: [2:7574435642028789302:0] socket: 60 qp: -1
2025-11-19T13:58:45.402646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435639713950598:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:45.402654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:45.401565Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642028789329:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.401618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435642028789301:2109] [node 6] ICP18 handshake (actor [2:7574435642028789329:2120]) is too old
2025-11-19T13:58:45.402400Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435638413751299:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:45.402530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435638413751218:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.402540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:45.402553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435638413751218:2050] [node 3] ICP22 created new session: [4:7574435638413751308:2050]
2025-11-19T13:58:45.402560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435638413751308:2050] [node 3] ICS09 handshake done sender: [4:7574435638413751261:2092] self: [4:7574435638413751294:0] peer: [3:7574435642152691155:0] socket: 57 qp: -1
2025-11-19T13:58:45.402565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435638413751308:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:45.402586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.404210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639713950597:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50703
2025-11-19T13:58:45.405958Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435639833501081:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.406021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435639833501023:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.407036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435639833501085:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21604
2025-11-19T13:58:45.407329Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435639833501081:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:45.407455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435639833501023:2084] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.405640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435638413751216:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.405658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:45.405664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435638413751216:2048] [node 1] ICP22 created new session: [4:7574435638413751310:2048]
2025-11-19T13:58:45.405670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435638413751310:2048] [node 1] ICS09 handshake done sender: [4:7574435638413751292:2115] self: [4:7574435638413751297:0] peer: [1:7574435639684481647:0] socket: 71 qp: -1
2025-11-19T13:58:45.405676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435638413751310:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:45.405682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:45.406343Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435638413751291:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.406348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435638413751291:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.406672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435638413751217:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.406680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:45.406686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435638413751217:2049] [node 2] ICP22 created new session: [4:7574435638413751312:2049]
2025-11-19T13:58:45.406690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435638413751312:2049] [node 2] ICS09 handshake done sender: [4:7574435638413751299:2118] self: [4:7574435638413751301:0] peer: [2:7574435642028789297:0] socket: 107 qp: -1
2025-11-19T13:58:45.406696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435638413751312:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:45.406701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:45.406965Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435638413751291:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:45.407410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435638413751268:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.407421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:45.407427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435638413751268:2097] [node 5] ICP22 created new session: [4:7574435638413751315:2097]
2025-11-19T13:58:45.407431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435638413751315:2097] [node 5] ICS09 handshake done sender: [4:7574435638413751291:2114] self: [4:7574435638413751269:0] peer: [5:7574435639833501084:0] socket: 77 qp: -1
2025-11-19T13:58:45.407435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435638413751315:2097] [node 5] ICS10 traffic start
2025-11-19T13:58:45.407444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:45.407716Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435638413751305:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.408367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:45.408382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435639833501023:2084] [node 4] ICP22 created new session: [5:7574435639833501086:2084]
2025-11-19T13:58:45.408388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435639833501086:2084] [node 4] ICS09 handshake done sender: [5:7574435639833501081:2122] self: [5:7574435639833501084:0] peer: [4:7574435638413751269:0] socket: 118 qp: -1
2025-11-19T13:58:45.408394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435639833501086:2084] [node 4] ICS10 traffic start
2025-11-19T13:58:45.408404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:45.410071Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435639713950597:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.410120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435639713950574:2082] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.411233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639713950601:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50703
2025-11-19T13:58:45.410322Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435638413751304:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.410333Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435638413751304:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.413767Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435638413751304:2121] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:45.414023Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435639713950597:2123] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:45.414195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435639713950574:2082] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.414214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:45.414221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435639713950574:2082] [node 4] ICP22 created new session: [6:7574435639713950602:2082]
2025-11-19T13:58:45.414228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435639713950602:2082] [node 4] ICS09 handshake done sender: [6:7574435639713950597:2123] self: [6:7574435639713950600:0] peer: [4:7574435638413751303:0] socket: 91 qp: -1
2025-11-19T13:58:45.414233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435639713950602:2082] [node 4] ICS10 traffic start
2025-11-19T13:58:45.414248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:45.415975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435638413751296:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.416005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:45.416012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435638413751296:2098] [node 6] ICP22 created new session: [4:7574435638413751319:2098]
2025-11-19T13:58:45.416017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435638413751319:2098] [node 6] ICS09 handshake done sender: [4:7574435638413751304:2121] self: [4:7574435638413751303:0] peer: [6:7574435639713950600:0] socket: 85 qp: -1
2025-11-19T13:58:45.416024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435638413751319:2098] [node 6] ICS10 traffic start
2025-11-19T13:58:45.416048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:45.430742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:45.430832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435642028789317:2048] [node 1] ICS01 socket: 66 reason# EndOfStream
2025-11-19T13:58:45.430843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435637733821933:2048] [node 1] ICP30 unregister session Session# [2:7574435642028789317:2048] VirtualId# [2:7574435642028789309:0]
2025-11-19T13:58:45.430852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435642028789317:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.430889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:45.430909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:45.432536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:45.432619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435642152691207:2048] [node 1] ICS01 socket: 69 reason# EndOfStream
2025-11-19T13:58:45.432628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435637857723811:2048] [node 1] ICP30 unregister session Session# [3:7574435642152691207:2048] VirtualId# [3:7574435642152691193:0]
2025-11-19T13:58:45.432633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435642152691207:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.432666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:45.432684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:45.435842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:45.435923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435638413751310:2048] [node 1] ICS01 socket: 71 reason# EndOfStream
2025-11-19T13:58:45.435931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435638413751216:2048] [node 1] ICP30 unregister session Session# [4:7574435638413751310:2048] VirtualId# [4:7574435638413751297:0]
2025-11-19T13:58:45.435938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435638413751310:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.435982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:45.436010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:45.445136Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:45.445229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435639713950581:2048] [node 1] ICS07 socket disconnect 67 reason# EndOfStream
2025-11-19T13:58:45.445235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435639713950581:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.445291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:45.445313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435639713950581:2048] [node 1] ICS15 start handshake
2025-11-19T13:58:45.445751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:45.445819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435639833501082:2048] [node 1] ICS01 socket: 110 reason# EndOfStream
2025-11-19T13:58:45.445824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435639833500984:2048] [node 1] ICP30 unregister session Session# [5:7574435639833501082:2048] VirtualId# [5:7574435639833501079:0]
2025-11-19T13:58:45.445830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435639833501082:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.445856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:45.445882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:45.448055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435639713950499:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24364) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:45.448073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435639713950581:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:45.448079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435639713950499:2048] [node 1] ICP30 unregister session Session# [6:7574435639713950581:2048] VirtualId# [6:7574435639713950576:0]
2025-11-19T13:58:45.448096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:45.448101Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435639713950499:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24364) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:45.448104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24364) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:45.456877Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:45.456992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435642152691204:2049] [node 2] ICS01 socket: 68 reason# EndOfStream
2025-11-19T13:58:45.457004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435637857723812:2049] [node 2] ICP30 unregister session Session# [3:7574435642152691204:2049] VirtualId# [3:7574435642152691190:0]
2025-11-19T13:58:45.457012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435642152691204:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.457065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:45.457089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:45.458243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435639713950499:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:58:45.463990Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:45.464083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435639833501068:2049] [node 2] ICS01 socket: 64 reason# EndOfStream
2025-11-19T13:58:45.464094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435639833500985:2049] [node 2] ICP30 unregister session Session# [5:7574435639833501068:2049] VirtualId# [5:7574435639833501062:0]
2025-11-19T13:58:45.464101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435639833501068:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.464158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:45.464186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:45.464228Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:45.464288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435638413751312:2049] [node 2] ICS01 socket: 107 reason# EndOfStream
2025-11-19T13:58:45.464293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435638413751217:2049] [node 2] ICP30 unregister session Session# [4:7574435638413751312:2049] VirtualId# [4:7574435638413751301:0]
2025-11-19T13:58:45.464298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435638413751312:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.464329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:45.464351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:45.464382Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:45.464439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435639713950598:2049] [node 2] ICS01 socket: 60 reason# EndOfStream
2025-11-19T13:58:45.464444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435639713950500:2049] [node 2] ICP30 unregister session Session# [6:7574435639713950598:2049] VirtualId# [6:7574435639713950589:0]
2025-11-19T13:58:45.464448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435639713950598:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.464477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:45.464498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:45.489022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T13:58:45.489130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435638413751308:2050] [node 3] ICS01 socket: 57 reason# EndOfStream
2025-11-19T13:58:45.489142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435638413751218:2050] [node 3] ICP30 unregister session Session# [4:7574435638413751308:2050] VirtualId# [4:7574435638413751294:0]
2025-11-19T13:58:45.489150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435638413751308:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.489222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:45.489250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:45.489388Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:45.489451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435639713950593:2050] [node 3] ICS01 socket: 109 reason# EndOfStream
2025-11-19T13:58:45.489456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435639713950501:2050] [node 3] ICP30 unregister session Session# [6:7574435639713950593:2050] VirtualId# [6:7574435639713950584:0]
2025-11-19T13:58:45.489462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435639713950593:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.489499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:45.489522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:45.489625Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:45.489691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435639833501073:2050] [node 3] ICS01 socket: 87 reason# EndOfStream
2025-11-19T13:58:45.489695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435639833500986:2050] [node 3] ICP30 unregister session Session# [5:7574435639833501073:2050] VirtualId# [5:7574435639833501066:0]
2025-11-19T13:58:45.489701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435639833501073:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.489737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:45.489759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:45.503990Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:45.504085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435639713950602:2082] [node 4] ICS01 socket: 91 reason# EndOfStream
2025-11-19T13:58:45.504094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435639713950574:2082] [node 4] ICP30 unregister session Session# [6:7574435639713950602:2082] VirtualId# [6:7574435639713950600:0]
2025-11-19T13:58:45.504101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435639713950602:2082] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.504167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:45.504192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:58:45.514683Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:58:45.514798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435639713950587:2070] [node 5] ICS01 socket: 99 reason# EndOfStream
2025-11-19T13:58:45.514808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435639713950525:2070] [node 5] ICP30 unregister session Session# [6:7574435639713950587:2070] VirtualId# [6:7574435639713950583:0]
2025-11-19T13:58:45.514817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435639713950587:2070] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:45.514892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:58:45.514922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:45.596009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435642290364169:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:45.596023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435642290364170:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:45.621647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435642618670316:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:45.621664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435642618670317:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:45.655922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435639364661589:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:45.655939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435639364661590:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:45.694198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435640646462610:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:45.694224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435640646462611:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:45.694229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435640646462612:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:45.724153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435641268598576:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:45.724169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435641268598577:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:45.724173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435641268598578:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:45.827781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435639750944424:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:45.827794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435639750944422:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:45.827852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435639750944423:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:45.840181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435641268598601:2068] [node 6] ICP01 ready to work
2025-11-19T13:58:45.840386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435639750944436:2059] [node 4] ICP01 ready to work
2025-11-19T13:58:45.840485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435641268598605:2074] [node 4] ICP01 ready to work
2025-11-19T13:58:45.840985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435639364661636:2091] [node 6] ICP01 ready to work
2025-11-19T13:58:45.841436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435642290364234:2107] [node 5] ICP01 ready to work
2025-11-19T13:58:45.841509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435642290364235:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.841405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435642618670370:2100] [node 4] ICP01 ready to work
2025-11-19T13:58:45.842160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435642618670375:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.844008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435641268598620:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15686
2025-11-19T13:58:45.849482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435642618670377:2104] [node 5] ICP01 ready to work
2025-11-19T13:58:45.850776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435639750944450:2068] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.851598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435641268598628:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15686
2025-11-19T13:58:45.851614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435641268598629:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15686
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.855772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435642290364244:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44821
2025-11-19T13:58:45.859944Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435641268598629:2091] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.859884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435639750944424:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19433221ns) {Configure:104(19432522ns) {ProcessConfigured:135(18626810ns) {StartInitialHandshake:157(18625886ns) {DropHandshakes:469(315ns) {DropIncomingHandshake:441(23ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(98ns) PrepareNewSessionHandshake:476(786ns) {SwitchToState:178(260ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.861446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435639364661635:2089] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.864252Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435640646462612:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21701820ns) {Configure:104(21701274ns) {ProcessConfigured:135(21364187ns) {StartInitialHandshake:157(21363416ns) {DropHandshakes:469(431ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(49ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:476(756ns) {SwitchToState:178(290ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.869587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435641268598601:2068] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(29343793ns) {Configure:104(29343094ns) {ProcessConfigured:135(25553758ns) {StartInitialHandshake:157(25552307ns) {DropHandshakes:469(276ns) {DropIncomingHandshake:441(40ns) DropOutgoingHandshake:456(30ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(1265ns) {SwitchToState:178(335ns)}}}}}
2025-11-19T13:58:45.870180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435642618670386:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
2025-11-19T13:58:45.869786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435640646462682:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3710
2025-11-19T13:58:45.869803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435640646462683:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3710
2025-11-19T13:58:45.870341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435642618670374:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
2025-11-19T13:58:45.871707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435642618670384:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
2025-11-19T13:58:45.871823Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642618670374:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.871850Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642618670386:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.876269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435639364661662:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53236
2025-11-19T13:58:45.876313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435639364661663:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53236
2025-11-19T13:58:45.876323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435639364661664:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53236
2025-11-19T13:58:45.876343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435639364661665:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53236
2025-11-19T13:58:45.876421Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435639750944436:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(35974322ns) {Configure:104(35973887ns) {ProcessConfigured:135(35573119ns) {StartInitialHandshake:157(35572315ns) {DropHandshakes:469(480ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(76ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:476(760ns) {SwitchToState:178(294ns)}}}}}
2025-11-19T13:58:45.876908Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642618670384:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.877194Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435639364661662:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.877312Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435642290364244:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.879857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435640646462610:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(36919366ns) {Configure:104(19090946ns) {ProcessConfigured:135(18757482ns) {StartInitialHandshake:157(18756858ns) {DropHandshakes:469(1096ns) {DropIncomingHandshake:441(323ns) DropOutgoingHandshake:456(168ns)} GenerateSessionVirtualId:988(354ns) PrepareNewSessionHandshake:476(1261ns) {SwitchToState:178(643ns)}}}}}
2025-11-19T13:58:45.880329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435639364661635:2089] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18829600ns) {Configure:104(18829313ns) {ProcessConfigured:135(18564035ns) {StartInitialHandshake:157(18563677ns) {DropHandshakes:469(401ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(37ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:476(506ns) {SwitchToState:178(197ns)}}}}}
2025-11-19T13:58:45.880164Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435641268598576:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(19301546ns) {Configure:104(19300994ns) {ProcessConfigured:135(18984592ns) {StartInitialHandshake:157(18983917ns) {DropHandshakes:469(489ns) {DropIncomingHandshake:441(94ns) DropOutgoingHandshake:456(77ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:476(793ns) {SwitchToState:178(566ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.880613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435642290364248:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44821
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.880778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435640646462686:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3710
2025-11-19T13:58:45.884235Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435639750944422:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(30257004ns) {Configure:104(30254827ns) {ProcessConfigured:135(29983702ns) {StartInitialHandshake:157(29982899ns) {DropHandshakes:469(412ns) {DropIncomingHandshake:441(95ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:476(1370ns) {SwitchToState:178(681ns)}}}}}
2025-11-19T13:58:45.887773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435640646462671:2089] [node 5] ICP01 ready to work
2025-11-19T13:58:45.887805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435640646462665:2091] [node 6] ICP01 ready to work
2025-11-19T13:58:45.889241Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435639364661664:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.885023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435642290364246:2112] [node 4] ICP01 ready to work
2025-11-19T13:58:45.887970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435642618670389:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.892226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435642618670398:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
2025-11-19T13:58:45.892498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435642618670316:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(22205048ns) {Configure:104(22203802ns) {ProcessConfigured:135(21936898ns) {StartInitialHandshake:157(21936512ns) {DropHandshakes:469(525ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(170ns) PrepareNewSessionHandshake:476(860ns) {SwitchToState:178(247ns)}}}}}
2025-11-19T13:58:45.892313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435639364661636:2091] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(51274807ns) {Configure:104(51274342ns) {ProcessConfigured:135(51065452ns) {StartInitialHandshake:157(51064847ns) {DropHandshakes:469(258ns) {DropIncomingHandshake:441(44ns) DropOutgoingHandshake:456(29ns)} GenerateSessionVirtualId:988(167ns) PrepareNewSessionHandshake:476(654ns) {SwitchToState:178(235ns)}}}}}
2025-11-19T13:58:45.907690Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435641268598628:2090] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.907766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435641268598577:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.907784Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435640646462686:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.907821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435640646462612:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.907963Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435639364661666:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.907975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435639364661666:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.908204Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642618670375:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.908310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435642618670316:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.908394Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435641268598620:2083] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.908422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435641268598576:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.908625Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435642290364237:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.908629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435642290364246:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(23525143ns) {Configure:104(23524138ns) {ProcessConfigured:135(23199164ns) {StartInitialHandshake:157(23198380ns) {DropHandshakes:469(1133ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(113ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:476(482ns) {SwitchToState:178(150ns)}}}}}
2025-11-19T13:58:45.908637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435642290364237:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.908875Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435640646462682:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.908929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435640646462611:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.908970Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435639364661663:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.909060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435639364661668:2097] [node 5] ICP01 ready to work
2025-11-19T13:58:45.910665Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435640646462683:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.910893Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435642290364233:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.910910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435642290364233:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.911016Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642618670381:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.911027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435642618670381:2106] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.915911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639750944500:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63994
2025-11-19T13:58:45.919942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435642618670389:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(31870837ns) {Configure:104(31870386ns) {ProcessConfigured:135(23924248ns) {StartInitialHandshake:157(23897823ns) {DropHandshakes:469(1127ns) {DropIncomingHandshake:441(349ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(521ns) PrepareNewSessionHandshake:476(2140ns) {SwitchToState:178(1099ns)}} IncomingHandshake:261(10936ns) {DropIncomingHandshake:441(304ns) PrepareNewSessionHandshake:476(144ns) {SwitchToState:178(98ns)} GenerateSessionVirtualId:988(402ns) IssueIncomingHandshakeReply:193(510ns)}}}}
2025-11-19T13:58:45.920081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435640646462671:2089] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(32229507ns) {Configure:104(32229103ns) {ProcessConfigured:135(24052815ns) {StartInitialHandshake:157(24052264ns) {DropHandshakes:469(658ns) {DropIncomingHandshake:441(188ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:476(756ns) {SwitchToState:178(131ns)}}}}}
2025-11-19T13:58:45.921056Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642618670372:2101] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.921068Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435642618670372:2101] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.921136Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642618670379:2105] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.921142Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435642618670379:2105] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.921278Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435642290364237:2110] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:45.921396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435642290364234:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.921412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:45.921420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435642290364234:2107] [node 5] ICP22 created new session: [1:7574435642290364252:2107]
2025-11-19T13:58:45.921425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435642290364252:2107] [node 5] ICS09 handshake done sender: [1:7574435642290364237:2110] self: [1:7574435642290364236:0] peer: [5:7574435641268598659:0] socket: 65 qp: -1
2025-11-19T13:58:45.921432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435642290364252:2107] [node 5] ICS10 traffic start
2025-11-19T13:58:45.921440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:45.922328Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435639750944500:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.922394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435639750944422:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.922793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435639364661590:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.924066Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435639364661666:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:45.924173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435639364661676:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53236
2025-11-19T13:58:45.924300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435641268598660:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15686
2025-11-19T13:58:45.924702Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435641268598620:2083] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:45.924983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435639364661635:2089] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.924999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:45.925006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435639364661635:2089] [node 4] ICP22 created new session: [3:7574435639364661677:2089]
2025-11-19T13:58:45.925011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435639364661677:2089] [node 4] ICS09 handshake done sender: [3:7574435639364661666:2116] self: [3:7574435639364661655:0] peer: [4:7574435640646462690:0] socket: 93 qp: -1
2025-11-19T13:58:45.925019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435639364661677:2089] [node 4] ICS10 traffic start
2025-11-19T13:58:45.925031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.926645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639750944502:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63994
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.929553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435640646462694:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3710
2025-11-19T13:58:45.929931Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435642290364233:2109] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.930013Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435642290364239:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.930024Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435642290364239:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.930725Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435639364661665:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.930819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435639364661680:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53236
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.930901Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435642618670381:2106] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:45.930876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639750944503:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63994
2025-11-19T13:58:45.931000Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435642290364239:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:45.931133Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435639750944500:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:45.931266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435641268598576:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.931291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:45.931297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435641268598576:2048] [node 1] ICP22 created new session: [5:7574435641268598662:2048]
2025-11-19T13:58:45.931302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435641268598662:2048] [node 1] ICS09 handshake done sender: [5:7574435641268598620:2083] self: [5:7574435641268598659:0] peer: [1:7574435642290364236:0] socket: 69 qp: -1
2025-11-19T13:58:45.931309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435641268598662:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:45.931321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:45.931333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435639750944422:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.931357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:45.931365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435639750944422:2048] [node 1] ICP22 created new session: [6:7574435639750944504:2048]
2025-11-19T13:58:45.931349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435640646462695:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3710
2025-11-19T13:58:45.931373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435639750944504:2048] [node 1] ICS09 handshake done sender: [6:7574435639750944500:2115] self: [6:7574435639750944501:0] peer: [1:7574435642290364238:0] socket: 72 qp: -1
2025-11-19T13:58:45.931379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435639750944504:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:45.931403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:45.931333Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435639364661676:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.931403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435639364661589:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.931472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435641268598661:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15686
2025-11-19T13:58:45.931546Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435640646462686:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:45.931551Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435641268598628:2090] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:45.931542Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435639750944502:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.931594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435640646462612:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.931570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435639750944424:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.931596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435641268598577:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.931608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:45.931614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435641268598577:2049] [node 2] ICP22 created new session: [5:7574435641268598664:2049]
2025-11-19T13:58:45.931618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435641268598664:2049] [node 2] ICS09 handshake done sender: [5:7574435641268598628:2090] self: [5:7574435641268598658:0] peer: [2:7574435642618670380:0] socket: 76 qp: -1
2025-11-19T13:58:45.931622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435641268598664:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:45.931636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:45.931704Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435642618670372:2101] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:45.931607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:45.931612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435640646462612:2050] [node 3] ICP22 created new session: [4:7574435640646462696:2050]
2025-11-19T13:58:45.931617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435640646462696:2050] [node 3] ICS09 handshake done sender: [4:7574435640646462686:2115] self: [4:7574435640646462690:0] peer: [3:7574435639364661655:0] socket: 95 qp: -1
2025-11-19T13:58:45.931622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435640646462696:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:45.931628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:45.931852Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435642618670379:2105] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.932228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435642290364235:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.932243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:45.932248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435642290364235:2108] [node 6] ICP22 created new session: [1:7574435642290364257:2108]
2025-11-19T13:58:45.932253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435642290364257:2108] [node 6] ICS09 handshake done sender: [1:7574435642290364239:2111] self: [1:7574435642290364238:0] peer: [6:7574435639750944501:0] socket: 67 qp: -1
2025-11-19T13:58:45.932296Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435639364661663:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:45.932406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435639364661635:2089] [node 4] ICP18 handshake (actor [3:7574435639364661665:2115]) is too old
2025-11-19T13:58:45.933372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435642290364169:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.933390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:45.933397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435642290364169:2048] [node 2] ICP22 created new session: [1:7574435642290364258:2048]
2025-11-19T13:58:45.933404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435642290364258:2048] [node 2] ICS09 handshake done sender: [1:7574435642290364233:2109] self: [1:7574435642290364229:0] peer: [2:7574435642618670401:0] socket: 63 qp: -1
2025-11-19T13:58:45.933411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435642290364258:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:45.933427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:45.932522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435642618670370:2100] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.932536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:45.932547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435642618670370:2100] [node 4] ICP22 created new session: [2:7574435642618670409:2100]
2025-11-19T13:58:45.932553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435642618670409:2100] [node 4] ICS09 handshake done sender: [2:7574435642618670372:2101] self: [2:7574435642618670371:0] peer: [4:7574435640646462691:0] socket: 61 qp: -1
2025-11-19T13:58:45.932558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435642618670409:2100] [node 4] ICS10 traffic start
2025-11-19T13:58:45.932566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.933879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639750944508:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63994
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.934059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639750944509:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63994
2025-11-19T13:58:45.933689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435642618670317:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.933704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:45.933710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435642618670317:2049] [node 3] ICP22 created new session: [2:7574435642618670412:2049]
2025-11-19T13:58:45.933715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435642618670412:2049] [node 3] ICS09 handshake done sender: [2:7574435642618670379:2105] self: [2:7574435642618670378:0] peer: [3:7574435639364661671:0] socket: 73 qp: -1
2025-11-19T13:58:45.933726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435642618670412:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:45.933741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:45.935284Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435639364661669:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.935299Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435639364661669:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.935369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435642618670407:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34139
2025-11-19T13:58:45.935431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435642290364257:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:45.935445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:45.935524Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435642618670375:2103] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:45.935651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435639364661590:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.935665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:45.935669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435639364661590:2049] [node 2] ICP22 created new session: [3:7574435639364661682:2049]
2025-11-19T13:58:45.935675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435639364661682:2049] [node 2] ICS09 handshake done sender: [3:7574435639364661663:2113] self: [3:7574435639364661671:0] peer: [2:7574435642618670378:0] socket: 89 qp: -1
2025-11-19T13:58:45.935679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435639364661682:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:45.935694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:45.944774Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435639750944508:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.944829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435639750944450:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.952054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435639750944422:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20676157ns) {HandleHandshakeStatus:329(20675521ns) {DropIncomingHandshake:441(266ns) DropOutgoingHandshake:456(4629ns) SwitchToState:178(303ns) ProcessPendingSessionEvents:495(19787378ns) {ForwardSessionEventToSession:612(19772624ns) ForwardSessionEventToSession:612(1990ns) ForwardSessionEventToSession:612(1802ns) ForwardSessionEventToSession:612(816ns) ForwardSessionEventToSession:612(583ns) ForwardSessionEventToSession:612(1272ns) ForwardSessionEventToSession:612(755ns) ForwardSessionEventToSession:612(649ns) ForwardSessionEventToSession:612(806ns) ForwardSessionEventToSession:612(1539ns)}}}
2025-11-19T13:58:45.952481Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435640646462612:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(20866238ns) {HandleHandshakeStatus:329(20866002ns) {DropIncomingHandshake:441(130ns) DropOutgoingHandshake:456(989ns) SwitchToState:178(54ns) ProcessPendingSessionEvents:495(20143962ns) {ForwardSessionEventToSession:612(20133687ns) ForwardSessionEventToSession:612(1786ns) ForwardSessionEventToSession:612(1074ns) ForwardSessionEventToSession:612(568ns) ForwardSessionEventToSession:612(620ns) ForwardSessionEventToSession:612(850ns) ForwardSessionEventToSession:612(914ns) ForwardSessionEventToSession:612(515ns) ForwardSessionEventToSession:612(716ns) ForwardSessionEventToSession:612(1013ns)}}}
2025-11-19T13:58:45.952267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435641268598576:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(25885765ns) {HandleHandshakeStatus:329(25885225ns) {DropIncomingHandshake:441(1064ns) DropOutgoingHandshake:456(4680ns) SwitchToState:178(236ns) ProcessPendingSessionEvents:495(19929846ns) {ForwardSessionEventToSession:612(19919680ns) ForwardSessionEventToSession:612(1343ns) ForwardSessionEventToSession:612(983ns) ForwardSessionEventToSession:612(584ns) ForwardSessionEventToSession:612(525ns) ForwardSessionEventToSession:612(1193ns) ForwardSessionEventToSession:612(690ns) ForwardSessionEventToSession:612(873ns) ForwardSessionEventToSession:612(518ns) ForwardSessionEventToSession:612(784ns)}}}
2025-11-19T13:58:45.956255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435642290364234:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(34829088ns) {HandleHandshakeStatus:329(34827899ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(297ns) SwitchToState:178(103ns) ProcessPendingSessionEvents:495(23890179ns) {ForwardSessionEventToSession:612(23878246ns) ForwardSessionEventToSession:612(2120ns) ForwardSessionEventToSession:612(1060ns) ForwardSessionEventToSession:612(509ns) ForwardSessionEventToSession:612(634ns) ForwardSessionEventToSession:612(760ns) ForwardSessionEventToSession:612(896ns) ForwardSessionEventToSession:612(564ns) ForwardSessionEventToSession:612(506ns) ForwardSessionEventToSession:612(1167ns)}}}
2025-11-19T13:58:45.957605Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435641268598655:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.957617Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435641268598655:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.959819Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435642290364245:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.959840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435642290364245:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.957218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435642618670316:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.957244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:45.957251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435642618670316:2048] [node 1] ICP22 created new session: [2:7574435642618670414:2048]
2025-11-19T13:58:45.957259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435642618670414:2048] [node 1] ICS09 handshake done sender: [2:7574435642618670375:2103] self: [2:7574435642618670401:0] peer: [1:7574435642290364229:0] socket: 66 qp: -1
2025-11-19T13:58:45.957266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435642618670414:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:45.957495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435642618670377:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.957510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:45.957516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435642618670377:2104] [node 5] ICP22 created new session: [2:7574435642618670415:2104]
2025-11-19T13:58:45.957523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435642618670415:2104] [node 5] ICS09 handshake done sender: [2:7574435642618670381:2106] self: [2:7574435642618670380:0] peer: [5:7574435641268598658:0] socket: 74 qp: -1
2025-11-19T13:58:45.957526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435642618670415:2104] [node 5] ICS10 traffic start
2025-11-19T13:58:45.957549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:45.959988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435639364661590:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(24304580ns) {HandleHandshakeStatus:329(24303915ns) {DropIncomingHandshake:441(148ns) DropOutgoingHandshake:456(963ns) SwitchToState:178(79ns) ProcessPendingSessionEvents:495(39005ns) {ForwardSessionEventToSession:612(28642ns) ForwardSessionEventToSession:612(641ns) ForwardSessionEventToSession:612(1125ns) ForwardSessionEventToSession:612(695ns) ForwardSessionEventToSession:612(920ns) ForwardSessionEventToSession:612(1584ns) ForwardSessionEventToSession:612(716ns) ForwardSessionEventToSession:612(792ns) ForwardSessionEventToSession:612(630ns) ForwardSessionEventToSession:612(1125ns)}}}
2025-11-19T13:58:45.960086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435639364661590:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.961091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435640646462698:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3710
2025-11-19T13:58:45.961109Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435640646462682:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:45.963836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435640646462611:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.963859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:45.963867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435640646462611:2049] [node 2] ICP22 created new session: [4:7574435640646462699:2049]
2025-11-19T13:58:45.963874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435640646462699:2049] [node 2] ICS09 handshake done sender: [4:7574435640646462682:2112] self: [4:7574435640646462691:0] peer: [2:7574435642618670371:0] socket: 70 qp: -1
2025-11-19T13:58:45.963881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435640646462699:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:45.963903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:45.963690Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435642290364169:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(30294656ns) {HandleHandshakeStatus:329(30294166ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(153ns) SwitchToState:178(67ns) ProcessPendingSessionEvents:495(25697896ns) {ForwardSessionEventToSession:612(25685866ns) ForwardSessionEventToSession:612(1511ns) ForwardSessionEventToSession:612(1022ns) ForwardSessionEventToSession:612(862ns) ForwardSessionEventToSession:612(657ns) ForwardSessionEventToSession:612(721ns) ForwardSessionEventToSession:612(592ns) ForwardSessionEventToSession:612(515ns) ForwardSessionEventToSession:612(499ns) ForwardSessionEventToSession:612(1066ns)}}}
2025-11-19T13:58:45.964481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:45.964774Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435639364661669:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:45.968161Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435642290364235:2108] [node 6] ICP35 event processing took too much time PendingConnection:278(36932847ns) {HandleHandshakeStatus:329(36931946ns) {DropIncomingHandshake:441(268ns) DropOutgoingHandshake:456(198ns) SwitchToState:178(337ns) ProcessPendingSessionEvents:495(10527535ns) {ForwardSessionEventToSession:612(10515317ns) ForwardSessionEventToSession:612(1164ns) ForwardSessionEventToSession:612(929ns) ForwardSessionEventToSession:612(893ns) ForwardSessionEventToSession:612(640ns) ForwardSessionEventToSession:612(1689ns) ForwardSessionEventToSession:612(673ns) ForwardSessionEventToSession:612(634ns) ForwardSessionEventToSession:612(530ns) ForwardSessionEventToSession:612(783ns)}}}
2025-11-19T13:58:45.969748Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435641268598655:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:45.969905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435641268598601:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.969926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:45.969937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435641268598601:2068] [node 6] ICP22 created new session: [5:7574435641268598668:2068]
2025-11-19T13:58:45.969943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435641268598668:2068] [node 6] ICS09 handshake done sender: [5:7574435641268598655:2116] self: [5:7574435641268598623:0] peer: [6:7574435639750944510:0] socket: 62 qp: -1
2025-11-19T13:58:45.969949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435641268598668:2068] [node 6] ICS10 traffic start
2025-11-19T13:58:45.969957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:45.970293Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435642290364245:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:45.970382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435642290364170:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.970396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:45.970401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435642290364170:2049] [node 3] ICP22 created new session: [1:7574435642290364263:2049]
2025-11-19T13:58:45.970406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435642290364263:2049] [node 3] ICS09 handshake done sender: [1:7574435642290364245:2114] self: [1:7574435642290364230:0] peer: [3:7574435639364661681:0] socket: 99 qp: -1
2025-11-19T13:58:45.970410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435642290364263:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:45.970414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:45.970355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435639364661636:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.970387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:45.970396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435639364661636:2091] [node 6] ICP22 created new session: [3:7574435639364661685:2091]
2025-11-19T13:58:45.970403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435639364661685:2091] [node 6] ICS09 handshake done sender: [3:7574435639364661669:2117] self: [3:7574435639364661637:0] peer: [6:7574435639750944506:0] socket: 102 qp: -1
2025-11-19T13:58:45.970409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435639364661685:2091] [node 6] ICS10 traffic start
2025-11-19T13:58:45.970418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.974193Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642618670402:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.974205Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435642618670402:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.975009Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435642618670402:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:45.975085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435642618670389:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.975099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:45.975104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435642618670389:2108] [node 6] ICP22 created new session: [2:7574435642618670419:2108]
2025-11-19T13:58:45.975115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435642618670419:2108] [node 6] ICS09 handshake done sender: [2:7574435642618670402:2119] self: [2:7574435642618670400:0] peer: [6:7574435639750944514:0] socket: 116 qp: -1
2025-11-19T13:58:45.975120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435642618670419:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:45.975125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.973082Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435639750944509:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.973132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639750944511:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63994
2025-11-19T13:58:45.973283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639750944512:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63994
2025-11-19T13:58:45.973389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639750944513:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63994
2025-11-19T13:58:45.973442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435639750944423:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.973451Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435639750944502:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:45.973539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435639750944424:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.973551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:45.973556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435639750944424:2050] [node 3] ICP22 created new session: [6:7574435639750944515:2050]
2025-11-19T13:58:45.973560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435639750944515:2050] [node 3] ICS09 handshake done sender: [6:7574435639750944502:2116] self: [6:7574435639750944506:0] peer: [3:7574435639364661637:0] socket: 103 qp: -1
2025-11-19T13:58:45.973566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435639750944515:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:45.973572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:45.975460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639750944517:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63994
2025-11-19T13:58:45.975852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435642618670377:2104] [node 5] ICP35 event processing took too much time PendingConnection:278(43816106ns) {HandleHandshakeStatus:329(43815390ns) {DropIncomingHandshake:441(4040ns) DropOutgoingHandshake:456(226ns) SwitchToState:178(54ns) ProcessPendingSessionEvents:495(11247034ns) {ForwardSessionEventToSession:612(11235644ns) ForwardSessionEventToSession:612(1417ns) ForwardSessionEventToSession:612(1118ns) ForwardSessionEventToSession:612(551ns) ForwardSessionEventToSession:612(711ns) ForwardSessionEventToSession:612(1363ns) ForwardSessionEventToSession:612(533ns) ForwardSessionEventToSession:612(972ns) ForwardSessionEventToSession:612(707ns) ForwardSessionEventToSession:612(799ns)}}}
2025-11-19T13:58:45.979938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435642618670398:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.979991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435642618670370:2100] [node 4] ICP18 handshake (actor [2:7574435642618670398:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.980936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435640646462702:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3710
2025-11-19T13:58:45.981634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435641268598670:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15686
2025-11-19T13:58:45.981218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435639364661589:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5551) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435639364661676:2119] held: no
2025-11-19T13:58:45.981332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435639364661687:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53236
2025-11-19T13:58:45.988266Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435639750944513:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.988363Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435639750944423:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23429) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435639750944509:2119] held: no
2025-11-19T13:58:45.988421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435639750944436:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.988576Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435640646462689:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.988587Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435640646462689:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.989443Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435640646462689:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:45.989552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435640646462665:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.989569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:45.989578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435640646462665:2091] [node 6] ICP22 created new session: [4:7574435640646462704:2091]
2025-11-19T13:58:45.989584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435640646462704:2091] [node 6] ICS09 handshake done sender: [4:7574435640646462689:2116] self: [4:7574435640646462688:0] peer: [6:7574435639750944518:0] socket: 85 qp: -1
2025-11-19T13:58:45.989592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435640646462704:2091] [node 6] ICS10 traffic start
2025-11-19T13:58:45.989600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:45.988003Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435639364661676:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:45.990358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435639364661589:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.990375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:45.990382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435639364661589:2048] [node 1] ICP22 created new session: [3:7574435639364661689:2048]
2025-11-19T13:58:45.990387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435639364661689:2048] [node 1] ICS09 handshake done sender: [3:7574435639364661676:2119] self: [3:7574435639364661681:0] peer: [1:7574435642290364230:0] socket: 100 qp: -1
2025-11-19T13:58:45.990393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435639364661689:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:45.990407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.991149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435641268598670:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.991221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435641268598671:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15686
2025-11-19T13:58:45.991254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435641268598578:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:45.991643Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435641268598671:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.991672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435641268598605:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.991996Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435639750944509:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:45.992027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435639750944519:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63994
2025-11-19T13:58:45.992084Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435640646462692:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:45.992095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435640646462692:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:45.992115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435639750944423:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.992129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:45.992135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435639750944423:2049] [node 2] ICP22 created new session: [6:7574435639750944520:2049]
2025-11-19T13:58:45.992140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435639750944520:2049] [node 2] ICS09 handshake done sender: [6:7574435639750944509:2119] self: [6:7574435639750944514:0] peer: [2:7574435642618670400:0] socket: 117 qp: -1
2025-11-19T13:58:45.992146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435639750944520:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:45.992166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:45.992261Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435639750944513:2122] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:45.996560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435639750944450:2068] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17981) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435639750944508:2118] held: no
2025-11-19T13:58:45.996622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435639750944450:2068] [node 5] ICP35 event processing took too much time PendingConnection:278(25504140ns) {HandleHandshakeStatus:395(25503294ns) {LogHandshakeFail:472(2882ns) {UpdateErrorStateLog:544(736ns)} DropOutgoingHandshake:456(378ns) ProcessPendingSessionEvents:495(12079ns) {DropSessionEvent:512(3045ns) DropSessionEvent:512(1187ns) DropSessionEvent:512(807ns) DropSessionEvent:512(826ns) DropSessionEvent:512(706ns) DropSessionEvent:512(1168ns) DropSessionEvent:512(698ns) DropSessionEvent:512(843ns) DropSessionEvent:512(772ns) DropSessionEvent:512(877ns)}}}
2025-11-19T13:58:45.996632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435639750944436:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:45.996644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:45.996651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435639750944436:2059] [node 4] ICP22 created new session: [6:7574435639750944522:2059]
2025-11-19T13:58:45.996655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435639750944522:2059] [node 4] ICS09 handshake done sender: [6:7574435639750944513:2122] self: [6:7574435639750944518:0] peer: [4:7574435640646462688:0] socket: 81 qp: -1
2025-11-19T13:58:45.996661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435639750944522:2059] [node 4] ICS10 traffic start
2025-11-19T13:58:45.996669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:45.996966Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435639750944508:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:45.997306Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435640646462692:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:45.997501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435640646462671:2089] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:45.997515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:45.997520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435640646462671:2089] [node 5] ICP22 created new session: [4:7574435640646462708:2089]
2025-11-19T13:58:45.997526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435640646462708:2089] [node 5] ICS09 handshake done sender: [4:7574435640646462692:2117] self: [4:7574435640646462687:0] peer: [5:7574435641268598673:0] socket: 114 qp: -1
2025-11-19T13:58:45.997530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435640646462708:2089] [node 5] ICS10 traffic start
2025-11-19T13:58:45.997536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:46.000007Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435639364661674:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.000031Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435639364661674:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.001744Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435639364661674:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:46.001945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435639364661668:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.001967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:46.001976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435639364661668:2097] [node 5] ICP22 created new session: [3:7574435639364661692:2097]
2025-11-19T13:58:46.001982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435639364661692:2097] [node 5] ICS09 handshake done sender: [3:7574435639364661674:2118] self: [3:7574435639364661672:0] peer: [5:7574435641268598672:0] socket: 83 qp: -1
2025-11-19T13:58:46.001987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435639364661692:2097] [node 5] ICS10 traffic start
2025-11-19T13:58:46.001996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:46.003877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435639750944450:2068] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.003893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:46.003899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435639750944450:2068] [node 5] ICP22 created new session: [6:7574435644045911820:2068]
2025-11-19T13:58:46.003905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435644045911820:2068] [node 5] ICS09 handshake done sender: [6:7574435639750944508:2118] self: [6:7574435639750944510:0] peer: [5:7574435641268598623:0] socket: 115 qp: -1
2025-11-19T13:58:46.003911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435644045911820:2068] [node 5] ICS10 traffic start
2025-11-19T13:58:46.003917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:46.004259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435642290364265:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44821
2025-11-19T13:58:46.004515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435642290364169:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:46.027447Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435640646462702:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.027602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435640646462610:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.027828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435640646462671:2089] [node 5] ICP35 event processing took too much time PendingConnection:278(30290375ns) {HandleHandshakeStatus:329(30289873ns) {DropIncomingHandshake:441(3346ns) DropOutgoingHandshake:456(90ns) SwitchToState:178(67ns) ProcessPendingSessionEvents:495(30232223ns) {ForwardSessionEventToSession:612(30217981ns) ForwardSessionEventToSession:612(3036ns) ForwardSessionEventToSession:612(910ns) ForwardSessionEventToSession:612(605ns) ForwardSessionEventToSession:612(619ns) ForwardSessionEventToSession:612(2651ns) ForwardSessionEventToSession:612(887ns) ForwardSessionEventToSession:612(536ns) ForwardSessionEventToSession:612(659ns) ForwardSessionEventToSession:612(897ns)}}}
2025-11-19T13:58:46.028166Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435642290364250:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.028181Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435642290364250:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.029091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435644941430007:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3710
2025-11-19T13:58:46.029194Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435642290364250:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:46.029382Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435640646462702:2121] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:46.029439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435640646462610:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.029458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:46.029465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435640646462610:2048] [node 1] ICP22 created new session: [4:7574435644941430008:2048]
2025-11-19T13:58:46.029471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435644941430008:2048] [node 1] ICS09 handshake done sender: [4:7574435640646462702:2121] self: [4:7574435644941430006:0] peer: [1:7574435642290364249:0] socket: 122 qp: -1
2025-11-19T13:58:46.029478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435644941430008:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:46.029484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:46.029686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435642290364246:2112] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.029697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:46.029704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435642290364246:2112] [node 4] ICP22 created new session: [1:7574435646585331563:2112]
2025-11-19T13:58:46.029708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435646585331563:2112] [node 4] ICS09 handshake done sender: [1:7574435642290364250:2116] self: [1:7574435642290364249:0] peer: [4:7574435644941430006:0] socket: 64 qp: -1
2025-11-19T13:58:46.029713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435646585331563:2112] [node 4] ICS10 traffic start
2025-11-19T13:58:46.029719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:46.004674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435641268598605:2074] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32270) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435641268598671:2121] held: no
2025-11-19T13:58:46.040156Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435641268598578:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62671) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435641268598670:2120] held: no
2025-11-19T13:58:46.040244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435641268598578:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(32175282ns) {HandleHandshakeStatus:395(32174398ns) {LogHandshakeFail:472(2450ns) {UpdateErrorStateLog:544(647ns)} DropOutgoingHandshake:456(750ns) ProcessPendingSessionEvents:495(22897ns) {DropSessionEvent:512(11856ns) DropSessionEvent:512(1195ns) DropSessionEvent:512(975ns) DropSessionEvent:512(987ns) DropSessionEvent:512(951ns) DropSessionEvent:512(872ns) DropSessionEvent:512(960ns) DropSessionEvent:512(935ns) DropSessionEvent:512(1170ns) DropSessionEvent:512(947ns)}}}
2025-11-19T13:58:46.040471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435639750944436:2059] [node 4] ICP35 event processing took too much time PendingConnection:278(43813094ns) {HandleHandshakeStatus:329(43812406ns) {DropIncomingHandshake:441(211ns) DropOutgoingHandshake:456(1664ns) SwitchToState:178(72ns) ProcessPendingSessionEvents:495(36210757ns) {ForwardSessionEventToSession:612(36197776ns) ForwardSessionEventToSession:612(1803ns) ForwardSessionEventToSession:612(1267ns) ForwardSessionEventToSession:612(591ns) ForwardSessionEventToSession:612(577ns) ForwardSessionEventToSession:612(1542ns) ForwardSessionEventToSession:612(975ns) ForwardSessionEventToSession:612(754ns) ForwardSessionEventToSession:612(821ns) ForwardSessionEventToSession:612(1309ns)}}}
2025-11-19T13:58:46.041047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435639364661668:2097] [node 5] ICP35 event processing took too much time PendingConnection:278(39077290ns) {HandleHandshakeStatus:329(39076430ns) {DropIncomingHandshake:441(7417ns) DropOutgoingHandshake:456(191ns) SwitchToState:178(373ns) ProcessPendingSessionEvents:495(38965740ns) {ForwardSessionEventToSession:612(38955840ns) ForwardSessionEventToSession:612(1960ns) ForwardSessionEventToSession:612(727ns) ForwardSessionEventToSession:612(1343ns) ForwardSessionEventToSession:612(537ns) ForwardSessionEventToSession:612(666ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(525ns) ForwardSessionEventToSession:612(647ns) ForwardSessionEventToSession:612(632ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.047800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435645563565970:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15686
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.052090Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435641268598671:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:46.052113Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435642290364248:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.052175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435642290364246:2112] [node 4] ICP18 handshake (actor [1:7574435642290364248:2115]) is too old
2025-11-19T13:58:46.065292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435645563565971:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15686
2025-11-19T13:58:46.084589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435641268598605:2074] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.084621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:46.084635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435641268598605:2074] [node 4] ICP22 created new session: [5:7574435645563565972:2074]
2025-11-19T13:58:46.084644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435645563565972:2074] [node 4] ICS09 handshake done sender: [5:7574435641268598671:2121] self: [5:7574435641268598673:0] peer: [4:7574435640646462687:0] socket: 125 qp: -1
2025-11-19T13:58:46.084655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435645563565972:2074] [node 4] ICS10 traffic start
2025-11-19T13:58:46.084667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:46.084857Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435641268598670:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:46.085835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435641268598578:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.085853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:46.085860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435641268598578:2050] [node 3] ICP22 created new session: [5:7574435645563565974:2050]
2025-11-19T13:58:46.085864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435645563565974:2050] [node 3] ICS09 handshake done sender: [5:7574435641268598670:2120] self: [5:7574435641268598672:0] peer: [3:7574435639364661672:0] socket: 90 qp: -1
2025-11-19T13:58:46.085870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435645563565974:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:46.085878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:46.088429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435640646462698:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.088586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435640646462665:2091] [node 6] ICP18 handshake (actor [4:7574435640646462698:2120]) is too old
2025-11-19T13:58:46.089945Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:46.090031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574435642618670414:2048] [node 1] ICS07 socket disconnect 66 reason# EndOfStream
2025-11-19T13:58:46.090036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435642618670414:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.090069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:46.090083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435642618670414:2048] [node 1] ICS15 start handshake
2025-11-19T13:58:46.091796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:46.091867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435639364661689:2048] [node 1] ICS01 socket: 100 reason# EndOfStream
2025-11-19T13:58:46.091875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435639364661589:2048] [node 1] ICP30 unregister session Session# [3:7574435639364661689:2048] VirtualId# [3:7574435639364661681:0]
2025-11-19T13:58:46.091882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435639364661689:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.091909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:46.091927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:46.092580Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435642618670316:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5551) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:58:46.092593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435642618670414:2048] [node 1] ICS15 start handshake
2025-11-19T13:58:46.093061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435642618670316:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5551) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:46.093072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435642618670414:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:46.093077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435642618670316:2048] [node 1] ICP30 unregister session Session# [2:7574435642618670414:2048] VirtualId# [2:7574435642618670401:0]
2025-11-19T13:58:46.093094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:46.093100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574435642618670316:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:46.093103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:46.107362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7574435642618670316:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:58:46.111872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:46.111981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435641268598662:2048] [node 1] ICS01 socket: 69 reason# EndOfStream
2025-11-19T13:58:46.111992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435641268598576:2048] [node 1] ICP30 unregister session Session# [5:7574435641268598662:2048] VirtualId# [5:7574435641268598659:0]
2025-11-19T13:58:46.112000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435641268598662:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.112046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:46.112073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:46.111872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:46.111976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:46.112034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435644941430008:2048] [node 1] ICS01 socket: 122 reason# EndOfStream
2025-11-19T13:58:46.112039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435640646462610:2048] [node 1] ICP30 unregister session Session# [4:7574435644941430008:2048] VirtualId# [4:7574435644941430006:0]
2025-11-19T13:58:46.112042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435644941430008:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.112077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:46.112108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:46.112293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435639750944504:2048] [node 1] ICS01 socket: 72 reason# EndOfStream
2025-11-19T13:58:46.112300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435639750944422:2048] [node 1] ICP30 unregister session Session# [6:7574435639750944504:2048] VirtualId# [6:7574435639750944501:0]
2025-11-19T13:58:46.112304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435639750944504:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.112333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:46.112351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:46.114960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:46.115028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435639364661682:2049] [node 2] ICS01 socket: 89 reason# EndOfStream
2025-11-19T13:58:46.115033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435639364661590:2049] [node 2] ICP30 unregister session Session# [3:7574435639364661682:2049] VirtualId# [3:7574435639364661671:0]
2025-11-19T13:58:46.115038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435639364661682:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.115063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:46.115088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:46.119412Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:46.119477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435641268598664:2049] [node 2] ICS01 socket: 76 reason# EndOfStream
2025-11-19T13:58:46.119483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435641268598577:2049] [node 2] ICP30 unregister session Session# [5:7574435641268598664:2049] VirtualId# [5:7574435641268598658:0]
2025-11-19T13:58:46.119487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435641268598664:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.119524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:46.119542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:46.119598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:46.119652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435640646462699:2049] [node 2] ICS01 socket: 70 reason# EndOfStream
2025-11-19T13:58:46.119656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435640646462611:2049] [node 2] ICP30 unregister session Session# [4:7574435640646462699:2049] VirtualId# [4:7574435640646462691:0]
2025-11-19T13:58:46.119658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435640646462699:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.119678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:46.119694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:46.119932Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:46.119999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435639750944520:2049] [node 2] ICS01 socket: 117 reason# EndOfStream
2025-11-19T13:58:46.120004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435639750944423:2049] [node 2] ICP30 unregister session Session# [6:7574435639750944520:2049] VirtualId# [6:7574435639750944514:0]
2025-11-19T13:58:46.120007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435639750944520:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.120045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:46.120066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:46.138624Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:46.138738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435639750944515:2050] [node 3] ICS01 socket: 103 reason# EndOfStream
2025-11-19T13:58:46.138749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435639750944424:2050] [node 3] ICP30 unregister session Session# [6:7574435639750944515:2050] VirtualId# [6:7574435639750944506:0]
2025-11-19T13:58:46.138757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435639750944515:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.138801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:46.138825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:46.141358Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:46.141447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435645563565974:2050] [node 3] ICS01 socket: 90 reason# EndOfStream
2025-11-19T13:58:46.141458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435641268598578:2050] [node 3] ICP30 unregister session Session# [5:7574435645563565974:2050] VirtualId# [5:7574435641268598672:0]
2025-11-19T13:58:46.141465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435645563565974:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.141510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:46.141532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:46.157548Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T13:58:46.157663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435645563565972:2074] [node 4] ICS01 socket: 125 reason# EndOfStream
2025-11-19T13:58:46.157674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435641268598605:2074] [node 4] ICP30 unregister session Session# [5:7574435645563565972:2074] VirtualId# [5:7574435641268598673:0]
2025-11-19T13:58:46.157684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435645563565972:2074] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.157728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:46.157751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:58:46.159290Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:46.159362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435639750944522:2059] [node 4] ICS07 socket disconnect 81 reason# EndOfStream
2025-11-19T13:58:46.159368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435639750944522:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.159407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:46.159426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435639750944522:2059] [node 4] ICS15 start handshake
2025-11-19T13:58:46.160876Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435639750944436:2059] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32270) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:46.160893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435639750944522:2059] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:46.160899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435639750944436:2059] [node 4] ICP30 unregister session Session# [6:7574435639750944522:2059] VirtualId# [6:7574435639750944518:0]
2025-11-19T13:58:46.160917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:58:46.160923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435639750944436:2059] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32270) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:46.160927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32270) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:46.171915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435639750944436:2059] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:46.359991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435646014095052:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:46.360008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435646014095053:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:46.388101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435644913419137:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:46.388119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435644913419138:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:46.448201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435643623608972:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:46.448223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435643623608973:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:46.501740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435646147306122:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:46.501755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435646147306123:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:46.501760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435646147306124:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:46.524227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435644343541610:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:46.524241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435644343541611:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:46.524246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435644343541612:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:46.570215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435643167903315:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:46.570271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435643167903316:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:46.570277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435643167903317:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:46.576543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435644343541623:2059] [node 6] ICP01 ready to work
2025-11-19T13:58:46.576184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435646147306144:2068] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.577485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435643167903326:2058] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48473
2025-11-19T13:58:46.577749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435644343541631:2064] [node 4] ICP01 ready to work
2025-11-19T13:58:46.579946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435643623609015:2089] [node 6] ICP01 ready to work
2025-11-19T13:58:46.580145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435646014095114:2107] [node 5] ICP01 ready to work
2025-11-19T13:58:46.580238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435646014095115:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:46.592200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435646147306191:2077] [node 6] ICP01 ready to work
2025-11-19T13:58:46.596538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435643167903371:2067] [node 4] ICP01 ready to work
2025-11-19T13:58:46.596632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435643167903372:2069] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.601591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435646014095129:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:46.601992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435646014095117:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38141
2025-11-19T13:58:46.602007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435646014095118:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38141
2025-11-19T13:58:46.602035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435646014095119:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38141
2025-11-19T13:58:46.602110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435646014095120:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38141
2025-11-19T13:58:46.603386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435644913419204:2098] [node 6] ICP01 ready to work
2025-11-19T13:58:46.603481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435644913419205:2103] [node 4] ICP01 ready to work
2025-11-19T13:58:46.603556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435644913419206:2106] [node 5] ICP01 ready to work
2025-11-19T13:58:46.608025Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435646014095119:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.615196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435646147306193:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3120
2025-11-19T13:58:46.615216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435646147306194:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3120
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.633444Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435643623609015:2089] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(53415842ns) {Configure:104(53414961ns) {ProcessConfigured:135(53070265ns) {StartInitialHandshake:157(53069761ns) {DropHandshakes:469(758ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(123ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:476(1859ns) {SwitchToState:178(994ns)}}}}}
2025-11-19T13:58:46.633763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435643623609041:2090] [node 4] ICP01 ready to work
2025-11-19T13:58:46.633842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435643623609042:2094] [node 5] ICP01 ready to work
2025-11-19T13:58:46.634969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435643623609038:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17756
2025-11-19T13:58:46.635052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435643623609039:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17756
2025-11-19T13:58:46.635090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435643623609040:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17756
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.639058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435644913419209:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3639
2025-11-19T13:58:46.639125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435644913419210:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3639
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.644040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435643167903394:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48473
2025-11-19T13:58:46.647977Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435644913419209:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.648282Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435643623609038:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.648383Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435643623609039:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.660457Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435643623609040:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.660480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435643623608973:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.660543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435643623608972:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.660970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435646147306202:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3120
2025-11-19T13:58:46.661008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435646147306203:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3120
2025-11-19T13:58:46.661160Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435644343541612:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19260997ns) {Configure:104(19260004ns) {ProcessConfigured:135(18795000ns) {StartInitialHandshake:157(18794213ns) {DropHandshakes:469(1110ns) {DropIncomingHandshake:441(207ns) DropOutgoingHandshake:456(124ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:476(975ns) {SwitchToState:178(574ns)}}}}}
2025-11-19T13:58:46.660063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435646014095136:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38141
2025-11-19T13:58:46.660166Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435646014095117:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.661590Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435646014095136:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.663877Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435643167903394:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.663939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435643167903317:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.664263Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435644913419189:2097] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.664281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435644913419189:2097] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.664682Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435644913419210:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.665226Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435643623609037:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.665241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435643623609037:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.665894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435643623609057:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17756
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.665065Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435646014095126:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.665083Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435646014095126:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.666098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435644343541679:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3320
2025-11-19T13:58:46.666169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435644343541685:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3320
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.668787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435643167903396:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48473
2025-11-19T13:58:46.669007Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435644913419189:2097] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:46.669528Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435643623609037:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:46.669758Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435643167903394:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:46.669831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435643167903317:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.669844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:46.669850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435643167903317:2050] [node 3] ICP22 created new session: [6:7574435643167903397:2050]
2025-11-19T13:58:46.669855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435643167903397:2050] [node 3] ICS09 handshake done sender: [6:7574435643167903394:2116] self: [6:7574435643167903395:0] peer: [3:7574435643623609016:0] socket: 100 qp: -1
2025-11-19T13:58:46.669862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435643167903397:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:46.669868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:46.670742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435644913419138:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.670760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:46.670765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435644913419138:2049] [node 3] ICP22 created new session: [2:7574435644913419221:2049]
2025-11-19T13:58:46.670771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435644913419221:2049] [node 3] ICS09 handshake done sender: [2:7574435644913419189:2097] self: [2:7574435644913419188:0] peer: [3:7574435643623609054:0] socket: 70 qp: -1
2025-11-19T13:58:46.670778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435644913419221:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:46.670784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:46.671440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435643623609015:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.671466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:46.671472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435643623609015:2089] [node 6] ICP22 created new session: [3:7574435643623609060:2089]
2025-11-19T13:58:46.671477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435643623609060:2089] [node 6] ICS09 handshake done sender: [3:7574435643623609037:2110] self: [3:7574435643623609016:0] peer: [6:7574435643167903395:0] socket: 94 qp: -1
2025-11-19T13:58:46.671484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435643623609060:2089] [node 6] ICS10 traffic start
2025-11-19T13:58:46.671492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:46.668404Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435646014095118:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.668907Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435646014095126:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:46.669166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435646014095053:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.669184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:46.669191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435646014095053:2049] [node 3] ICP22 created new session: [1:7574435646014095141:2049]
2025-11-19T13:58:46.669199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435646014095141:2049] [node 3] ICS09 handshake done sender: [1:7574435646014095126:2116] self: [1:7574435646014095125:0] peer: [3:7574435643623609056:0] socket: 85 qp: -1
2025-11-19T13:58:46.669205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435646014095141:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:46.669212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:46.671966Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435646147306202:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.672030Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435646147306203:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.672142Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435646147306194:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.672170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435646147306124:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.672180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435646147306123:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.672247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435646147306122:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.673578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435644913419224:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3639
2025-11-19T13:58:46.674118Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435646014095122:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.674134Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435646014095122:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.675983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435646147306207:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3120
2025-11-19T13:58:46.676162Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435644913419212:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.676172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435644913419212:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.676856Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435646014095122:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:46.676938Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435644913419212:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:46.677115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435646014095115:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.677150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:46.677159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435646014095115:2108] [node 4] ICP22 created new session: [1:7574435646014095145:2108]
2025-11-19T13:58:46.677166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435646014095145:2108] [node 4] ICS09 handshake done sender: [1:7574435646014095122:2114] self: [1:7574435646014095121:0] peer: [4:7574435646147306206:0] socket: 83 qp: -1
2025-11-19T13:58:46.677172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435646014095145:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:46.677182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:46.677655Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435644913419224:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.677675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435644913419205:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.677693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:46.677700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435644913419205:2103] [node 4] ICP22 created new session: [2:7574435644913419226:2103]
2025-11-19T13:58:46.677706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435644913419226:2103] [node 4] ICS09 handshake done sender: [2:7574435644913419212:2114] self: [2:7574435644913419211:0] peer: [4:7574435646147306205:0] socket: 99 qp: -1
2025-11-19T13:58:46.677712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435644913419226:2103] [node 4] ICS10 traffic start
2025-11-19T13:58:46.677720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:46.678598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435644913419137:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.678967Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435646014095128:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.678975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435646014095128:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.682004Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435646147306122:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64916) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435646147306194:2113] held: no
2025-11-19T13:58:46.682060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435646147306123:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14094) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435646147306203:2118] held: no
2025-11-19T13:58:46.682095Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435646147306194:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:46.685769Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435644343541685:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.685809Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435644343541679:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.685944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435644343541610:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.685961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435644343541631:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.685984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435644343541690:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3320
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.701490Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435646014095124:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.701508Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435646014095124:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.702898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435643167903399:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48473
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.703525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435646147306208:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3120
2025-11-19T13:58:46.702960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435643167903401:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48473
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.703427Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435643623609038:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:46.703713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435644343541693:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3320
2025-11-19T13:58:46.703799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435646147306122:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.703817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:46.703823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435646147306122:2048] [node 1] ICP22 created new session: [4:7574435646147306209:2048]
2025-11-19T13:58:46.703842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435646147306209:2048] [node 1] ICS09 handshake done sender: [4:7574435646147306194:2113] self: [4:7574435646147306206:0] peer: [1:7574435646014095121:0] socket: 87 qp: -1
2025-11-19T13:58:46.703850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435646147306209:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:46.703859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:46.704058Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435646147306150:2072] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.704064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435646147306150:2072] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.704956Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435646147306203:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:46.704453Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435643167903326:2058] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.705898Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435646014095128:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.706084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435644913419230:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3639
2025-11-19T13:58:46.706168Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435643623609046:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.706183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435643623609046:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.706542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435643623609059:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17756
2025-11-19T13:58:46.706773Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435644913419224:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:46.706853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435644913419137:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.706870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:46.706876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435644913419137:2048] [node 1] ICP22 created new session: [2:7574435644913419231:2048]
2025-11-19T13:58:46.706882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435644913419231:2048] [node 1] ICS09 handshake done sender: [2:7574435644913419224:2117] self: [2:7574435644913419229:0] peer: [1:7574435646014095127:0] socket: 109 qp: -1
2025-11-19T13:58:46.706888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435644913419231:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:46.706896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:46.708063Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435646014095124:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.712338Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435646147306193:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.712361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435646147306123:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.712377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:46.712389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435646147306123:2049] [node 2] ICP22 created new session: [4:7574435646147306215:2049]
2025-11-19T13:58:46.712395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435646147306215:2049] [node 2] ICS09 handshake done sender: [4:7574435646147306203:2118] self: [4:7574435646147306205:0] peer: [2:7574435644913419211:0] socket: 101 qp: -1
2025-11-19T13:58:46.712403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435646147306215:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:46.712412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:46.712588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435646147306211:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3120
2025-11-19T13:58:46.713064Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435646147306150:2072] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:46.713237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435646147306144:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.713254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:46.713263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435646147306144:2068] [node 5] ICP22 created new session: [4:7574435646147306218:2068]
2025-11-19T13:58:46.713268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435646147306218:2068] [node 5] ICS09 handshake done sender: [4:7574435646147306150:2072] self: [4:7574435646147306149:0] peer: [5:7574435644343541692:0] socket: 69 qp: -1
2025-11-19T13:58:46.713274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435646147306218:2068] [node 5] ICS10 traffic start
2025-11-19T13:58:46.713282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:46.715595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435643623608973:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.715616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:46.715623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435643623608973:2049] [node 2] ICP22 created new session: [3:7574435643623609064:2049]
2025-11-19T13:58:46.715630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435643623609064:2049] [node 2] ICS09 handshake done sender: [3:7574435643623609038:2111] self: [3:7574435643623609054:0] peer: [2:7574435644913419188:0] socket: 91 qp: -1
2025-11-19T13:58:46.715637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435643623609064:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:46.715646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:46.716052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435646014095120:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.716119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435646014095052:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.716135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:46.716140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435646014095052:2048] [node 2] ICP22 created new session: [1:7574435646014095150:2048]
2025-11-19T13:58:46.716146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435646014095150:2048] [node 2] ICS09 handshake done sender: [1:7574435646014095128:2117] self: [1:7574435646014095127:0] peer: [2:7574435644913419229:0] socket: 98 qp: -1
2025-11-19T13:58:46.716150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435646014095150:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:46.716167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:46.716272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435646014095114:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.716278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:46.716282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435646014095114:2107] [node 5] ICP22 created new session: [1:7574435646014095152:2107]
2025-11-19T13:58:46.716285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435646014095152:2107] [node 5] ICS09 handshake done sender: [1:7574435646014095124:2115] self: [1:7574435646014095123:0] peer: [5:7574435644343541691:0] socket: 84 qp: -1
2025-11-19T13:58:46.716289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435646014095152:2107] [node 5] ICS10 traffic start
2025-11-19T13:58:46.716294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.717241Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435643167903399:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.717275Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435643167903401:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.717294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435643167903372:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.717373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435643167903402:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48473
2025-11-19T13:58:46.717381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435643167903316:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.717389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435643167903315:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.717560Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435644913419214:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.717569Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435644913419214:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.718924Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435643623608972:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64916) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435643623609040:2113] held: no
2025-11-19T13:58:46.719276Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435643623609040:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:46.719999Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435644343541690:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.720524Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435644343541625:2060] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.720533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435644343541625:2060] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.721015Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435644343541685:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.721515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435646147306220:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3120
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.721678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435644343541694:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3320
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.721841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435644343541695:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3320
2025-11-19T13:58:46.721861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435644343541611:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.722943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435644343541610:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.722962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:46.722968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435644343541610:2048] [node 1] ICP22 created new session: [5:7574435644343541698:2048]
2025-11-19T13:58:46.722975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435644343541698:2048] [node 1] ICS09 handshake done sender: [5:7574435644343541685:2114] self: [5:7574435644343541691:0] peer: [1:7574435646014095123:0] socket: 89 qp: -1
2025-11-19T13:58:46.722981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435644343541698:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:46.722989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:46.723141Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435644343541631:2064] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12300) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435644343541679:2108] held: no
2025-11-19T13:58:46.723240Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435644343541625:2060] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:46.724369Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435646014095131:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.724381Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435646014095131:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.724886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435646014095114:2107] [node 5] ICP18 handshake (actor [1:7574435646014095120:2113]) is too old
2025-11-19T13:58:46.725261Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435644343541679:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:46.725373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435644343541623:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.725387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:46.725392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435644343541623:2059] [node 6] ICP22 created new session: [5:7574435644343541700:2059]
2025-11-19T13:58:46.725398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435644343541700:2059] [node 6] ICS09 handshake done sender: [5:7574435644343541625:2060] self: [5:7574435644343541624:0] peer: [6:7574435643167903403:0] socket: 68 qp: -1
2025-11-19T13:58:46.725404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435644343541700:2059] [node 6] ICS10 traffic start
2025-11-19T13:58:46.725410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:46.725559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435644343541631:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.725564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:46.725568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435644343541631:2064] [node 4] ICP22 created new session: [5:7574435644343541702:2064]
2025-11-19T13:58:46.725571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435644343541702:2064] [node 4] ICS09 handshake done sender: [5:7574435644343541679:2108] self: [5:7574435644343541692:0] peer: [4:7574435646147306149:0] socket: 75 qp: -1
2025-11-19T13:58:46.725575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435644343541702:2064] [node 4] ICS10 traffic start
2025-11-19T13:58:46.725580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:46.726502Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435646014095131:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:46.726623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435646014095129:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.726645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:46.726651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435646014095129:2109] [node 6] ICP22 created new session: [1:7574435646014095155:2109]
2025-11-19T13:58:46.726656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435646014095155:2109] [node 6] ICS09 handshake done sender: [1:7574435646014095131:2118] self: [1:7574435646014095130:0] peer: [6:7574435643167903405:0] socket: 80 qp: -1
2025-11-19T13:58:46.726660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435646014095155:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:46.726667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:46.728028Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435644913419214:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:46.729016Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435646147306211:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.729390Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435644913419216:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.729404Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435644913419216:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.730151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435644913419204:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.730164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:46.730171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435644913419204:2098] [node 6] ICP22 created new session: [2:7574435644913419235:2098]
2025-11-19T13:58:46.730185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435644913419235:2098] [node 6] ICS09 handshake done sender: [2:7574435644913419214:2115] self: [2:7574435644913419213:0] peer: [6:7574435643167903404:0] socket: 107 qp: -1
2025-11-19T13:58:46.730191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435644913419235:2098] [node 6] ICS10 traffic start
2025-11-19T13:58:46.730200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:46.730710Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435643623609046:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:46.730941Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435646147306202:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:46.731023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435646147306124:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.731036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:46.731041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435646147306124:2050] [node 3] ICP22 created new session: [4:7574435646147306222:2050]
2025-11-19T13:58:46.731046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435646147306222:2050] [node 3] ICS09 handshake done sender: [4:7574435646147306202:2117] self: [4:7574435646147306204:0] peer: [3:7574435643623609045:0] socket: 97 qp: -1
2025-11-19T13:58:46.731051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435646147306222:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:46.731058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.732018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435643623608972:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.732040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:46.732046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435643623608972:2048] [node 1] ICP22 created new session: [3:7574435643623609068:2048]
2025-11-19T13:58:46.732052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435643623609068:2048] [node 1] ICS09 handshake done sender: [3:7574435643623609040:2113] self: [3:7574435643623609056:0] peer: [1:7574435646014095125:0] socket: 93 qp: -1
2025-11-19T13:58:46.732057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435643623609068:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:46.732064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:46.732172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435643623609041:2090] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.732177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:46.732181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435643623609041:2090] [node 4] ICP22 created new session: [3:7574435643623609070:2090]
2025-11-19T13:58:46.732185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435643623609070:2090] [node 4] ICS09 handshake done sender: [3:7574435643623609046:2115] self: [3:7574435643623609045:0] peer: [4:7574435646147306204:0] socket: 96 qp: -1
2025-11-19T13:58:46.732189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435643623609070:2090] [node 4] ICS10 traffic start
2025-11-19T13:58:46.732193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:46.732329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435643623609066:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17756
2025-11-19T13:58:46.733441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435643623609067:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17756
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.734689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435644913419234:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3639
2025-11-19T13:58:46.734983Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435644913419216:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:46.735086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435644913419206:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.735100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:46.735113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435644913419206:2106] [node 5] ICP22 created new session: [2:7574435644913419238:2106]
2025-11-19T13:58:46.735117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435644913419238:2106] [node 5] ICS09 handshake done sender: [2:7574435644913419216:2116] self: [2:7574435644913419215:0] peer: [5:7574435644343541697:0] socket: 90 qp: -1
2025-11-19T13:58:46.735122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435644913419238:2106] [node 5] ICS10 traffic start
2025-11-19T13:58:46.735131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.737142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435644343541706:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3320
2025-11-19T13:58:46.738579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435643623609041:2090] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:46.737775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435643167903315:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64916) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435643167903401:2119] held: no
2025-11-19T13:58:46.737840Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435643167903402:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.744264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435643167903406:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48473
2025-11-19T13:58:46.744474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435643167903407:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48473
2025-11-19T13:58:46.744608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435643167903408:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48473
2025-11-19T13:58:46.744762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435643167903316:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14094) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435643167903399:2118] held: no
2025-11-19T13:58:46.744819Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435643167903399:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:46.750328Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435646147306200:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.750340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435646147306200:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.751138Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435646147306200:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:46.751258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435646147306191:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.751274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:46.751280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435646147306191:2077] [node 6] ICP22 created new session: [4:7574435646147306225:2077]
2025-11-19T13:58:46.751286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435646147306225:2077] [node 6] ICS09 handshake done sender: [4:7574435646147306200:2116] self: [4:7574435646147306199:0] peer: [6:7574435643167903410:0] socket: 120 qp: -1
2025-11-19T13:58:46.751290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435646147306225:2077] [node 6] ICS10 traffic start
2025-11-19T13:58:46.751298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:46.751917Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435644343541695:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.752079Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435644343541690:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:46.752238Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435644913419234:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.752265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435644913419206:2106] [node 5] ICP18 handshake (actor [2:7574435644913419234:2119]) is too old
2025-11-19T13:58:46.752445Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435643623609067:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.749053Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435643167903326:2058] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:46.749158Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435643167903401:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:46.749245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435643167903371:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.749259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435643167903316:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.749271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:46.749277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435643167903316:2049] [node 2] ICP22 created new session: [6:7574435643167903411:2049]
2025-11-19T13:58:46.749283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435643167903411:2049] [node 2] ICS09 handshake done sender: [6:7574435643167903399:2118] self: [6:7574435643167903404:0] peer: [2:7574435644913419213:0] socket: 78 qp: -1
2025-11-19T13:58:46.749291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435643167903411:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:46.749300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:46.749526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435643167903372:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.749533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:46.749537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435643167903372:2069] [node 5] ICP22 created new session: [6:7574435643167903413:2069]
2025-11-19T13:58:46.749541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435643167903413:2069] [node 5] ICS09 handshake done sender: [6:7574435643167903326:2058] self: [6:7574435643167903403:0] peer: [5:7574435644343541624:0] socket: 71 qp: -1
2025-11-19T13:58:46.749545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435643167903413:2069] [node 5] ICS10 traffic start
2025-11-19T13:58:46.749552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:46.749666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435643167903315:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.749671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:46.749674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435643167903315:2048] [node 1] ICP22 created new session: [6:7574435643167903415:2048]
2025-11-19T13:58:46.749678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435643167903415:2048] [node 1] ICS09 handshake done sender: [6:7574435643167903401:2119] self: [6:7574435643167903405:0] peer: [1:7574435646014095130:0] socket: 114 qp: -1
2025-11-19T13:58:46.749682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435643167903415:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:46.749695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.758032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435644343541611:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.758063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:46.758072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435644343541611:2049] [node 2] ICP22 created new session: [5:7574435644343541708:2049]
2025-11-19T13:58:46.758079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435644343541708:2049] [node 2] ICS09 handshake done sender: [5:7574435644343541690:2117] self: [5:7574435644343541697:0] peer: [2:7574435644913419215:0] socket: 66 qp: -1
2025-11-19T13:58:46.758085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435644343541708:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:46.758093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:46.758172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435644343541611:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:46.758256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435644343541612:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:46.758284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435644343541707:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3320
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.758706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435643167903371:2067] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12300) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435643167903402:2120] held: no
2025-11-19T13:58:46.758768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435643167903417:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48473
2025-11-19T13:58:46.758910Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435643167903402:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:46.758960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435643167903371:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.758968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:46.758980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435643167903371:2067] [node 4] ICP22 created new session: [6:7574435643167903418:2067]
2025-11-19T13:58:46.758985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435643167903418:2067] [node 4] ICS09 handshake done sender: [6:7574435643167903402:2120] self: [6:7574435643167903410:0] peer: [4:7574435646147306199:0] socket: 67 qp: -1
2025-11-19T13:58:46.758990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435643167903418:2067] [node 4] ICS10 traffic start
2025-11-19T13:58:46.758997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:46.759641Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435643623609048:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:46.759651Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435643623609048:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:46.760604Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435643623609048:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:46.760703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435643623609042:2094] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:46.760720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:46.760725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435643623609042:2094] [node 5] ICP22 created new session: [3:7574435643623609074:2094]
2025-11-19T13:58:46.760731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435643623609074:2094] [node 5] ICS09 handshake done sender: [3:7574435643623609048:2116] self: [3:7574435643623609047:0] peer: [5:7574435644343541710:0] socket: 124 qp: -1
2025-11-19T13:58:46.760735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435643623609074:2094] [node 5] ICS10 traffic start
2025-11-19T13:58:46.760744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:46.761897Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435644343541612:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23621) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435644343541695:2120] held: no
2025-11-19T13:58:46.761960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435644343541711:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3320
2025-11-19T13:58:46.762144Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435644343541695:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:46.762218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435644343541612:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:46.762229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:46.762234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435644343541612:2050] [node 3] ICP22 created new session: [5:7574435644343541712:2050]
2025-11-19T13:58:46.762238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435644343541712:2050] [node 3] ICS09 handshake done sender: [5:7574435644343541695:2120] self: [5:7574435644343541710:0] peer: [3:7574435643623609047:0] socket: 126 qp: -1
2025-11-19T13:58:46.762243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435644343541712:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:46.762249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:46.773372Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:46.773470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435644913419231:2048] [node 1] ICS01 socket: 109 reason# EndOfStream
2025-11-19T13:58:46.773481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435644913419137:2048] [node 1] ICP30 unregister session Session# [2:7574435644913419231:2048] VirtualId# [2:7574435644913419229:0]
2025-11-19T13:58:46.773490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435644913419231:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.773563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:46.773589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:46.780603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:46.780703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435643623609068:2048] [node 1] ICS01 socket: 93 reason# EndOfStream
2025-11-19T13:58:46.780711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435643623608972:2048] [node 1] ICP30 unregister session Session# [3:7574435643623609068:2048] VirtualId# [3:7574435643623609056:0]
2025-11-19T13:58:46.780718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435643623609068:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.780752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:46.780772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:46.787981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:46.788081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435646147306209:2048] [node 1] ICS01 socket: 87 reason# EndOfStream
2025-11-19T13:58:46.788091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435646147306122:2048] [node 1] ICP30 unregister session Session# [4:7574435646147306209:2048] VirtualId# [4:7574435646147306206:0]
2025-11-19T13:58:46.788099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435646147306209:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.788156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:46.788179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:46.788685Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:46.788774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435644343541698:2048] [node 1] ICS01 socket: 89 reason# EndOfStream
2025-11-19T13:58:46.788786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435644343541610:2048] [node 1] ICP30 unregister session Session# [5:7574435644343541698:2048] VirtualId# [5:7574435644343541691:0]
2025-11-19T13:58:46.788792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435644343541698:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.788844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:46.788876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:46.789712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:46.789799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435643623609064:2049] [node 2] ICS01 socket: 91 reason# EndOfStream
2025-11-19T13:58:46.789806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435643623608973:2049] [node 2] ICP30 unregister session Session# [3:7574435643623609064:2049] VirtualId# [3:7574435643623609054:0]
2025-11-19T13:58:46.789811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435643623609064:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.789843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:46.789860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:46.788216Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:46.788278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435643167903415:2048] [node 1] ICS01 socket: 114 reason# EndOfStream
2025-11-19T13:58:46.788283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435643167903315:2048] [node 1] ICP30 unregister session Session# [6:7574435643167903415:2048] VirtualId# [6:7574435643167903405:0]
2025-11-19T13:58:46.788287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435643167903415:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.788319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:46.788342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:46.803430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:46.803533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435643167903411:2049] [node 2] ICS01 socket: 78 reason# EndOfStream
2025-11-19T13:58:46.803547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435643167903316:2049] [node 2] ICP30 unregister session Session# [6:7574435643167903411:2049] VirtualId# [6:7574435643167903404:0]
2025-11-19T13:58:46.803554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435643167903411:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.803672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:46.803740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435646147306215:2049] [node 2] ICS01 socket: 101 reason# EndOfStream
2025-11-19T13:58:46.803758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435646147306123:2049] [node 2] ICP30 unregister session Session# [4:7574435646147306215:2049] VirtualId# [4:7574435646147306205:0]
2025-11-19T13:58:46.803763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435646147306215:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.803794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:46.803813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:46.804493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:46.803611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:46.803637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:46.804585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435644343541708:2049] [node 2] ICS01 socket: 66 reason# EndOfStream
2025-11-19T13:58:46.804597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435644343541611:2049] [node 2] ICP30 unregister session Session# [5:7574435644343541708:2049] VirtualId# [5:7574435644343541697:0]
2025-11-19T13:58:46.804604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435644343541708:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.804658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:46.804686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:46.811866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:46.811973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435643167903397:2050] [node 3] ICS01 socket: 100 reason# EndOfStream
2025-11-19T13:58:46.811991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435643167903317:2050] [node 3] ICP30 unregister session Session# [6:7574435643167903397:2050] VirtualId# [6:7574435643167903395:0]
2025-11-19T13:58:46.811999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435643167903397:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.812040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:46.812062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:46.816001Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:46.816105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435644343541712:2050] [node 3] ICS01 socket: 126 reason# EndOfStream
2025-11-19T13:58:46.816114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435644343541612:2050] [node 3] ICP30 unregister session Session# [5:7574435644343541712:2050] VirtualId# [5:7574435644343541710:0]
2025-11-19T13:58:46.816121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435644343541712:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.816178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:46.816205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:46.861811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:46.861919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435643167903418:2067] [node 4] ICS01 socket: 67 reason# EndOfStream
2025-11-19T13:58:46.861929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435643167903371:2067] [node 4] ICP30 unregister session Session# [6:7574435643167903418:2067] VirtualId# [6:7574435643167903410:0]
2025-11-19T13:58:46.861936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435643167903418:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:46.862010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:46.862055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:46.943899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435643534081111:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:46.943914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435643534081112:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:47.000212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435645916149941:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:47.000228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435645916149942:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:47.044455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435650557448259:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:47.044469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435650557448260:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:47.108169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435649855663364:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:47.108192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435649855663365:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:47.108197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435649855663366:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:47.156443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435649260577013:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:47.156464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435649260577014:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:47.156469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435649260577015:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:47.223921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435650657500475:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:47.223934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435650657500476:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:47.223940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435650657500477:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:47.247900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435647829048455:2094] [node 5] ICP01 ready to work
2025-11-19T13:58:47.247638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435650557448295:2082] [node 5] ICP01 ready to work
2025-11-19T13:58:47.248718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435649855663406:2083] [node 6] ICP01 ready to work
2025-11-19T13:58:47.248771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435650657500493:2061] [node 4] ICP01 ready to work
2025-11-19T13:58:47.249582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435650557448304:2087] [node 6] ICP01 ready to work
2025-11-19T13:58:47.256639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435650211117288:2091] [node 4] ICP01 ready to work
2025-11-19T13:58:47.256748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435650211117289:2093] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.262573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435650657500540:2067] [node 5] ICP01 ready to work
2025-11-19T13:58:47.261403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435650557448321:2091] [node 4] ICP01 ready to work
2025-11-19T13:58:47.262226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435650557448315:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63500
2025-11-19T13:58:47.262242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435650557448316:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63500
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.265021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435649855663430:2091] [node 5] ICP01 ready to work
2025-11-19T13:58:47.268189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435650657500546:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57965
2025-11-19T13:58:47.268211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435650657500547:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57965
2025-11-19T13:58:47.272325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435647829048461:2097] [node 6] ICP01 ready to work
2025-11-19T13:58:47.272520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435647829048469:2098] [node 4] ICP01 ready to work
2025-11-19T13:58:47.270504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435650557448259:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(20855732ns) {Configure:104(20854700ns) {ProcessConfigured:135(20581239ns) {StartInitialHandshake:157(20580521ns) {DropHandshakes:469(429ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:476(1302ns) {SwitchToState:178(566ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.276658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435645916149941:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(19837888ns) {Configure:104(19836534ns) {ProcessConfigured:135(19391424ns) {StartInitialHandshake:157(19390326ns) {DropHandshakes:469(1098ns) {DropIncomingHandshake:441(242ns) DropOutgoingHandshake:456(159ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(1779ns) {SwitchToState:178(863ns)}}}}}
2025-11-19T13:58:47.277690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435650211117309:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55869
2025-11-19T13:58:47.278003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435647829048455:2094] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(30035743ns) {Configure:104(30035409ns) {ProcessConfigured:135(29580764ns) {StartInitialHandshake:157(29580457ns) {DropHandshakes:469(463ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:476(680ns) {SwitchToState:178(216ns)}}}}}
2025-11-19T13:58:47.278892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435647829048476:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50167
2025-11-19T13:58:47.278903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435647829048477:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50167
2025-11-19T13:58:47.280443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435650211117290:2092] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.282789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435649855663365:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(34528757ns) {Configure:104(34527616ns) {ProcessConfigured:135(34256384ns) {StartInitialHandshake:157(34255796ns) {DropHandshakes:469(223ns) {DropIncomingHandshake:441(41ns) DropOutgoingHandshake:456(18ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:476(1994ns) {SwitchToState:178(1251ns)}}}}}
2025-11-19T13:58:47.282842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435649855663428:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52240
2025-11-19T13:58:47.282863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435649855663429:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52240
2025-11-19T13:58:47.283563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435649855663443:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52240
2025-11-19T13:58:47.283573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435649855663444:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52240
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.288316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435650557448340:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63500
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.287995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435649260577073:2080] [node 6] ICP01 ready to work
2025-11-19T13:58:47.288185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435649260577080:2087] [node 4] ICP01 ready to work
2025-11-19T13:58:47.289371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435649260577083:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58435
2025-11-19T13:58:47.289383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435649260577084:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58435
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.291988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435650657500555:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57965
2025-11-19T13:58:47.292010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435650657500556:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57965
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.293328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435650211117315:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55869
2025-11-19T13:58:47.293355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435650211117316:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55869
2025-11-19T13:58:47.296538Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435650657500546:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.296582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435650657500493:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.296722Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435649855663408:2087] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.296734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435649855663408:2087] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:47.298749Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435649855663408:2087] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:47.298934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435649855663406:2083] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.298949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:47.298956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435649855663406:2083] [node 6] ICP22 created new session: [4:7574435649855663447:2083]
2025-11-19T13:58:47.298962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435649855663447:2083] [node 6] ICS09 handshake done sender: [4:7574435649855663408:2087] self: [4:7574435649855663407:0] peer: [6:7574435650657500557:0] socket: 75 qp: -1
2025-11-19T13:58:47.298969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435649855663447:2083] [node 6] ICS10 traffic start
2025-11-19T13:58:47.298979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.304439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435650657500558:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57965
2025-11-19T13:58:47.306316Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435650211117315:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.310780Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435647829048477:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.312258Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435650557448316:2100] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.314351Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435649260577084:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.316708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435649260577013:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.312692Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435643534081111:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(40126529ns) {Configure:104(40125497ns) {ProcessConfigured:135(39626089ns) {StartInitialHandshake:157(39625327ns) {DropHandshakes:469(1127ns) {DropIncomingHandshake:441(135ns) DropOutgoingHandshake:456(224ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:476(951ns) {SwitchToState:178(764ns)}}}}}
2025-11-19T13:58:47.317208Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435647829048475:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.317214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435647829048475:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.317470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435649855663406:2083] [node 6] ICP35 event processing took too much time PendingConnection:278(18498911ns) {HandleHandshakeStatus:329(18497911ns) {DropIncomingHandshake:441(122ns) DropOutgoingHandshake:456(243ns) SwitchToState:178(158ns) ProcessPendingSessionEvents:495(9335204ns) {ForwardSessionEventToSession:612(9318878ns) ForwardSessionEventToSession:612(3387ns) ForwardSessionEventToSession:612(952ns) ForwardSessionEventToSession:612(916ns) ForwardSessionEventToSession:612(697ns) ForwardSessionEventToSession:612(1158ns) ForwardSessionEventToSession:612(668ns) ForwardSessionEventToSession:612(875ns) ForwardSessionEventToSession:612(621ns) ForwardSessionEventToSession:612(949ns)}}}
2025-11-19T13:58:47.318163Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435650211117316:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.318526Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435650557448340:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.318615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435650557448260:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.317860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435649260577094:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58435
2025-11-19T13:58:47.321157Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435649260577083:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.318338Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435649855663443:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.318367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435649855663365:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.318709Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435649855663429:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.318734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435649855663366:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.318955Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435649855663444:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.318969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435649855663364:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.322178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435649260577015:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.319090Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435650211117306:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.319100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435650211117306:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:47.319609Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435650211117304:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.319611Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435650211117304:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:47.321904Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435650211117306:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:47.322834Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435650211117304:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:47.325381Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435647829048481:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.325394Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435647829048481:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:47.325955Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435647829048475:2110] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:47.323875Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435650557448324:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.323896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435650557448324:2104] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:47.324370Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435650557448315:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.324400Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435650557448303:2089] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.324403Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435650557448303:2089] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:47.326080Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435650557448324:2104] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.327404Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435649260577094:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.327479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435649260577096:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58435
2025-11-19T13:58:47.327319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435647829048455:2094] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.327335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:47.327342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435647829048455:2094] [node 5] ICP22 created new session: [1:7574435647829048491:2094]
2025-11-19T13:58:47.327349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435647829048491:2094] [node 5] ICS09 handshake done sender: [1:7574435647829048475:2110] self: [1:7574435647829048458:0] peer: [5:7574435649260577093:0] socket: 93 qp: -1
2025-11-19T13:58:47.327356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435647829048491:2094] [node 5] ICS10 traffic start
2025-11-19T13:58:47.327365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:47.327852Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435647829048481:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:47.327626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435650211117288:2091] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.327652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:47.327659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435650211117288:2091] [node 4] ICP22 created new session: [2:7574435650211117323:2091]
2025-11-19T13:58:47.327665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435650211117323:2091] [node 4] ICS09 handshake done sender: [2:7574435650211117306:2110] self: [2:7574435650211117305:0] peer: [4:7574435649855663450:0] socket: 90 qp: -1
2025-11-19T13:58:47.327671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435650211117323:2091] [node 4] ICS10 traffic start
2025-11-19T13:58:47.327680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:47.327848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435645916149942:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.327854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:47.327860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435645916149942:2049] [node 3] ICP22 created new session: [2:7574435650211117325:2049]
2025-11-19T13:58:47.327864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435650211117325:2049] [node 3] ICS09 handshake done sender: [2:7574435650211117304:2109] self: [2:7574435650211117303:0] peer: [3:7574435650557448342:0] socket: 89 qp: -1
2025-11-19T13:58:47.327868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435650211117325:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:47.327872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.328270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435649260577097:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58435
2025-11-19T13:58:47.327954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435647829048469:2098] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.327966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:47.327971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435647829048469:2098] [node 4] ICP22 created new session: [1:7574435647829048494:2098]
2025-11-19T13:58:47.327976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435647829048494:2098] [node 4] ICS09 handshake done sender: [1:7574435647829048481:2114] self: [1:7574435647829048480:0] peer: [4:7574435649855663452:0] socket: 95 qp: -1
2025-11-19T13:58:47.327982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435647829048494:2098] [node 4] ICS10 traffic start
2025-11-19T13:58:47.327988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:47.329299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435649855663365:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435649855663443:2117] held: no
2025-11-19T13:58:47.329365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435649260577080:2087] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.329477Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435649260577084:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:47.329559Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435649855663440:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.329567Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435649855663440:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:47.327121Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435650557448303:2089] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:47.331592Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435649260577083:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.331679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435649260577013:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.331696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:47.331702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435649260577013:2048] [node 1] ICP22 created new session: [5:7574435649260577100:2048]
2025-11-19T13:58:47.331708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435649260577100:2048] [node 1] ICS09 handshake done sender: [5:7574435649260577084:2113] self: [5:7574435649260577093:0] peer: [1:7574435647829048458:0] socket: 102 qp: -1
2025-11-19T13:58:47.331713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435649260577100:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:47.331719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:47.331995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435649260577015:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.332004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:47.332008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435649260577015:2050] [node 3] ICP22 created new session: [5:7574435649260577103:2050]
2025-11-19T13:58:47.332013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435649260577103:2050] [node 3] ICS09 handshake done sender: [5:7574435649260577083:2112] self: [5:7574435649260577095:0] peer: [3:7574435650557448301:0] socket: 101 qp: -1
2025-11-19T13:58:47.332018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435649260577103:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:47.332023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:47.332222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435649260577099:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58435
2025-11-19T13:58:47.335454Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435650657500546:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:47.336366Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435649855663440:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:47.336463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435650557448321:2091] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.336486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:47.336493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435650557448321:2091] [node 4] ICP22 created new session: [3:7574435650557448348:2091]
2025-11-19T13:58:47.336500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435650557448348:2091] [node 4] ICS09 handshake done sender: [3:7574435650557448324:2104] self: [3:7574435650557448323:0] peer: [4:7574435649855663451:0] socket: 83 qp: -1
2025-11-19T13:58:47.336505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435650557448348:2091] [node 4] ICS10 traffic start
2025-11-19T13:58:47.336541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:47.336648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435650557448295:2082] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.336656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:47.336662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435650557448295:2082] [node 5] ICP22 created new session: [3:7574435650557448350:2082]
2025-11-19T13:58:47.336666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435650557448350:2082] [node 5] ICS09 handshake done sender: [3:7574435650557448303:2089] self: [3:7574435650557448301:0] peer: [5:7574435649260577095:0] socket: 81 qp: -1
2025-11-19T13:58:47.336669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435650557448350:2082] [node 5] ICS10 traffic start
2025-11-19T13:58:47.336673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.336105Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435650657500556:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.336133Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435650657500547:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.336153Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435650657500555:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.337495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435650657500493:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.337507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:47.337514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435650657500493:2061] [node 4] ICP22 created new session: [6:7574435650657500560:2061]
2025-11-19T13:58:47.337520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435650657500560:2061] [node 4] ICS09 handshake done sender: [6:7574435650657500546:2109] self: [6:7574435650657500557:0] peer: [4:7574435649855663407:0] socket: 79 qp: -1
2025-11-19T13:58:47.337527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435650657500560:2061] [node 4] ICS10 traffic start
2025-11-19T13:58:47.337544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:47.337804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435650657500559:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57965
2025-11-19T13:58:47.337845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435650657500540:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.337785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435649855663430:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.337857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435650657500477:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.337798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:47.337867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435650657500475:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.337804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435649855663430:2091] [node 5] ICP22 created new session: [4:7574435649855663458:2091]
2025-11-19T13:58:47.337810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435649855663458:2091] [node 5] ICS09 handshake done sender: [4:7574435649855663440:2115] self: [4:7574435649855663439:0] peer: [5:7574435649260577098:0] socket: 110 qp: -1
2025-11-19T13:58:47.337814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435649855663458:2091] [node 5] ICS10 traffic start
2025-11-19T13:58:47.337819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:47.337934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435649855663455:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52240
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.338483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435649855663456:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52240
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.338637Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435649260577086:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.338648Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435649260577086:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:47.338953Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435650657500559:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.338977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435650657500476:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.339095Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435650211117309:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.338767Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435650557448319:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.338775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435650557448319:2102] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:47.339240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435650557448346:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63500
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.339401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435650557448347:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63500
2025-11-19T13:58:47.339591Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435650557448340:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:47.339733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435650557448260:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.339757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:47.339762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435650557448260:2049] [node 2] ICP22 created new session: [3:7574435650557448355:2049]
2025-11-19T13:58:47.339767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435650557448355:2049] [node 2] ICS09 handshake done sender: [3:7574435650557448340:2117] self: [3:7574435650557448342:0] peer: [2:7574435650211117303:0] socket: 96 qp: -1
2025-11-19T13:58:47.339772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435650557448355:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:47.339779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.343945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435649260577102:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58435
2025-11-19T13:58:47.343967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435650657500566:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57965
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.344271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435647829048497:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50167
2025-11-19T13:58:47.344322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435647829048498:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50167
2025-11-19T13:58:47.344293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435650211117330:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55869
2025-11-19T13:58:47.344314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435650211117331:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55869
2025-11-19T13:58:47.344729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435649260577099:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.344808Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435649260577094:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:47.345003Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435650657500547:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:47.345075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435650657500477:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.345091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:47.345099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435650657500477:2050] [node 3] ICP22 created new session: [6:7574435650657500567:2050]
2025-11-19T13:58:47.345105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435650657500567:2050] [node 3] ICS09 handshake done sender: [6:7574435650657500547:2110] self: [6:7574435650657500563:0] peer: [3:7574435650557448318:0] socket: 85 qp: -1
2025-11-19T13:58:47.345111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435650657500567:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:47.345118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:47.345615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435649260577080:2087] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.345630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:47.345635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435649260577080:2087] [node 4] ICP22 created new session: [5:7574435649260577107:2087]
2025-11-19T13:58:47.345640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435649260577107:2087] [node 4] ICS09 handshake done sender: [5:7574435649260577094:2117] self: [5:7574435649260577098:0] peer: [4:7574435649855663439:0] socket: 111 qp: -1
2025-11-19T13:58:47.345645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435649260577107:2087] [node 4] ICS10 traffic start
2025-11-19T13:58:47.345651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:47.345773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435649260577014:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.347567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435649260577106:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58435
2025-11-19T13:58:47.344106Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435650557448319:2102] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:47.347688Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435650211117314:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.347698Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435650211117314:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:47.348376Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435650211117314:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:47.348483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435650211117290:2092] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.348498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:47.348503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435650211117290:2092] [node 5] ICP22 created new session: [2:7574435650211117333:2092]
2025-11-19T13:58:47.348509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435650211117333:2092] [node 5] ICS09 handshake done sender: [2:7574435650211117314:2113] self: [2:7574435650211117313:0] peer: [5:7574435649260577109:0] socket: 123 qp: -1
2025-11-19T13:58:47.348514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435650211117333:2092] [node 5] ICS10 traffic start
2025-11-19T13:58:47.348522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:47.348943Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435650211117308:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.348950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435650211117308:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:47.351880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435649855663457:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52240
2025-11-19T13:58:47.350900Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435647829048476:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.351804Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435647829048479:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.351830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435647829048479:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:47.352808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435650557448304:2087] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.352827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:47.352833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435650557448304:2087] [node 6] ICP22 created new session: [3:7574435650557448357:2087]
2025-11-19T13:58:47.352838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435650557448357:2087] [node 6] ICS09 handshake done sender: [3:7574435650557448319:2102] self: [3:7574435650557448318:0] peer: [6:7574435650657500563:0] socket: 82 qp: -1
2025-11-19T13:58:47.352844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435650557448357:2087] [node 6] ICS10 traffic start
2025-11-19T13:58:47.352850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:47.354670Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435649260577106:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.356003Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435647829048479:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:47.356210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435647829048461:2097] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.356227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:47.356234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435647829048461:2097] [node 6] ICP22 created new session: [1:7574435647829048501:2097]
2025-11-19T13:58:47.356240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435647829048501:2097] [node 6] ICS09 handshake done sender: [1:7574435647829048479:2113] self: [1:7574435647829048478:0] peer: [6:7574435650657500564:0] socket: 94 qp: -1
2025-11-19T13:58:47.356246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435647829048501:2097] [node 6] ICS10 traffic start
2025-11-19T13:58:47.356255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:47.360010Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435650557448347:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.360098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435650557448259:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.360468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435649855663366:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3320) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435649855663429:2106] held: no
2025-11-19T13:58:47.360790Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435649855663443:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:47.360245Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435647829048498:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.362337Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435649855663429:2106] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:47.362663Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435649260577086:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:47.362875Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435650211117308:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:47.364015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435649260577014:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435649260577099:2120] held: no
2025-11-19T13:58:47.364045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435649260577073:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.364060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:47.364065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435649260577073:2080] [node 6] ICP22 created new session: [5:7574435649260577113:2080]
2025-11-19T13:58:47.364072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435649260577113:2080] [node 6] ICS09 handshake done sender: [5:7574435649260577086:2114] self: [5:7574435649260577085:0] peer: [6:7574435650657500562:0] socket: 104 qp: -1
2025-11-19T13:58:47.364079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435649260577113:2080] [node 6] ICS10 traffic start
2025-11-19T13:58:47.364086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:47.364437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435650211117289:2093] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.364465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:47.364472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435650211117289:2093] [node 6] ICP22 created new session: [2:7574435650211117336:2093]
2025-11-19T13:58:47.364478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435650211117336:2093] [node 6] ICS09 handshake done sender: [2:7574435650211117308:2111] self: [2:7574435650211117307:0] peer: [6:7574435650657500565:0] socket: 106 qp: -1
2025-11-19T13:58:47.364484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435650211117336:2093] [node 6] ICS10 traffic start
2025-11-19T13:58:47.364493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:47.368037Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435650211117290:2092] [node 5] ICP35 event processing took too much time PendingConnection:278(19512919ns) {HandleHandshakeStatus:329(19512234ns) {DropIncomingHandshake:441(2794ns) DropOutgoingHandshake:456(91ns) SwitchToState:178(82ns) ProcessPendingSessionEvents:495(19444101ns) {ForwardSessionEventToSession:612(19430558ns) ForwardSessionEventToSession:612(1822ns) ForwardSessionEventToSession:612(723ns) ForwardSessionEventToSession:612(662ns) ForwardSessionEventToSession:612(645ns) ForwardSessionEventToSession:612(1680ns) ForwardSessionEventToSession:612(988ns) ForwardSessionEventToSession:612(576ns) ForwardSessionEventToSession:612(720ns) ForwardSessionEventToSession:612(581ns)}}}
2025-11-19T13:58:47.368191Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435647829048483:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.368207Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435647829048483:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:47.368655Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435647829048497:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.368823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435647829048469:2098] [node 4] ICP18 handshake (actor [1:7574435647829048497:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.369294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435650557448360:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63500
2025-11-19T13:58:47.369653Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435647829048483:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:47.369838Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435650557448347:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:47.369927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435650557448259:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.369946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:47.369951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435650557448259:2048] [node 1] ICP22 created new session: [3:7574435650557448361:2048]
2025-11-19T13:58:47.369957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435650557448361:2048] [node 1] ICS09 handshake done sender: [3:7574435650557448347:2119] self: [3:7574435650557448359:0] peer: [1:7574435647829048482:0] socket: 131 qp: -1
2025-11-19T13:58:47.369963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435650557448361:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:47.369973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:47.375380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435650657500475:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435650657500555:2117] held: no
2025-11-19T13:58:47.375415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435650657500476:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435650657500559:2120] held: no
2025-11-19T13:58:47.373495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435643534081112:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.373525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:47.373536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435643534081112:2049] [node 3] ICP22 created new session: [1:7574435647829048505:2049]
2025-11-19T13:58:47.373544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435647829048505:2049] [node 3] ICS09 handshake done sender: [1:7574435647829048483:2115] self: [1:7574435647829048482:0] peer: [3:7574435650557448359:0] socket: 122 qp: -1
2025-11-19T13:58:47.373552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435647829048505:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:47.373561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:47.376281Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435650211117331:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.376319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435645916149941:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:47.376453Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435647829048487:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:47.376460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435647829048487:2116] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.377277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435650211117339:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55869
2025-11-19T13:58:47.378324Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435647829048487:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:47.378562Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435650211117331:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:47.378680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435645916149941:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.378704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:47.378712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435645916149941:2048] [node 1] ICP22 created new session: [2:7574435650211117340:2048]
2025-11-19T13:58:47.378718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435650211117340:2048] [node 1] ICS09 handshake done sender: [2:7574435650211117331:2119] self: [2:7574435650211117338:0] peer: [1:7574435647829048470:0] socket: 135 qp: -1
2025-11-19T13:58:47.378737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435650211117340:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:47.378745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:47.379877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435643534081111:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:47.379896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:47.379902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435643534081111:2048] [node 2] ICP22 created new session: [1:7574435647829048508:2048]
2025-11-19T13:58:47.379906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435647829048508:2048] [node 2] ICS09 handshake done sender: [1:7574435647829048487:2116] self: [1:7574435647829048470:0] peer: [2:7574435650211117338:0] socket: 132 qp: -1
2025-11-19T13:58:47.379910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435647829048508:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:47.379917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.380393Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435649855663444:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:47.380445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435649855663365:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.380455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:47.380461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435649855663365:2049] [node 2] ICP22 created new session: [4:7574435649855663461:2049]
2025-11-19T13:58:47.380467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435649855663461:2049] [node 2] ICS09 handshake done sender: [4:7574435649855663443:2117] self: [4:7574435649855663450:0] peer: [2:7574435650211117305:0] socket: 98 qp: -1
2025-11-19T13:58:47.380471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435649855663461:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:47.380477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:47.380546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435649855663366:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.380553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:47.380556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435649855663366:2050] [node 3] ICP22 created new session: [4:7574435649855663463:2050]
2025-11-19T13:58:47.380560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435649855663463:2050] [node 3] ICS09 handshake done sender: [4:7574435649855663429:2106] self: [4:7574435649855663451:0] peer: [3:7574435650557448323:0] socket: 84 qp: -1
2025-11-19T13:58:47.380564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435649855663463:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:47.380569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:47.380547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435649260577112:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58435
2025-11-19T13:58:47.380735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435650657500570:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57965
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.381517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435650657500571:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57965
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.385111Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435649260577099:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:47.385561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435650657500572:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57965
2025-11-19T13:58:47.386010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435649260577014:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.386042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:47.386050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435649260577014:2049] [node 2] ICP22 created new session: [5:7574435649260577115:2049]
2025-11-19T13:58:47.386057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435649260577115:2049] [node 2] ICS09 handshake done sender: [5:7574435649260577099:2120] self: [5:7574435649260577109:0] peer: [2:7574435650211117313:0] socket: 124 qp: -1
2025-11-19T13:58:47.386067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435649260577115:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:47.386079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:47.386538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435649855663364:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435649855663444:2118] held: no
2025-11-19T13:58:47.386567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435649855663364:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.386575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:47.386580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435649855663364:2048] [node 1] ICP22 created new session: [4:7574435649855663465:2048]
2025-11-19T13:58:47.386586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435649855663465:2048] [node 1] ICS09 handshake done sender: [4:7574435649855663444:2118] self: [4:7574435649855663452:0] peer: [1:7574435647829048480:0] socket: 99 qp: -1
2025-11-19T13:58:47.386591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435649855663465:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:47.386598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:47.386691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435649855663406:2083] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:47.386716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435649855663460:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52240
2025-11-19T13:58:47.388340Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435650657500555:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:47.388442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435650657500540:2067] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17380) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435650657500556:2118] held: no
2025-11-19T13:58:47.388477Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435650657500559:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:47.388557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435650657500475:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.388573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:47.388579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435650657500475:2048] [node 1] ICP22 created new session: [6:7574435650657500573:2048]
2025-11-19T13:58:47.388586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435650657500573:2048] [node 1] ICS09 handshake done sender: [6:7574435650657500555:2117] self: [6:7574435650657500564:0] peer: [1:7574435647829048478:0] socket: 97 qp: -1
2025-11-19T13:58:47.388592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435650657500573:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:47.388600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:47.388717Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435650657500556:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:47.388906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435650657500476:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.388919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:47.388923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435650657500476:2049] [node 2] ICP22 created new session: [6:7574435650657500575:2049]
2025-11-19T13:58:47.388927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435650657500575:2049] [node 2] ICS09 handshake done sender: [6:7574435650657500559:2120] self: [6:7574435650657500565:0] peer: [2:7574435650211117307:0] socket: 128 qp: -1
2025-11-19T13:58:47.388933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435650657500575:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:47.388937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:47.389051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435650657500540:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:47.389056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:47.389059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435650657500540:2067] [node 5] ICP22 created new session: [6:7574435650657500577:2067]
2025-11-19T13:58:47.389063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435650657500577:2067] [node 5] ICS09 handshake done sender: [6:7574435650657500556:2118] self: [6:7574435650657500562:0] peer: [5:7574435649260577085:0] socket: 105 qp: -1
2025-11-19T13:58:47.389067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435650657500577:2067] [node 5] ICS10 traffic start
2025-11-19T13:58:47.389072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:47.404381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:47.404502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435650211117340:2048] [node 1] ICS01 socket: 135 reason# EndOfStream
2025-11-19T13:58:47.404516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435645916149941:2048] [node 1] ICP30 unregister session Session# [2:7574435650211117340:2048] VirtualId# [2:7574435650211117338:0]
2025-11-19T13:58:47.404523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435650211117340:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:47.404578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:47.404603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:47.405085Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:47.405157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435650557448361:2048] [node 1] ICS01 socket: 131 reason# EndOfStream
2025-11-19T13:58:47.405162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435650557448259:2048] [node 1] ICP30 unregister session Session# [3:7574435650557448361:2048] VirtualId# [3:7574435650557448359:0]
2025-11-19T13:58:47.405174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435650557448361:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:47.405205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:47.405228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:47.411906Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:47.411999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435650657500573:2048] [node 1] ICS01 socket: 97 reason# EndOfStream
2025-11-19T13:58:47.412007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435650657500475:2048] [node 1] ICP30 unregister session Session# [6:7574435650657500573:2048] VirtualId# [6:7574435650657500564:0]
2025-11-19T13:58:47.412014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435650657500573:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:47.412063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:47.412141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435649855663465:2048] [node 1] ICS01 socket: 99 reason# EndOfStream
2025-11-19T13:58:47.412147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435649855663364:2048] [node 1] ICP30 unregister session Session# [4:7574435649855663465:2048] VirtualId# [4:7574435649855663452:0]
2025-11-19T13:58:47.412154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435649855663465:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:47.412209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:47.412232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:47.412155Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:47.412231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435649260577100:2048] [node 1] ICS01 socket: 102 reason# EndOfStream
2025-11-19T13:58:47.412236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435649260577013:2048] [node 1] ICP30 unregister session Session# [5:7574435649260577100:2048] VirtualId# [5:7574435649260577093:0]
2025-11-19T13:58:47.412241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435649260577100:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:47.412270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:47.412296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:47.412082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:47.412113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:47.415701Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:47.415816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435650557448355:2049] [node 2] ICS01 socket: 96 reason# EndOfStream
2025-11-19T13:58:47.415823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435650557448260:2049] [node 2] ICP30 unregister session Session# [3:7574435650557448355:2049] VirtualId# [3:7574435650557448342:0]
2025-11-19T13:58:47.415830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435650557448355:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:47.415885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:47.415907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:47.423216Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:47.423315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435649855663461:2049] [node 2] ICS01 socket: 98 reason# EndOfStream
2025-11-19T13:58:47.423322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435649855663365:2049] [node 2] ICP30 unregister session Session# [4:7574435649855663461:2049] VirtualId# [4:7574435649855663450:0]
2025-11-19T13:58:47.423330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435649855663461:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:47.423387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:47.423414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:47.423455Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:47.423525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435650657500575:2049] [node 2] ICS01 socket: 128 reason# EndOfStream
2025-11-19T13:58:47.423538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435650657500476:2049] [node 2] ICP30 unregister session Session# [6:7574435650657500575:2049] VirtualId# [6:7574435650657500565:0]
2025-11-19T13:58:47.423543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435650657500575:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:47.423197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:47.423311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435649260577115:2049] [node 2] ICS01 socket: 124 reason# EndOfStream
2025-11-19T13:58:47.423320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435649260577014:2049] [node 2] ICP30 unregister session Session# [5:7574435649260577115:2049] VirtualId# [5:7574435649260577109:0]
2025-11-19T13:58:47.423327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435649260577115:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:47.423380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:47.423414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:47.423575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:47.423604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:47.440743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:47.440844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435649260577103:2050] [node 3] ICS01 socket: 101 reason# EndOfStream
2025-11-19T13:58:47.440854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435649260577015:2050] [node 3] ICP30 unregister session Session# [5:7574435649260577103:2050] VirtualId# [5:7574435649260577095:0]
2025-11-19T13:58:47.440862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435649260577103:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:47.440920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:47.440947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:47.440986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:47.441047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435650657500567:2050] [node 3] ICS01 socket: 85 reason# EndOfStream
2025-11-19T13:58:47.441051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435650657500477:2050] [node 3] ICP30 unregister session Session# [6:7574435650657500567:2050] VirtualId# [6:7574435650657500563:0]
2025-11-19T13:58:47.441056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435650657500567:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:47.441081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:47.441101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:47.452295Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:47.452411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435650657500560:2061] [node 4] ICS01 socket: 79 reason# EndOfStream
2025-11-19T13:58:47.452421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435650657500493:2061] [node 4] ICP30 unregister session Session# [6:7574435650657500560:2061] VirtualId# [6:7574435650657500557:0]
2025-11-19T13:58:47.452428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435650657500560:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:47.452494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:47.452523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:58:47.460001Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:58:47.460096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435650657500577:2067] [node 5] ICS01 socket: 105 reason# EndOfStream
2025-11-19T13:58:47.460106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435650657500540:2067] [node 5] ICP30 unregister session Session# [6:7574435650657500577:2067] VirtualId# [6:7574435650657500562:0]
2025-11-19T13:58:47.460114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435650657500577:2067] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:47.460163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:58:47.460186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:47.879955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435649173116687:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:47.879979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435649173116688:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:47.951929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435649371689072:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:47.951947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435649371689073:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:48.023893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435652308764894:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:48.024179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435652308764893:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:48.062907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435653500745295:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:48.062926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435653500745296:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:48.062931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435653500745297:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:48.123913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435654773477951:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:48.123941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435654773477952:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:48.123947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435654773477953:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:48.151503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435655112404026:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:48.151518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435655112404027:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:48.151523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435655112404028:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:48.204281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435653500745340:2091] [node 6] ICP01 ready to work
2025-11-19T13:58:48.204392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435653468084044:2107] [node 6] ICP01 ready to work
2025-11-19T13:58:48.205121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435654773477970:2059] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.205399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435655112404050:2065] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.206746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435653500745344:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2171
2025-11-19T13:58:48.206869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435653500745347:2095] [node 5] ICP01 ready to work
2025-11-19T13:58:48.207986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655112404056:2070] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14950
2025-11-19T13:58:48.205579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435653666656429:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51805
2025-11-19T13:58:48.210710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435653666656434:2109] [node 4] ICP01 ready to work
2025-11-19T13:58:48.210785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435653666656435:2110] [node 5] ICP01 ready to work
2025-11-19T13:58:48.215953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435652308764954:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:48.215951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655112404058:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14950
2025-11-19T13:58:48.216042Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435655112404056:2070] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.216161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435655112404083:2083] [node 5] ICP01 ready to work
2025-11-19T13:58:48.224123Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435655112404028:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(16129361ns) {Configure:104(16128794ns) {ProcessConfigured:135(15881300ns) {StartInitialHandshake:157(15880597ns) {DropHandshakes:469(523ns) {DropIncomingHandshake:441(115ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(266ns) PrepareNewSessionHandshake:476(2259ns) {SwitchToState:178(1420ns)}}}}}
2025-11-19T13:58:48.224160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435655112404050:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.231592Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435649173116687:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(21367753ns) {Configure:104(21366897ns) {ProcessConfigured:135(21104874ns) {StartInitialHandshake:157(21104330ns) {DropHandshakes:469(436ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(584ns) PrepareNewSessionHandshake:476(723ns) {SwitchToState:178(143ns)}}}}}
2025-11-19T13:58:48.231833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435653468084056:2111] [node 4] ICP01 ready to work
2025-11-19T13:58:48.231407Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435654773477952:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(22709692ns) {Configure:104(22708780ns) {ProcessConfigured:135(22441569ns) {StartInitialHandshake:157(22440806ns) {DropHandshakes:469(464ns) {DropIncomingHandshake:441(35ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(270ns) PrepareNewSessionHandshake:476(1821ns) {SwitchToState:178(797ns)}}}}}
2025-11-19T13:58:48.241798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435653666656434:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(30957498ns) {Configure:104(30956705ns) {ProcessConfigured:135(30695963ns) {StartInitialHandshake:157(30695306ns) {DropHandshakes:469(335ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(38ns) PrepareNewSessionHandshake:476(1245ns) {SwitchToState:178(312ns)}}}}}
2025-11-19T13:58:48.241994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435653666656440:2111] [node 6] ICP01 ready to work
2025-11-19T13:58:48.244375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435653468084052:2110] [node 5] ICP01 ready to work
2025-11-19T13:58:48.248120Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435655112404058:2072] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.248167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435655112404026:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.244855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435654773477982:2068] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.251910Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435653500745342:2092] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.251929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435653500745342:2092] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.254952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435653500745374:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2171
2025-11-19T13:58:48.255019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435653500745375:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2171
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.256249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655112404107:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14950
2025-11-19T13:58:48.255671Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435653666656429:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.259155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435653666656435:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(48042786ns) {Configure:104(43135574ns) {ProcessConfigured:135(42850658ns) {StartInitialHandshake:157(42848762ns) {DropHandshakes:469(961ns) {DropIncomingHandshake:441(160ns) DropOutgoingHandshake:456(46ns)} GenerateSessionVirtualId:988(183ns) PrepareNewSessionHandshake:476(654ns) {SwitchToState:178(559ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.259719Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435653468084046:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.259742Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435653468084046:2108] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.261361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435653468084061:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
2025-11-19T13:58:48.261407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435653468084062:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
2025-11-19T13:58:48.261420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435653468084063:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
2025-11-19T13:58:48.261463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435653468084064:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33121
2025-11-19T13:58:48.261444Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435653500745342:2092] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:48.261521Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435653500745374:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.264262Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435653468084046:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:48.268685Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435653500745347:2095] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(60794345ns) {Configure:104(60793177ns) {ProcessConfigured:135(60499176ns) {StartInitialHandshake:157(60498653ns) {DropHandshakes:469(687ns) {DropIncomingHandshake:441(188ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(141ns) PrepareNewSessionHandshake:476(1223ns) {SwitchToState:178(502ns)}}}}}
2025-11-19T13:58:48.268297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435653468084052:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23737440ns) {Configure:104(23736064ns) {ProcessConfigured:135(23437530ns) {StartInitialHandshake:157(23436625ns) {DropHandshakes:469(604ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(85ns)} GenerateSessionVirtualId:988(250ns) PrepareNewSessionHandshake:476(1756ns) {SwitchToState:178(724ns)}}}}}
2025-11-19T13:58:48.268324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435653468084044:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.268339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:48.268349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435653468084044:2107] [node 6] ICP22 created new session: [1:7574435653468084067:2107]
2025-11-19T13:58:48.268354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435653468084067:2107] [node 6] ICS09 handshake done sender: [1:7574435653468084046:2108] self: [1:7574435653468084045:0] peer: [6:7574435655112404104:0] socket: 81 qp: -1
2025-11-19T13:58:48.268372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435653468084067:2107] [node 6] ICS10 traffic start
2025-11-19T13:58:48.268386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:48.268437Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435653468084061:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.268576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435653468084044:2107] [node 6] ICP18 handshake (actor [1:7574435653468084061:2114]) is too old
2025-11-19T13:58:48.270562Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435654773477982:2068] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25647106ns) {Configure:104(25645827ns) {ProcessConfigured:135(25400868ns) {StartInitialHandshake:157(25400176ns) {DropHandshakes:469(253ns) {DropIncomingHandshake:441(29ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:476(885ns) {SwitchToState:178(695ns)}}}}}
2025-11-19T13:58:48.270957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435652308764954:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(54543374ns) {Configure:104(25555167ns) {ProcessConfigured:135(25293499ns) {StartInitialHandshake:157(25292296ns) {DropHandshakes:469(421ns) {DropIncomingHandshake:441(28ns) DropOutgoingHandshake:456(34ns)} GenerateSessionVirtualId:988(175ns) PrepareNewSessionHandshake:476(1067ns) {SwitchToState:178(877ns)}}}}}
2025-11-19T13:58:48.271182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435652308764960:2108] [node 5] ICP01 ready to work
2025-11-19T13:58:48.271267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435652308764961:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:48.271651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435655112404083:2083] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(55436944ns) {Configure:104(55436365ns) {ProcessConfigured:135(55171583ns) {StartInitialHandshake:157(55170839ns) {DropHandshakes:469(424ns) {DropIncomingHandshake:441(81ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(142ns) PrepareNewSessionHandshake:476(776ns) {SwitchToState:178(729ns)}}}}}
2025-11-19T13:58:48.271691Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435655112404056:2070] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:48.270016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435653500745340:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.270037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:48.270057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435653500745340:2091] [node 6] ICP22 created new session: [4:7574435653500745377:2091]
2025-11-19T13:58:48.270063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435653500745377:2091] [node 6] ICS09 handshake done sender: [4:7574435653500745342:2092] self: [4:7574435653500745341:0] peer: [6:7574435655112404091:0] socket: 78 qp: -1
2025-11-19T13:58:48.270071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435653500745377:2091] [node 6] ICS10 traffic start
2025-11-19T13:58:48.270091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:48.270169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435653500745340:2091] [node 6] ICP18 handshake (actor [4:7574435653500745374:2115]) is too old
2025-11-19T13:58:48.270263Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435653500745375:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.272845Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435653468084063:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.281489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435653500745296:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.281536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435653500745379:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2171
2025-11-19T13:58:48.281833Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435653666656439:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.281850Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435653666656439:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.283202Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435653666656439:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:48.283346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435653666656434:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.283368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:48.283376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435653666656434:2109] [node 4] ICP22 created new session: [2:7574435653666656450:2109]
2025-11-19T13:58:48.283384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435653666656450:2109] [node 4] ICS09 handshake done sender: [2:7574435653666656439:2112] self: [2:7574435653666656436:0] peer: [4:7574435653500745381:0] socket: 96 qp: -1
2025-11-19T13:58:48.283393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435653666656450:2109] [node 4] ICS10 traffic start
2025-11-19T13:58:48.283417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.283886Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435652308764894:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(67872852ns) {Configure:104(67871934ns) {ProcessConfigured:135(67479389ns) {StartInitialHandshake:157(67477664ns) {DropHandshakes:469(532ns) {DropIncomingHandshake:441(87ns) DropOutgoingHandshake:456(103ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:476(741ns) {SwitchToState:178(465ns)}}}}}
2025-11-19T13:58:48.284458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435652308764963:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52526
2025-11-19T13:58:48.284521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435652308764964:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52526
2025-11-19T13:58:48.284535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435652308764966:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52526
2025-11-19T13:58:48.284573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435652308764967:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52526
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.304491Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435653468084064:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.304718Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435652308764893:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(33389017ns) {Configure:104(33388228ns) {ProcessConfigured:135(33057808ns) {StartInitialHandshake:157(33056863ns) {DropHandshakes:469(416ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(180ns) PrepareNewSessionHandshake:476(861ns) {SwitchToState:178(413ns)}}}}}
2025-11-19T13:58:48.304753Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435652308764966:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.304781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435652308764894:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.308706Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435652308764963:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.308751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435652308764893:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.308917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435653666656453:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51805
2025-11-19T13:58:48.308970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435653666656454:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51805
2025-11-19T13:58:48.309007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435653666656455:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51805
2025-11-19T13:58:48.308963Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435653468084048:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.308977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435653468084048:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.309603Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435653468084048:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:48.309695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435649173116688:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.309714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:48.309720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435649173116688:2049] [node 3] ICP22 created new session: [1:7574435653468084073:2049]
2025-11-19T13:58:48.309728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435653468084073:2049] [node 3] ICS09 handshake done sender: [1:7574435653468084048:2109] self: [1:7574435653468084047:0] peer: [3:7574435652308764975:0] socket: 82 qp: -1
2025-11-19T13:58:48.309735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435653468084073:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:48.309743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.312317Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435653666656453:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.312361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435649371689072:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.312822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435653500745383:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2171
2025-11-19T13:58:48.313047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435653500745382:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2171
2025-11-19T13:58:48.311914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435655112404050:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.311941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:48.311950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435655112404050:2065] [node 4] ICP22 created new session: [6:7574435655112404111:2065]
2025-11-19T13:58:48.311957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435655112404111:2065] [node 4] ICS09 handshake done sender: [6:7574435655112404056:2070] self: [6:7574435655112404091:0] peer: [4:7574435653500745341:0] socket: 89 qp: -1
2025-11-19T13:58:48.311966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435655112404111:2065] [node 4] ICS10 traffic start
2025-11-19T13:58:48.311978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:48.312075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435655112404050:2065] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:48.312198Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435655112404026:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24961) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435655112404058:2072] held: no
2025-11-19T13:58:48.312295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655112404109:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14950
2025-11-19T13:58:48.315873Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435653666656442:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.315891Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435653666656442:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.316142Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435653666656454:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.316183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435653666656434:2109] [node 4] ICP18 handshake (actor [2:7574435653666656454:2117]) is too old
2025-11-19T13:58:48.316579Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435653500745375:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:48.316844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435653500745296:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.316862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:48.316869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435653500745296:2049] [node 2] ICP22 created new session: [4:7574435653500745384:2049]
2025-11-19T13:58:48.316876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435653500745384:2049] [node 2] ICS09 handshake done sender: [4:7574435653500745375:2116] self: [4:7574435653500745381:0] peer: [2:7574435653666656436:0] socket: 102 qp: -1
2025-11-19T13:58:48.316883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435653500745384:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:48.316891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:48.316862Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435653468084055:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.316876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435653468084055:2112] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.317681Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435653666656442:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:48.317816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435653666656458:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51805
2025-11-19T13:58:48.317922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435649371689073:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.317934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:48.317939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435649371689073:2049] [node 3] ICP22 created new session: [2:7574435653666656459:2049]
2025-11-19T13:58:48.317945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435653666656459:2049] [node 3] ICS09 handshake done sender: [2:7574435653666656442:2113] self: [2:7574435653666656441:0] peer: [3:7574435652308764974:0] socket: 97 qp: -1
2025-11-19T13:58:48.317950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435653666656459:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:48.317972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:48.320898Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435655112404058:2072] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:48.323285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655112404110:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14950
2025-11-19T13:58:48.323873Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435653468084055:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.324114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435654773478029:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24895
2025-11-19T13:58:48.324003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435649173116687:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.324024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:48.324032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435649173116687:2048] [node 2] ICP22 created new session: [1:7574435653468084077:2048]
2025-11-19T13:58:48.324039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435653468084077:2048] [node 2] ICS09 handshake done sender: [1:7574435653468084055:2112] self: [1:7574435653468084053:0] peer: [2:7574435653666656456:0] socket: 95 qp: -1
2025-11-19T13:58:48.324044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435653468084077:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:48.324050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:48.323794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435652308764961:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(39853234ns) {Configure:104(39851876ns) {ProcessConfigured:135(10527553ns) {StartInitialHandshake:157(10526295ns) {DropHandshakes:469(1151ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(530ns)} GenerateSessionVirtualId:988(331ns) PrepareNewSessionHandshake:476(1009ns) {SwitchToState:178(485ns)}}}}}
2025-11-19T13:58:48.324277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435653500745379:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.324382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435653500745295:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.328024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435653468084058:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.328034Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435653468084058:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.327905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435655112404026:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.327927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:48.327933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435655112404026:2048] [node 1] ICP22 created new session: [6:7574435655112404115:2048]
2025-11-19T13:58:48.327940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435655112404115:2048] [node 1] ICS09 handshake done sender: [6:7574435655112404058:2072] self: [6:7574435655112404104:0] peer: [1:7574435653468084045:0] socket: 90 qp: -1
2025-11-19T13:58:48.327948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435655112404115:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:48.327959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.328452Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435652308764967:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.328649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435652308764979:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52526
2025-11-19T13:58:48.331834Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435655112404110:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.331872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435655112404027:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.332612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435652308764980:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52526
2025-11-19T13:58:48.332002Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435653666656453:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.336713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655112404118:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14950
2025-11-19T13:58:48.337945Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435653468084058:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:48.336013Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435653666656444:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.336020Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435653666656444:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.336641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435649371689072:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.336654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:48.336662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435649371689072:2048] [node 1] ICP22 created new session: [2:7574435653666656463:2048]
2025-11-19T13:58:48.336669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435653666656463:2048] [node 1] ICS09 handshake done sender: [2:7574435653666656453:2116] self: [2:7574435653666656456:0] peer: [1:7574435653468084053:0] socket: 98 qp: -1
2025-11-19T13:58:48.336675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435653666656463:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:48.336683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:48.338205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435653468084056:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.338221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:48.338226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435653468084056:2111] [node 4] ICP22 created new session: [1:7574435653468084080:2111]
2025-11-19T13:58:48.338232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435653468084080:2111] [node 4] ICS09 handshake done sender: [1:7574435653468084058:2113] self: [1:7574435653468084057:0] peer: [4:7574435653500745387:0] socket: 109 qp: -1
2025-11-19T13:58:48.338236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435653468084080:2111] [node 4] ICS10 traffic start
2025-11-19T13:58:48.338242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:48.338640Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435653666656444:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:48.338760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435653666656440:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.338773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:48.338778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435653666656440:2111] [node 6] ICP22 created new session: [2:7574435653666656465:2111]
2025-11-19T13:58:48.338782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435653666656465:2111] [node 6] ICS09 handshake done sender: [2:7574435653666656444:2114] self: [2:7574435653666656443:0] peer: [6:7574435655112404117:0] socket: 111 qp: -1
2025-11-19T13:58:48.338785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435653666656465:2111] [node 6] ICS10 traffic start
2025-11-19T13:58:48.338794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:48.340371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435655112404027:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24010) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435655112404110:2119] held: no
2025-11-19T13:58:48.340401Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435655112404110:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:48.340473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435655112404027:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.340485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:48.340490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435655112404027:2049] [node 2] ICP22 created new session: [6:7574435655112404119:2049]
2025-11-19T13:58:48.340494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435655112404119:2049] [node 2] ICS09 handshake done sender: [6:7574435655112404110:2119] self: [6:7574435655112404117:0] peer: [2:7574435653666656443:0] socket: 121 qp: -1
2025-11-19T13:58:48.340499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435655112404119:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:48.340504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:48.341376Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435653500745295:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24961) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435653500745379:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.341819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435652308764981:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52526
2025-11-19T13:58:48.341879Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435652308764963:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:48.342181Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435652308764966:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:48.341437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435653500745388:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2171
2025-11-19T13:58:48.342408Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435653500745379:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:48.352100Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435653468084062:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.355135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435654773478030:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24895
2025-11-19T13:58:48.355206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435654773478031:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24895
2025-11-19T13:58:48.355239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435654773478032:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24895
2025-11-19T13:58:48.355275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435654773478033:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24895
2025-11-19T13:58:48.356045Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435654773478033:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.356075Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435654773478031:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.356103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435654773477970:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.356102Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435654773478030:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.356113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435654773477952:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.356153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435654773477951:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.351083Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435652308764981:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.342489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435653500745295:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.342505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:48.342510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435653500745295:2048] [node 1] ICP22 created new session: [4:7574435653500745389:2048]
2025-11-19T13:58:48.342515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435653500745389:2048] [node 1] ICS09 handshake done sender: [4:7574435653500745379:2118] self: [4:7574435653500745387:0] peer: [1:7574435653468084057:0] socket: 110 qp: -1
2025-11-19T13:58:48.342521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435653500745389:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:48.342528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:48.356223Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435653500745344:2093] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.356256Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435653500745376:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.356259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435653500745376:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.356231Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435653468084066:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.356241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435653468084066:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.357082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435652308764894:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.357105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:48.357111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435652308764894:2049] [node 2] ICP22 created new session: [3:7574435652308764983:2049]
2025-11-19T13:58:48.357117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435652308764983:2049] [node 2] ICS09 handshake done sender: [3:7574435652308764966:2114] self: [3:7574435652308764974:0] peer: [2:7574435653666656441:0] socket: 113 qp: -1
2025-11-19T13:58:48.357125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435652308764983:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:48.357133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:48.357252Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435653468084066:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:48.357337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435653468084052:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.357352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:48.357358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435653468084052:2110] [node 5] ICP22 created new session: [1:7574435653468084084:2110]
2025-11-19T13:58:48.357363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435653468084084:2110] [node 5] ICS09 handshake done sender: [1:7574435653468084066:2118] self: [1:7574435653468084060:0] peer: [5:7574435654773478036:0] socket: 123 qp: -1
2025-11-19T13:58:48.357367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435653468084084:2110] [node 5] ICS10 traffic start
2025-11-19T13:58:48.357373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.360761Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435654773478029:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.360787Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435654773478032:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.360822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435654773477953:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.360837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435654773478037:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24895
2025-11-19T13:58:48.360840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435654773478038:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24895
2025-11-19T13:58:48.361160Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435653500745383:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.361234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435653500745297:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.359902Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435653666656448:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.359919Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435653666656448:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.361216Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435653666656455:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.361258Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435653666656448:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:48.361405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435649371689073:2049] [node 3] ICP18 handshake (actor [2:7574435653666656455:2118]) is too old
2025-11-19T13:58:48.361417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435653666656435:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.361424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:48.361429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435653666656435:2110] [node 5] ICP22 created new session: [2:7574435653666656468:2110]
2025-11-19T13:58:48.361434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435653666656468:2110] [node 5] ICS09 handshake done sender: [2:7574435653666656448:2115] self: [2:7574435653666656438:0] peer: [5:7574435654773478035:0] socket: 128 qp: -1
2025-11-19T13:58:48.361439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435653666656468:2110] [node 5] ICS10 traffic start
2025-11-19T13:58:48.361445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.361933Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435653500745376:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:48.362291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435653500745347:2095] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.362302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:48.362307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435653500745347:2095] [node 5] ICP22 created new session: [4:7574435653500745394:2095]
2025-11-19T13:58:48.362312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435653500745394:2095] [node 5] ICS09 handshake done sender: [4:7574435653500745376:2117] self: [4:7574435653500745349:0] peer: [5:7574435654773478034:0] socket: 100 qp: -1
2025-11-19T13:58:48.362317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435653500745394:2095] [node 5] ICS10 traffic start
2025-11-19T13:58:48.362322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:48.362229Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435654773478031:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:48.362642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435654773478041:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24895
2025-11-19T13:58:48.363476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435654773477952:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.363485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:48.363490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435654773477952:2049] [node 2] ICP22 created new session: [5:7574435654773478042:2049]
2025-11-19T13:58:48.363495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435654773478042:2049] [node 2] ICS09 handshake done sender: [5:7574435654773478031:2117] self: [5:7574435654773478035:0] peer: [2:7574435653666656438:0] socket: 86 qp: -1
2025-11-19T13:58:48.363500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435654773478042:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:48.363516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:48.363611Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435654773478030:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:48.363770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435654773477951:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24961) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435654773478030:2116] held: no
2025-11-19T13:58:48.363786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435654773477951:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.363795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:48.363800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435654773477951:2048] [node 1] ICP22 created new session: [5:7574435654773478044:2048]
2025-11-19T13:58:48.363805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435654773478044:2048] [node 1] ICS09 handshake done sender: [5:7574435654773478030:2116] self: [5:7574435654773478036:0] peer: [1:7574435653468084060:0] socket: 79 qp: -1
2025-11-19T13:58:48.363812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435654773478044:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:48.363817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:48.363875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435652308764893:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.363890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:48.363896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435652308764893:2048] [node 1] ICP22 created new session: [3:7574435652308764985:2048]
2025-11-19T13:58:48.363902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435652308764985:2048] [node 1] ICS09 handshake done sender: [3:7574435652308764963:2111] self: [3:7574435652308764975:0] peer: [1:7574435653468084047:0] socket: 93 qp: -1
2025-11-19T13:58:48.363906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435652308764985:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:48.363921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:48.364076Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435652308764964:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.364099Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435652308764969:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.364102Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435652308764969:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.364467Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435652308764959:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.364470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435652308764959:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.367152Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435652308764969:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:48.367276Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435652308764959:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:48.367374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435652308764960:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.367388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:48.367394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435652308764960:2108] [node 5] ICP22 created new session: [3:7574435652308764991:2108]
2025-11-19T13:58:48.367400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435652308764991:2108] [node 5] ICS09 handshake done sender: [3:7574435652308764969:2116] self: [3:7574435652308764968:0] peer: [5:7574435654773478039:0] socket: 117 qp: -1
2025-11-19T13:58:48.367405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435652308764991:2108] [node 5] ICS10 traffic start
2025-11-19T13:58:48.367411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:48.367490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435652308764954:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.367497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:48.367501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435652308764954:2107] [node 4] ICP22 created new session: [3:7574435652308764993:2107]
2025-11-19T13:58:48.367505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435652308764993:2107] [node 4] ICS09 handshake done sender: [3:7574435652308764959:2110] self: [3:7574435652308764958:0] peer: [4:7574435653500745393:0] socket: 115 qp: -1
2025-11-19T13:58:48.367508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435652308764993:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:48.367523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.368216Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435653500745297:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11981) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435653500745383:2120] held: no
2025-11-19T13:58:48.368447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435653500745396:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2171
2025-11-19T13:58:48.368593Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435653500745383:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:48.368692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435653500745297:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.368699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:48.368703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435653500745297:2050] [node 3] ICP22 created new session: [4:7574435653500745397:2050]
2025-11-19T13:58:48.368708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435653500745397:2050] [node 3] ICS09 handshake done sender: [4:7574435653500745383:2120] self: [4:7574435653500745393:0] peer: [3:7574435652308764958:0] socket: 120 qp: -1
2025-11-19T13:58:48.368712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435653500745397:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:48.368716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:48.368165Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435654773478033:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:48.369600Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435654773477970:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31496) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435654773478033:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.370313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435654773477953:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11981) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435654773478029:2115] held: no
2025-11-19T13:58:48.370427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435654773478046:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24895
2025-11-19T13:58:48.371566Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435654773478029:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:48.366832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655112404121:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14950
2025-11-19T13:58:48.371161Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435655112404121:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.371247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435655112404028:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.371264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655112404122:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14950
2025-11-19T13:58:48.371647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435654773477953:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.371659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:48.371665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435654773477953:2050] [node 3] ICP22 created new session: [5:7574435654773478048:2050]
2025-11-19T13:58:48.371670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435654773478048:2050] [node 3] ICS09 handshake done sender: [5:7574435654773478029:2115] self: [5:7574435654773478039:0] peer: [3:7574435652308764968:0] socket: 118 qp: -1
2025-11-19T13:58:48.371674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435654773478048:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:48.371681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:48.371972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435654773477970:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.371981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:48.371986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435654773477970:2059] [node 4] ICP22 created new session: [5:7574435654773478050:2059]
2025-11-19T13:58:48.371990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435654773478050:2059] [node 4] ICS09 handshake done sender: [5:7574435654773478033:2119] self: [5:7574435654773478034:0] peer: [4:7574435653500745349:0] socket: 101 qp: -1
2025-11-19T13:58:48.371995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435654773478050:2059] [node 4] ICS10 traffic start
2025-11-19T13:58:48.372000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:48.373227Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435655112404122:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.373265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435655112404083:2083] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.373371Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435654773478028:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.373378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435654773478028:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.374039Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435654773478028:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:48.375280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435654773477982:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.375296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:48.375301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435654773477982:2068] [node 6] ICP22 created new session: [5:7574435654773478053:2068]
2025-11-19T13:58:48.375306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435654773478053:2068] [node 6] ICS09 handshake done sender: [5:7574435654773478028:2114] self: [5:7574435654773478016:0] peer: [6:7574435655112404124:0] socket: 112 qp: -1
2025-11-19T13:58:48.375310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435654773478053:2068] [node 6] ICS10 traffic start
2025-11-19T13:58:48.375315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:48.375863Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435652308764977:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.375873Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435652308764977:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.377155Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435652308764977:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:48.378402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435652308764961:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.378419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:48.378426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435652308764961:2109] [node 6] ICP22 created new session: [3:7574435652308764996:2109]
2025-11-19T13:58:48.378431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435652308764996:2109] [node 6] ICS09 handshake done sender: [3:7574435652308764977:2118] self: [3:7574435652308764976:0] peer: [6:7574435655112404123:0] socket: 104 qp: -1
2025-11-19T13:58:48.378435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435652308764996:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:48.378441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:48.373907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655112404125:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14950
2025-11-19T13:58:48.374213Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435655112404122:2122] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:48.374310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435655112404083:2083] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.374323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:48.374327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435655112404083:2083] [node 5] ICP22 created new session: [6:7574435655112404126:2083]
2025-11-19T13:58:48.374332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435655112404126:2083] [node 5] ICS09 handshake done sender: [6:7574435655112404122:2122] self: [6:7574435655112404124:0] peer: [5:7574435654773478016:0] socket: 114 qp: -1
2025-11-19T13:58:48.374336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435655112404126:2083] [node 5] ICS10 traffic start
2025-11-19T13:58:48.374342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:48.376752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655112404128:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14950
2025-11-19T13:58:48.377317Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435655112404121:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:48.377392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435655112404028:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.377402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:48.377406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435655112404028:2050] [node 3] ICP22 created new session: [6:7574435655112404129:2050]
2025-11-19T13:58:48.377411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435655112404129:2050] [node 3] ICS09 handshake done sender: [6:7574435655112404121:2121] self: [6:7574435655112404123:0] peer: [3:7574435652308764976:0] socket: 140 qp: -1
2025-11-19T13:58:48.377416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435655112404129:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:48.377423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:48.382751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:48.382835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435653666656463:2048] [node 1] ICS01 socket: 98 reason# EndOfStream
2025-11-19T13:58:48.382843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435649371689072:2048] [node 1] ICP30 unregister session Session# [2:7574435653666656463:2048] VirtualId# [2:7574435653666656456:0]
2025-11-19T13:58:48.382849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435653666656463:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.382892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:48.382912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:48.383320Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:48.383396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435652308764985:2048] [node 1] ICS01 socket: 93 reason# EndOfStream
2025-11-19T13:58:48.383402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435652308764893:2048] [node 1] ICP30 unregister session Session# [3:7574435652308764985:2048] VirtualId# [3:7574435652308764975:0]
2025-11-19T13:58:48.383406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435652308764985:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.383433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:48.383448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:48.390766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:48.390856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435654773478044:2048] [node 1] ICS01 socket: 79 reason# EndOfStream
2025-11-19T13:58:48.390864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435654773477951:2048] [node 1] ICP30 unregister session Session# [5:7574435654773478044:2048] VirtualId# [5:7574435654773478036:0]
2025-11-19T13:58:48.390870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435654773478044:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.390915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:48.390936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:48.390965Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:48.391021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435653500745389:2048] [node 1] ICS01 socket: 110 reason# EndOfStream
2025-11-19T13:58:48.391025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435653500745295:2048] [node 1] ICP30 unregister session Session# [4:7574435653500745389:2048] VirtualId# [4:7574435653500745387:0]
2025-11-19T13:58:48.391028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435653500745389:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.391051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:48.391070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:48.391505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:48.391571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435655112404115:2048] [node 1] ICS01 socket: 90 reason# EndOfStream
2025-11-19T13:58:48.391579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435655112404026:2048] [node 1] ICP30 unregister session Session# [6:7574435655112404115:2048] VirtualId# [6:7574435655112404104:0]
2025-11-19T13:58:48.391583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435655112404115:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.391610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:48.391630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:48.397765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:48.397872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435652308764983:2049] [node 2] ICS07 socket disconnect 113 reason# EndOfStream
2025-11-19T13:58:48.397877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435652308764983:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.397918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:48.397935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435652308764983:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:48.404348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435652308764894:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24010) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:58:48.404365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435652308764983:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:48.404839Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435652308764894:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24010) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:48.404852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435652308764983:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:48.404858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435652308764894:2049] [node 2] ICP30 unregister session Session# [3:7574435652308764983:2049] VirtualId# [3:7574435652308764974:0]
2025-11-19T13:58:48.404879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:48.404885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435652308764894:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24010) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:48.404897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24010) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:48.410658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:48.410730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435655112404119:2049] [node 2] ICS01 socket: 121 reason# EndOfStream
2025-11-19T13:58:48.410737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435655112404027:2049] [node 2] ICP30 unregister session Session# [6:7574435655112404119:2049] VirtualId# [6:7574435655112404117:0]
2025-11-19T13:58:48.410741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435655112404119:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.410163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:48.410242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435653500745384:2049] [node 2] ICS07 socket disconnect 102 reason# EndOfStream
2025-11-19T13:58:48.410249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435653500745384:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.410295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:48.410314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435653500745384:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:48.410506Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:48.410572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435654773478042:2049] [node 2] ICS01 socket: 86 reason# EndOfStream
2025-11-19T13:58:48.410578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435654773477952:2049] [node 2] ICP30 unregister session Session# [5:7574435654773478042:2049] VirtualId# [5:7574435654773478035:0]
2025-11-19T13:58:48.410583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435654773478042:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.410608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:48.410629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:48.411043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435653500745296:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24010) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:48.411053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435653500745384:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:48.411057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435653500745296:2049] [node 2] ICP30 unregister session Session# [4:7574435653500745384:2049] VirtualId# [4:7574435653500745381:0]
2025-11-19T13:58:48.411070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:48.411076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435653500745296:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24010) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:48.411079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24010) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:48.410761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:48.410782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:48.423832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435653500745296:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:58:48.429579Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:48.429684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435655112404129:2050] [node 3] ICS01 socket: 140 reason# EndOfStream
2025-11-19T13:58:48.429695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435655112404028:2050] [node 3] ICP30 unregister session Session# [6:7574435655112404129:2050] VirtualId# [6:7574435655112404123:0]
2025-11-19T13:58:48.429703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435655112404129:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.429750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:48.429773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:48.431337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:48.431418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435654773478048:2050] [node 3] ICS01 socket: 118 reason# EndOfStream
2025-11-19T13:58:48.431424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435654773477953:2050] [node 3] ICP30 unregister session Session# [5:7574435654773478048:2050] VirtualId# [5:7574435654773478039:0]
2025-11-19T13:58:48.431430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435654773478048:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.431474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:48.431496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:48.439454Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:48.439547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435655112404111:2065] [node 4] ICS01 socket: 89 reason# EndOfStream
2025-11-19T13:58:48.439555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435655112404050:2065] [node 4] ICP30 unregister session Session# [6:7574435655112404111:2065] VirtualId# [6:7574435655112404091:0]
2025-11-19T13:58:48.439562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435655112404111:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.439609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:48.439634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:48.563378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435654398347146:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:48.563391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435654398347147:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:48.652864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435655126587554:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:48.652876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435655126587555:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:48.744036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435653225023318:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:48.744053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435653225023319:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:48.767887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435654880951410:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:48.767902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435654880951411:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:48.767906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435654880951412:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:48.779847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435652728661964:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:48.779858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435652728661965:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:48.779863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435652728661966:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:48.842104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435654880951462:2096] [node 5] ICP01 ready to work
2025-11-19T13:58:48.841446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435654398347209:2107] [node 6] ICP01 ready to work
2025-11-19T13:58:48.844078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435654398347211:2109] [node 4] ICP01 ready to work
2025-11-19T13:58:48.847890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435653225023377:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.848288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435655126587616:2106] [node 4] ICP01 ready to work
2025-11-19T13:58:48.848846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435655126587617:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35891
2025-11-19T13:58:48.852119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435654398347208:2105] [node 5] ICP01 ready to work
2025-11-19T13:58:48.852331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435654880951470:2101] [node 6] ICP01 ready to work
2025-11-19T13:58:48.860447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435653225023390:2111] [node 4] ICP01 ready to work
2025-11-19T13:58:48.868428Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435654398347208:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16226004ns) {Configure:104(16224948ns) {ProcessConfigured:135(15875880ns) {StartInitialHandshake:157(15875267ns) {DropHandshakes:469(893ns) {DropIncomingHandshake:441(163ns) DropOutgoingHandshake:456(145ns)} GenerateSessionVirtualId:988(388ns) PrepareNewSessionHandshake:476(1345ns) {SwitchToState:178(811ns)}}}}}
2025-11-19T13:58:48.867862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435652728661964:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24508394ns) {Configure:104(24507584ns) {ProcessConfigured:135(24184272ns) {StartInitialHandshake:157(24183655ns) {DropHandshakes:469(289ns) {DropIncomingHandshake:441(43ns) DropOutgoingHandshake:456(38ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:476(1421ns) {SwitchToState:178(723ns)}}}}}
2025-11-19T13:58:48.832164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435651915342534:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:48.832227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435651915342533:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:48.832261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435651915342532:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:48.861394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435651915342586:2094] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.869443Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435655126587555:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(26265150ns) {Configure:104(380421ns) {ProcessConfigured:135(28213ns) {StartInitialHandshake:157(27974ns) {DropHandshakes:469(721ns) {DropIncomingHandshake:441(133ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:476(379ns) {SwitchToState:178(104ns)}}}}}
2025-11-19T13:58:48.869609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435655126587625:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35891
2025-11-19T13:58:48.869622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435655126587626:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35891
2025-11-19T13:58:48.870281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435653225023394:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.872220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435652728662035:2110] [node 6] ICP01 ready to work
2025-11-19T13:58:48.873269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435652728662036:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49928
2025-11-19T13:58:48.875956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435655126587628:2104] [node 6] ICP01 ready to work
2025-11-19T13:58:48.880106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435654880951483:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43075
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.887471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435653225023397:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54845
2025-11-19T13:58:48.887496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435653225023398:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54845
2025-11-19T13:58:48.887711Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435651915342586:2094] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(26252232ns) {Configure:104(26251371ns) {ProcessConfigured:135(25880515ns) {StartInitialHandshake:157(25879925ns) {DropHandshakes:469(627ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(304ns) PrepareNewSessionHandshake:476(1370ns) {SwitchToState:178(931ns)}}}}}
2025-11-19T13:58:48.887927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435651915342604:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14934
2025-11-19T13:58:48.887942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435651915342605:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14934
2025-11-19T13:58:48.887976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435651915342534:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21945599ns) {Configure:104(21944550ns) {ProcessConfigured:135(21571533ns) {StartInitialHandshake:157(21570892ns) {DropHandshakes:469(1136ns) {DropIncomingHandshake:441(138ns) DropOutgoingHandshake:456(140ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:476(847ns) {SwitchToState:178(376ns)}}}}}
2025-11-19T13:58:48.885756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435655126587632:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35891
2025-11-19T13:58:48.887807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435654880951412:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(26089937ns) {Configure:104(26089154ns) {ProcessConfigured:135(25768779ns) {StartInitialHandshake:157(25768087ns) {DropHandshakes:469(497ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(83ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:476(2072ns) {SwitchToState:178(1426ns)}}}}}
2025-11-19T13:58:48.892459Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435653225023398:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.892509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435653225023319:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.893281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435654398347226:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40740
2025-11-19T13:58:48.893305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435654398347227:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40740
2025-11-19T13:58:48.893369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435654398347228:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40740
2025-11-19T13:58:48.893404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435654398347229:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40740
2025-11-19T13:58:48.893398Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435654880951483:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.893497Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435654880951470:2101] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(41098802ns) {Configure:104(41097645ns) {ProcessConfigured:135(40768599ns) {StartInitialHandshake:157(40767815ns) {DropHandshakes:469(872ns) {DropIncomingHandshake:441(126ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(66ns) PrepareNewSessionHandshake:476(705ns) {SwitchToState:178(202ns)}}}}}
2025-11-19T13:58:48.892681Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435655126587624:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.892691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435655126587624:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.894811Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435654398347228:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.893710Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435655126587632:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.893793Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435655126587619:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.893796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435655126587619:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.895160Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435654398347227:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.895307Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435655126587619:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:48.895607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435655126587616:2106] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.895628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:48.895635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435655126587616:2106] [node 4] ICP22 created new session: [2:7574435655126587635:2106]
2025-11-19T13:58:48.895641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435655126587635:2106] [node 4] ICS09 handshake done sender: [2:7574435655126587619:2109] self: [2:7574435655126587618:0] peer: [4:7574435654880951488:0] socket: 95 qp: -1
2025-11-19T13:58:48.895649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435655126587635:2106] [node 4] ICS10 traffic start
2025-11-19T13:58:48.895659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:48.895831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435655126587628:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19777307ns) {Configure:104(19776160ns) {ProcessConfigured:135(19407690ns) {StartInitialHandshake:157(19407043ns) {DropHandshakes:469(884ns) {DropIncomingHandshake:441(185ns) DropOutgoingHandshake:456(106ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:476(675ns) {SwitchToState:178(294ns)}}}}}
2025-11-19T13:58:48.894716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435654880951490:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43075
2025-11-19T13:58:48.900149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435655126587626:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.900587Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435653225023397:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.900646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435653225023318:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.900860Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435654398347213:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.900873Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435654398347213:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.901608Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435654398347213:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:48.901897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435654398347147:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.901924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:48.901931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435654398347147:2049] [node 3] ICP22 created new session: [1:7574435654398347233:2049]
2025-11-19T13:58:48.901938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435654398347233:2049] [node 3] ICS09 handshake done sender: [1:7574435654398347213:2110] self: [1:7574435654398347212:0] peer: [3:7574435653225023400:0] socket: 86 qp: -1
2025-11-19T13:58:48.901945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435654398347233:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:48.901954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:48.902465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435651915342606:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14934
2025-11-19T13:58:48.902592Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435651915342605:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.902819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435651915342583:2091] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.904473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435654880951411:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.904679Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435654880951483:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.908098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435654880951491:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43075
2025-11-19T13:58:48.908369Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435653225023381:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.908382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435653225023381:2108] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.909265Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435653225023381:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.909703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435654880951492:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43075
2025-11-19T13:58:48.909352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435653225023377:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.909366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:48.909373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435653225023377:2105] [node 6] ICP22 created new session: [3:7574435653225023402:2105]
2025-11-19T13:58:48.909379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435653225023402:2105] [node 6] ICS09 handshake done sender: [3:7574435653225023381:2108] self: [3:7574435653225023380:0] peer: [6:7574435651915342612:0] socket: 92 qp: -1
2025-11-19T13:58:48.909385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435653225023402:2105] [node 6] ICS10 traffic start
2025-11-19T13:58:48.909403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:48.911364Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435654880951492:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.911393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435654880951412:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.909058Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435655126587625:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.911492Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435653225023392:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.911499Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435653225023392:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.907893Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435651915342533:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(48661728ns) {Configure:104(48660300ns) {ProcessConfigured:135(48301611ns) {StartInitialHandshake:157(48300796ns) {DropHandshakes:469(1418ns) {DropIncomingHandshake:441(119ns) DropOutgoingHandshake:456(616ns)} GenerateSessionVirtualId:988(872ns) PrepareNewSessionHandshake:476(957ns) {SwitchToState:178(522ns)}}}}}
2025-11-19T13:58:48.907995Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435651915342604:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.908042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435651915342534:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.908199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435651915342532:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.908355Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435651915342606:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.912479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435654880951494:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43075
2025-11-19T13:58:48.912582Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435653225023392:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:48.912762Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435654880951492:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:48.912820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435654880951412:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.912836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:48.912842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435654880951412:2050] [node 3] ICP22 created new session: [4:7574435654880951495:2050]
2025-11-19T13:58:48.912849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435654880951495:2050] [node 3] ICS09 handshake done sender: [4:7574435654880951492:2118] self: [4:7574435654880951493:0] peer: [3:7574435653225023391:0] socket: 126 qp: -1
2025-11-19T13:58:48.912854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435654880951495:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:48.912884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:48.914512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435653225023390:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.914529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:48.914538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435653225023390:2111] [node 4] ICP22 created new session: [3:7574435653225023406:2111]
2025-11-19T13:58:48.914544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435653225023406:2111] [node 4] ICS09 handshake done sender: [3:7574435653225023392:2113] self: [3:7574435653225023391:0] peer: [4:7574435654880951493:0] socket: 125 qp: -1
2025-11-19T13:58:48.914550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435653225023406:2111] [node 4] ICS10 traffic start
2025-11-19T13:58:48.914558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.915016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435652728662040:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49928
2025-11-19T13:58:48.916626Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435655126587624:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:48.916805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435655126587555:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.916827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:48.916834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435655126587555:2049] [node 3] ICP22 created new session: [2:7574435655126587641:2049]
2025-11-19T13:58:48.916840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435655126587641:2049] [node 3] ICS09 handshake done sender: [2:7574435655126587624:2111] self: [2:7574435655126587623:0] peer: [3:7574435653225023399:0] socket: 101 qp: -1
2025-11-19T13:58:48.916846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435655126587641:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:48.916867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:48.917077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435654398347225:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40740
2025-11-19T13:58:48.917371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435653225023318:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9375) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435653225023397:2115] held: no
2025-11-19T13:58:48.917477Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435652728662040:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.920094Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435654880951411:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13196) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435654880951483:2112] held: no
2025-11-19T13:58:48.920164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435654880951411:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.920178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:48.920186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435654880951411:2049] [node 2] ICP22 created new session: [4:7574435654880951498:2049]
2025-11-19T13:58:48.920194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435654880951498:2049] [node 2] ICS09 handshake done sender: [4:7574435654880951483:2112] self: [4:7574435654880951488:0] peer: [2:7574435655126587618:0] socket: 102 qp: -1
2025-11-19T13:58:48.920203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435654880951498:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:48.920212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.919813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435652728662034:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.920762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435653225023409:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54845
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.924160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435653225023411:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54845
2025-11-19T13:58:48.924330Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435653225023397:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:48.924414Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435653225023398:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.925853Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435654398347225:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.924735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435651915342614:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14934
2025-11-19T13:58:48.924795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435651915342615:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14934
2025-11-19T13:58:48.926029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435651915342583:2091] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23006425ns) {Configure:104(23005344ns) {ProcessConfigured:135(22679898ns) {StartInitialHandshake:157(22679398ns) {DropHandshakes:469(576ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(180ns) PrepareNewSessionHandshake:476(713ns) {SwitchToState:178(52ns)}}}}}
2025-11-19T13:58:48.926049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435651915342583:2091] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.926438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435654398347147:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(24514836ns) {HandleHandshakeStatus:329(24514154ns) {DropIncomingHandshake:441(4466ns) DropOutgoingHandshake:456(258ns) SwitchToState:178(66ns) ProcessPendingSessionEvents:495(24404859ns) {ForwardSessionEventToSession:612(24392944ns) ForwardSessionEventToSession:612(2320ns) ForwardSessionEventToSession:612(820ns) ForwardSessionEventToSession:612(971ns) ForwardSessionEventToSession:612(629ns) ForwardSessionEventToSession:612(1293ns) ForwardSessionEventToSession:612(657ns) ForwardSessionEventToSession:612(742ns) ForwardSessionEventToSession:612(783ns) ForwardSessionEventToSession:612(964ns)}}}
2025-11-19T13:58:48.926621Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435654398347215:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.926629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435654398347215:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.924534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435652728661966:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(57199742ns) {Configure:104(57198997ns) {ProcessConfigured:135(56874912ns) {StartInitialHandshake:157(56874491ns) {DropHandshakes:469(571ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(121ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(1237ns) {SwitchToState:178(124ns)}}}}}
2025-11-19T13:58:48.924579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435652728661966:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.924477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435653225023318:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.924499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:48.924507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435653225023318:2048] [node 1] ICP22 created new session: [3:7574435653225023412:2048]
2025-11-19T13:58:48.924513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435653225023412:2048] [node 1] ICS09 handshake done sender: [3:7574435653225023397:2115] self: [3:7574435653225023400:0] peer: [1:7574435654398347212:0] socket: 90 qp: -1
2025-11-19T13:58:48.924521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435653225023412:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:48.924531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:48.924630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435653225023319:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.924639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:48.924643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435653225023319:2049] [node 2] ICP22 created new session: [3:7574435653225023414:2049]
2025-11-19T13:58:48.924648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435653225023414:2049] [node 2] ICS09 handshake done sender: [3:7574435653225023398:2116] self: [3:7574435653225023399:0] peer: [2:7574435655126587623:0] socket: 108 qp: -1
2025-11-19T13:58:48.924652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435653225023414:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:48.924657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:48.926855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435653225023410:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54845
2025-11-19T13:58:48.927098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435654398347229:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.929560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435655126587643:2105] [node 5] ICP01 ready to work
2025-11-19T13:58:48.929386Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435652728662036:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.931906Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435651915342605:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:48.932897Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435655126587617:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.932940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435655126587554:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.933048Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435654398347217:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.933058Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435654398347217:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.933981Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435651915342614:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.934010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435651915342534:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.934025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:48.934032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435651915342534:2050] [node 3] ICP22 created new session: [6:7574435651915342618:2050]
2025-11-19T13:58:48.934037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435651915342618:2050] [node 3] ICS09 handshake done sender: [6:7574435651915342605:2113] self: [6:7574435651915342612:0] peer: [3:7574435653225023380:0] socket: 110 qp: -1
2025-11-19T13:58:48.934044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435651915342618:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:48.934080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.934540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435652728662044:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49928
2025-11-19T13:58:48.934698Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435654398347217:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.934920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435654398347146:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.934939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:48.934944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435654398347146:2048] [node 2] ICP22 created new session: [1:7574435654398347239:2048]
2025-11-19T13:58:48.934950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435654398347239:2048] [node 2] ICS09 handshake done sender: [1:7574435654398347217:2112] self: [1:7574435654398347216:0] peer: [2:7574435655126587648:0] socket: 88 qp: -1
2025-11-19T13:58:48.934955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435654398347239:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:48.934962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.936046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435652728662034:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16152633ns) {Configure:104(16151669ns) {ProcessConfigured:135(15788834ns) {StartInitialHandshake:157(15787778ns) {DropHandshakes:469(706ns) {DropIncomingHandshake:441(146ns) DropOutgoingHandshake:456(106ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(1074ns) {SwitchToState:178(80ns)}}}}}
2025-11-19T13:58:48.934800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435655126587649:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35891
2025-11-19T13:58:48.936376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435651915342533:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.935282Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435655126587617:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:48.935354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435655126587554:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.935362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:48.935367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435655126587554:2048] [node 1] ICP22 created new session: [2:7574435655126587650:2048]
2025-11-19T13:58:48.935373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435655126587650:2048] [node 1] ICS09 handshake done sender: [2:7574435655126587617:2108] self: [2:7574435655126587648:0] peer: [1:7574435654398347216:0] socket: 91 qp: -1
2025-11-19T13:58:48.935377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435655126587650:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:48.935383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:48.936094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435652728662034:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.938158Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435654880951464:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.938177Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435654880951464:2099] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.938889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435651915342620:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14934
2025-11-19T13:58:48.938562Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435652728662038:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.938574Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435652728662038:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.938774Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435653225023396:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.938788Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435653225023396:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.939622Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435651915342620:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.939652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435651915342586:2094] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.939795Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435655126587637:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.939806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435655126587637:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.940469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435655126587652:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35891
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.942344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435651915342624:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14934
2025-11-19T13:58:48.942360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435651915342625:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14934
2025-11-19T13:58:48.942461Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435654398347215:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:48.942654Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435651915342604:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:48.942712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435651915342532:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.942728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:48.942734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435651915342532:2048] [node 1] ICP22 created new session: [6:7574435651915342626:2048]
2025-11-19T13:58:48.942740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435651915342626:2048] [node 1] ICS09 handshake done sender: [6:7574435651915342604:2112] self: [6:7574435651915342613:0] peer: [1:7574435654398347214:0] socket: 109 qp: -1
2025-11-19T13:58:48.942746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435651915342626:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:48.942752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:48.943014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435654398347209:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.943026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:48.943031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435654398347209:2107] [node 6] ICP22 created new session: [1:7574435654398347243:2107]
2025-11-19T13:58:48.943036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435654398347243:2107] [node 6] ICS09 handshake done sender: [1:7574435654398347215:2111] self: [1:7574435654398347214:0] peer: [6:7574435651915342613:0] socket: 87 qp: -1
2025-11-19T13:58:48.943041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435654398347243:2107] [node 6] ICS10 traffic start
2025-11-19T13:58:48.943047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:48.943832Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435654880951489:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.943848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435654880951489:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.944739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435654880951411:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(24553138ns) {HandleHandshakeStatus:329(24552707ns) {DropIncomingHandshake:441(309ns) DropOutgoingHandshake:456(48ns) SwitchToState:178(253ns) ProcessPendingSessionEvents:495(15597457ns) {ForwardSessionEventToSession:612(15593919ns) ForwardSessionEventToSession:612(1270ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.945078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435654880951501:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43075
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.947941Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435655126587637:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:48.948132Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435654880951489:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:48.948401Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435654880951501:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.948416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435654880951470:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.948427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:48.948433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435654880951470:2101] [node 6] ICP22 created new session: [4:7574435654880951502:2101]
2025-11-19T13:58:48.948438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435654880951502:2101] [node 6] ICS09 handshake done sender: [4:7574435654880951489:2115] self: [4:7574435654880951471:0] peer: [6:7574435651915342623:0] socket: 129 qp: -1
2025-11-19T13:58:48.948443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435654880951502:2101] [node 6] ICS10 traffic start
2025-11-19T13:58:48.948450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:48.948610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435655126587628:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.948623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:48.948628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435655126587628:2104] [node 6] ICP22 created new session: [2:7574435655126587654:2104]
2025-11-19T13:58:48.948634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435655126587654:2104] [node 6] ICS09 handshake done sender: [2:7574435655126587637:2117] self: [2:7574435655126587631:0] peer: [6:7574435651915342621:0] socket: 118 qp: -1
2025-11-19T13:58:48.948639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435655126587654:2104] [node 6] ICS10 traffic start
2025-11-19T13:58:48.948645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:48.952576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435654880951410:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.954503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435651915342629:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14934
2025-11-19T13:58:48.954748Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435654880951464:2099] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:48.954821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435654880951462:2096] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.954836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:48.954842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435654880951462:2096] [node 5] ICP22 created new session: [4:7574435654880951506:2096]
2025-11-19T13:58:48.954848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435654880951506:2096] [node 5] ICS09 handshake done sender: [4:7574435654880951464:2099] self: [4:7574435654880951463:0] peer: [5:7574435652728662046:0] socket: 84 qp: -1
2025-11-19T13:58:48.954854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435654880951506:2096] [node 5] ICS10 traffic start
2025-11-19T13:58:48.954863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:48.955828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435655126587555:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(38986768ns) {HandleHandshakeStatus:329(38985432ns) {DropIncomingHandshake:441(6028ns) DropOutgoingHandshake:456(166ns) SwitchToState:178(77ns) ProcessPendingSessionEvents:495(29507430ns) {ForwardSessionEventToSession:612(29490831ns) ForwardSessionEventToSession:612(2094ns) ForwardSessionEventToSession:612(880ns) ForwardSessionEventToSession:612(870ns) ForwardSessionEventToSession:612(812ns) ForwardSessionEventToSession:612(1412ns) ForwardSessionEventToSession:612(716ns) ForwardSessionEventToSession:612(929ns) ForwardSessionEventToSession:612(1008ns) ForwardSessionEventToSession:612(1237ns)}}}
2025-11-19T13:58:48.956349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435652728662044:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.956396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435652728661965:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.957269Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435655126587645:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.957280Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435655126587645:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.957478Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435652728662038:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:48.957561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435652728662035:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.957603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:48.957609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435652728662035:2110] [node 6] ICP22 created new session: [5:7574435652728662049:2110]
2025-11-19T13:58:48.957614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435652728662049:2110] [node 6] ICS09 handshake done sender: [5:7574435652728662038:2113] self: [5:7574435652728662037:0] peer: [6:7574435651915342617:0] socket: 104 qp: -1
2025-11-19T13:58:48.957619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435652728662049:2110] [node 6] ICS10 traffic start
2025-11-19T13:58:48.957640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:48.959987Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435653225023396:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.960319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435653225023394:2107] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.960334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:48.960341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435653225023394:2107] [node 5] ICP22 created new session: [3:7574435653225023417:2107]
2025-11-19T13:58:48.960348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435653225023417:2107] [node 5] ICS09 handshake done sender: [3:7574435653225023396:2114] self: [3:7574435653225023395:0] peer: [5:7574435652728662043:0] socket: 130 qp: -1
2025-11-19T13:58:48.960354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435653225023417:2107] [node 5] ICS10 traffic start
2025-11-19T13:58:48.960366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:48.960494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435652728662051:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49928
2025-11-19T13:58:48.960538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435652728662052:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49928
2025-11-19T13:58:48.960549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435652728662053:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49928
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.960724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435652728662054:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49928
2025-11-19T13:58:48.961301Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435654398347219:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.961316Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435654398347219:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.961728Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435651915342620:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:48.961960Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435651915342614:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:48.961971Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435652728662051:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.962005Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435652728662040:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:48.962104Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435652728662036:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.962198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435653225023419:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54845
2025-11-19T13:58:48.962232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435651915342586:2094] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.962245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:48.962249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435651915342586:2094] [node 4] ICP22 created new session: [6:7574435651915342631:2094]
2025-11-19T13:58:48.962253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435651915342631:2094] [node 4] ICS09 handshake done sender: [6:7574435651915342620:2120] self: [6:7574435651915342623:0] peer: [4:7574435654880951471:0] socket: 135 qp: -1
2025-11-19T13:58:48.962258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435651915342631:2094] [node 4] ICS10 traffic start
2025-11-19T13:58:48.962264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:48.962398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435651915342533:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.962404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:48.962409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435651915342533:2049] [node 2] ICP22 created new session: [6:7574435651915342633:2049]
2025-11-19T13:58:48.962413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435651915342633:2049] [node 2] ICS09 handshake done sender: [6:7574435651915342614:2117] self: [6:7574435651915342621:0] peer: [2:7574435655126587631:0] socket: 120 qp: -1
2025-11-19T13:58:48.962416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435651915342633:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:48.962421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:48.963191Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435655126587652:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.963290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435655126587628:2104] [node 6] ICP18 handshake (actor [2:7574435655126587652:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.963727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435651915342630:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14934
2025-11-19T13:58:48.963929Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435651915342606:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:48.963986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435651915342583:2091] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.963997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:48.964002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435651915342583:2091] [node 5] ICP22 created new session: [6:7574435651915342635:2091]
2025-11-19T13:58:48.964006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435651915342635:2091] [node 5] ICS09 handshake done sender: [6:7574435651915342606:2114] self: [6:7574435651915342617:0] peer: [5:7574435652728662037:0] socket: 111 qp: -1
2025-11-19T13:58:48.964011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435651915342635:2091] [node 5] ICS10 traffic start
2025-11-19T13:58:48.964016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.966401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435652728661964:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:48.966442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435652728661966:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.966453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:48.966458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435652728661966:2050] [node 3] ICP22 created new session: [5:7574435652728662057:2050]
2025-11-19T13:58:48.966463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435652728662057:2050] [node 3] ICS09 handshake done sender: [5:7574435652728662040:2114] self: [5:7574435652728662043:0] peer: [3:7574435653225023395:0] socket: 131 qp: -1
2025-11-19T13:58:48.966468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435652728662057:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:48.966477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:48.967579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435652728662059:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49928
2025-11-19T13:58:48.967878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435652728662034:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.967900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:48.967907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435652728662034:2108] [node 4] ICP22 created new session: [5:7574435652728662061:2108]
2025-11-19T13:58:48.967912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435652728662061:2108] [node 4] ICS09 handshake done sender: [5:7574435652728662036:2112] self: [5:7574435652728662046:0] peer: [4:7574435654880951463:0] socket: 103 qp: -1
2025-11-19T13:58:48.967918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435652728662061:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:48.967940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:48.967876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435654880951508:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43075
2025-11-19T13:58:48.969971Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435654398347224:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:48.969983Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435654398347224:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:48.971278Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435654880951462:2096] [node 5] ICP35 event processing took too much time PendingConnection:278(16424216ns) {HandleHandshakeStatus:329(16423344ns) {DropIncomingHandshake:441(239ns) DropOutgoingHandshake:456(220ns) SwitchToState:178(176ns) ProcessPendingSessionEvents:495(9768414ns) {ForwardSessionEventToSession:612(9759020ns) ForwardSessionEventToSession:612(1273ns) ForwardSessionEventToSession:612(877ns) ForwardSessionEventToSession:612(654ns) ForwardSessionEventToSession:612(510ns) ForwardSessionEventToSession:612(847ns) ForwardSessionEventToSession:612(495ns) ForwardSessionEventToSession:612(576ns) ForwardSessionEventToSession:612(693ns) ForwardSessionEventToSession:612(939ns)}}}
2025-11-19T13:58:48.971549Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435654398347219:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:48.971585Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435655126587645:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:48.971801Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435654398347224:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:48.971824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435654398347211:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.971840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:48.971848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435654398347211:2109] [node 4] ICP22 created new session: [1:7574435654398347247:2109]
2025-11-19T13:58:48.971853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435654398347247:2109] [node 4] ICS09 handshake done sender: [1:7574435654398347219:2113] self: [1:7574435654398347218:0] peer: [4:7574435654880951505:0] socket: 136 qp: -1
2025-11-19T13:58:48.971858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435654398347247:2109] [node 4] ICS10 traffic start
2025-11-19T13:58:48.971865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:48.972187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435655126587643:2105] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.972206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:48.972212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435655126587643:2105] [node 5] ICP22 created new session: [2:7574435655126587657:2105]
2025-11-19T13:58:48.972217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435655126587657:2105] [node 5] ICS09 handshake done sender: [2:7574435655126587645:2118] self: [2:7574435655126587644:0] peer: [5:7574435652728662047:0] socket: 137 qp: -1
2025-11-19T13:58:48.972222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435655126587657:2105] [node 5] ICS10 traffic start
2025-11-19T13:58:48.972230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:48.972516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435654398347208:2105] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:48.972532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:48.972538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435654398347208:2105] [node 5] ICP22 created new session: [1:7574435654398347249:2105]
2025-11-19T13:58:48.972544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435654398347249:2105] [node 5] ICS09 handshake done sender: [1:7574435654398347224:2114] self: [1:7574435654398347223:0] peer: [5:7574435652728662056:0] socket: 114 qp: -1
2025-11-19T13:58:48.972548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435654398347249:2105] [node 5] ICS10 traffic start
2025-11-19T13:58:48.972553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:48.972767Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435654880951410:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9375) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435654880951501:2120] held: no
2025-11-19T13:58:48.972877Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435652728661965:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13196) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435652728662044:2116] held: no
2025-11-19T13:58:48.973211Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435652728662044:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:48.973275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435652728661965:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.973284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:48.973291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435652728661965:2049] [node 2] ICP22 created new session: [5:7574435652728662063:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.973356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435654880951470:2101] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:48.973395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435654880951509:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43075
2025-11-19T13:58:48.973296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435652728662063:2049] [node 2] ICS09 handshake done sender: [5:7574435652728662044:2116] self: [5:7574435652728662047:0] peer: [2:7574435655126587644:0] socket: 106 qp: -1
2025-11-19T13:58:48.973307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435652728662063:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:48.973313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:48.974321Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435654880951501:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:48.981619Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435652728661964:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9375) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435652728662051:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:48.984283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435654880951410:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.984304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:48.984313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435654880951410:2048] [node 1] ICP22 created new session: [4:7574435654880951510:2048]
2025-11-19T13:58:48.984320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435654880951510:2048] [node 1] ICS09 handshake done sender: [4:7574435654880951501:2120] self: [4:7574435654880951505:0] peer: [1:7574435654398347218:0] socket: 97 qp: -1
2025-11-19T13:58:48.984327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435654880951510:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:48.984340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:48.985221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435652728662065:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49928
2025-11-19T13:58:48.985316Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435652728662035:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:48.985442Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435652728662051:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:48.985518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435652728661964:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:48.985533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:48.985540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435652728661964:2048] [node 1] ICP22 created new session: [5:7574435652728662066:2048]
2025-11-19T13:58:48.985545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435652728662066:2048] [node 1] ICS09 handshake done sender: [5:7574435652728662051:2118] self: [5:7574435652728662056:0] peer: [1:7574435654398347223:0] socket: 139 qp: -1
2025-11-19T13:58:48.985550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435652728662066:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:48.985558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:48.993811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:48.993901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435655126587650:2048] [node 1] ICS01 socket: 91 reason# EndOfStream
2025-11-19T13:58:48.993911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435655126587554:2048] [node 1] ICP30 unregister session Session# [2:7574435655126587650:2048] VirtualId# [2:7574435655126587648:0]
2025-11-19T13:58:48.993918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435655126587650:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.993952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:48.993973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:48.994361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:48.994428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435653225023412:2048] [node 1] ICS01 socket: 90 reason# EndOfStream
2025-11-19T13:58:48.994434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435653225023318:2048] [node 1] ICP30 unregister session Session# [3:7574435653225023412:2048] VirtualId# [3:7574435653225023400:0]
2025-11-19T13:58:48.994439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435653225023412:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:48.994465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:48.994480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:49.004645Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:49.004753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435651915342626:2048] [node 1] ICS01 socket: 109 reason# EndOfStream
2025-11-19T13:58:49.004762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435651915342532:2048] [node 1] ICP30 unregister session Session# [6:7574435651915342626:2048] VirtualId# [6:7574435651915342613:0]
2025-11-19T13:58:49.004769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435651915342626:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.004816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:49.004840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:49.004875Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:49.004934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435654880951510:2048] [node 1] ICS01 socket: 97 reason# EndOfStream
2025-11-19T13:58:49.004938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435654880951410:2048] [node 1] ICP30 unregister session Session# [4:7574435654880951510:2048] VirtualId# [4:7574435654880951505:0]
2025-11-19T13:58:49.004942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435654880951510:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.004968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:49.004988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:49.005130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:49.005186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435652728662066:2048] [node 1] ICS01 socket: 139 reason# EndOfStream
2025-11-19T13:58:49.005190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435652728661964:2048] [node 1] ICP30 unregister session Session# [5:7574435652728662066:2048] VirtualId# [5:7574435652728662056:0]
2025-11-19T13:58:49.005194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435652728662066:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.005220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:49.005239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:49.010475Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:49.010565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435653225023414:2049] [node 2] ICS01 socket: 108 reason# EndOfStream
2025-11-19T13:58:49.010573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435653225023319:2049] [node 2] ICP30 unregister session Session# [3:7574435653225023414:2049] VirtualId# [3:7574435653225023399:0]
2025-11-19T13:58:49.010579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435653225023414:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.010620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:49.010639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:49.023671Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:49.023798Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:49.023896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435654880951498:2049] [node 2] ICS07 socket disconnect 102 reason# EndOfStream
2025-11-19T13:58:49.023904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435654880951498:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.023960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:49.023987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435654880951498:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:49.024622Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:49.024692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435651915342633:2049] [node 2] ICS01 socket: 120 reason# EndOfStream
2025-11-19T13:58:49.024698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435651915342533:2049] [node 2] ICP30 unregister session Session# [6:7574435651915342633:2049] VirtualId# [6:7574435651915342621:0]
2025-11-19T13:58:49.024705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435651915342633:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.024738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:49.024763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:49.026709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435652728662063:2049] [node 2] ICS01 socket: 106 reason# EndOfStream
2025-11-19T13:58:49.026722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435652728661965:2049] [node 2] ICP30 unregister session Session# [5:7574435652728662063:2049] VirtualId# [5:7574435652728662047:0]
2025-11-19T13:58:49.026728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435652728662063:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.026773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:49.026799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:49.027170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435654880951411:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13196) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:49.027185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435654880951498:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:49.027189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435654880951411:2049] [node 2] ICP30 unregister session Session# [4:7574435654880951498:2049] VirtualId# [4:7574435654880951488:0]
2025-11-19T13:58:49.027201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:49.027206Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435654880951411:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:49.027209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:49.033926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:49.034013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435651915342618:2050] [node 3] ICS01 socket: 110 reason# EndOfStream
2025-11-19T13:58:49.034024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435651915342534:2050] [node 3] ICP30 unregister session Session# [6:7574435651915342618:2050] VirtualId# [6:7574435651915342612:0]
2025-11-19T13:58:49.034030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435651915342618:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.034086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:49.034109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:49.034143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:49.034201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435652728662057:2050] [node 3] ICS07 socket disconnect 131 reason# EndOfStream
2025-11-19T13:58:49.034206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435652728662057:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.034228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:49.034247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435652728662057:2050] [node 3] ICS15 start handshake
2025-11-19T13:58:49.035067Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435652728661966:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15830) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:49.035077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435652728662057:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:49.035082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435652728661966:2050] [node 3] ICP30 unregister session Session# [5:7574435652728662057:2050] VirtualId# [5:7574435652728662043:0]
2025-11-19T13:58:49.035094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:49.035099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435652728661966:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15830) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:49.035102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15830) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:49.034456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T13:58:49.034520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435654880951495:2050] [node 3] ICS01 socket: 126 reason# EndOfStream
2025-11-19T13:58:49.034525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435654880951412:2050] [node 3] ICP30 unregister session Session# [4:7574435654880951495:2050] VirtualId# [4:7574435654880951493:0]
2025-11-19T13:58:49.034530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435654880951495:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.034555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:49.034574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:49.035903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435654880951411:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:58:49.047878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435652728661966:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:58:49.072254Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:49.072351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435651915342631:2094] [node 4] ICS01 socket: 135 reason# EndOfStream
2025-11-19T13:58:49.072363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435651915342586:2094] [node 4] ICP30 unregister session Session# [6:7574435651915342631:2094] VirtualId# [6:7574435651915342623:0]
2025-11-19T13:58:49.072371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435651915342631:2094] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.072435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:49.072461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:49.180662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435655783558831:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:49.180673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435655783558832:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:49.207590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435659003417660:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:49.207603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435659003417661:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:49.225741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435655930712563:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:49.225753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435655930712564:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:49.251681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435657990893489:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:49.251697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435657990893490:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:49.251702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435657990893491:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:49.280320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435656861029385:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:49.280320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435656861029383:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:49.280426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435656861029384:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:49.355571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435655942628144:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:49.355587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435655942628145:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:49.355592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435655942628146:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.401675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435655783558891:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48734
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.402354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435655930712623:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48683
2025-11-19T13:58:49.401327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435657990893512:2069] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.404104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435656861029404:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29038
2025-11-19T13:58:49.404284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435656861029409:2068] [node 6] ICP01 ready to work
2025-11-19T13:58:49.404171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435655942628163:2064] [node 5] ICP01 ready to work
2025-11-19T13:58:49.404399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435659003417721:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:49.408319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435656861029417:2075] [node 4] ICP01 ready to work
2025-11-19T13:58:49.408152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435655783558894:2108] [node 6] ICP01 ready to work
2025-11-19T13:58:49.408258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435655783558895:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:49.409076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435655783558899:2110] [node 4] ICP01 ready to work
2025-11-19T13:58:49.413221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435655930712628:2109] [node 4] ICP01 ready to work
2025-11-19T13:58:49.414301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435655930712633:2111] [node 5] ICP01 ready to work
2025-11-19T13:58:49.415260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435657990893542:2090] [node 6] ICP01 ready to work
2025-11-19T13:58:49.420032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435657990893489:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16025562ns) {Configure:104(16024481ns) {ProcessConfigured:135(15540034ns) {StartInitialHandshake:157(15539425ns) {DropHandshakes:469(1244ns) {DropIncomingHandshake:441(170ns) DropOutgoingHandshake:456(494ns)} GenerateSessionVirtualId:988(335ns) PrepareNewSessionHandshake:476(826ns) {SwitchToState:178(318ns)}}}}}
2025-11-19T13:58:49.428371Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435656861029404:2066] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.428426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435656861029417:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:49.430019Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435655783558899:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(20868122ns) {Configure:104(20867168ns) {ProcessConfigured:135(20538921ns) {StartInitialHandshake:157(20538221ns) {DropHandshakes:469(664ns) {DropIncomingHandshake:441(159ns) DropOutgoingHandshake:456(88ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(1626ns) {SwitchToState:178(105ns)}}}}}
2025-11-19T13:58:49.431980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435655930712564:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(22398732ns) {Configure:104(22397588ns) {ProcessConfigured:135(22013681ns) {StartInitialHandshake:157(22013173ns) {DropHandshakes:469(508ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(41ns)} GenerateSessionVirtualId:988(667ns) PrepareNewSessionHandshake:476(1374ns) {SwitchToState:178(420ns)}}}}}
2025-11-19T13:58:49.432029Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435657990893516:2071] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.432047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435657990893516:2071] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.433081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435655783558895:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(24727352ns) {Configure:104(24709063ns) {ProcessConfigured:135(23001109ns) {StartInitialHandshake:157(22999733ns) {DropHandshakes:469(401ns) {DropIncomingHandshake:441(14ns) DropOutgoingHandshake:456(20ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:476(653ns) {SwitchToState:178(337ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.433613Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435657990893516:2071] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:49.433732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435657990893512:2069] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.433749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:49.433758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435657990893512:2069] [node 5] ICP22 created new session: [4:7574435657990893568:2069]
2025-11-19T13:58:49.433764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435657990893568:2069] [node 5] ICS09 handshake done sender: [4:7574435657990893516:2071] self: [4:7574435657990893515:0] peer: [5:7574435656861029460:0] socket: 93 qp: -1
2025-11-19T13:58:49.433771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435657990893568:2069] [node 5] ICS10 traffic start
2025-11-19T13:58:49.433780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:49.433388Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435655942628144:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23533511ns) {Configure:104(23533057ns) {ProcessConfigured:135(23278691ns) {StartInitialHandshake:157(23278264ns) {DropHandshakes:469(389ns) {DropIncomingHandshake:441(82ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:476(885ns) {SwitchToState:178(649ns)}}}}}
2025-11-19T13:58:49.433254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435655783558907:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48734
2025-11-19T13:58:49.436071Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435655783558894:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27756283ns) {Configure:104(27754864ns) {ProcessConfigured:135(25789336ns) {StartInitialHandshake:157(25788547ns) {DropHandshakes:469(431ns) {DropIncomingHandshake:441(29ns) DropOutgoingHandshake:456(14ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(755ns) {SwitchToState:178(413ns)}}}}}
2025-11-19T13:58:49.436224Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435655930712623:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.436562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435656861029461:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29038
2025-11-19T13:58:49.436581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435656861029462:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29038
2025-11-19T13:58:49.437343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435656861029463:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29038
2025-11-19T13:58:49.436675Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435659003417660:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(28847800ns) {Configure:104(24740081ns) {ProcessConfigured:135(24411215ns) {StartInitialHandshake:157(24411050ns) {DropHandshakes:469(473ns) {DropIncomingHandshake:441(103ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(367ns) PrepareNewSessionHandshake:476(1016ns) {SwitchToState:178(796ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.437583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435655783558911:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48734
2025-11-19T13:58:49.437724Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435656861029404:2066] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:49.437898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435656861029417:2075] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.437920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:49.437927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435656861029417:2075] [node 4] ICP22 created new session: [5:7574435656861029464:2075]
2025-11-19T13:58:49.437935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435656861029464:2075] [node 4] ICS09 handshake done sender: [5:7574435656861029404:2066] self: [5:7574435656861029460:0] peer: [4:7574435657990893515:0] socket: 95 qp: -1
2025-11-19T13:58:49.437942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435656861029464:2075] [node 4] ICS10 traffic start
2025-11-19T13:58:49.437976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.440153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655942628214:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6692
2025-11-19T13:58:49.440047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435659003417734:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65021
2025-11-19T13:58:49.440071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435659003417735:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65021
2025-11-19T13:58:49.440086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435659003417736:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65021
2025-11-19T13:58:49.446341Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435655783558891:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.447894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435659003417733:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65021
2025-11-19T13:58:49.448034Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435659003417736:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.448241Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435659003417733:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.447955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435655930712626:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.452429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435655930712641:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48683
2025-11-19T13:58:49.452454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435655930712642:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48683
2025-11-19T13:58:49.452464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435655930712643:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48683
2025-11-19T13:58:49.452475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435655930712644:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48683
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.453902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435657990893571:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6268
2025-11-19T13:58:49.454117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435657990893572:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6268
2025-11-19T13:58:49.454131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435657990893573:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6268
2025-11-19T13:58:49.454149Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435657990893512:2069] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:49.453587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435655930712633:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(38956124ns) {Configure:104(38955640ns) {ProcessConfigured:135(38764573ns) {StartInitialHandshake:157(38764039ns) {DropHandshakes:469(175ns) {DropIncomingHandshake:441(21ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(1285ns) {SwitchToState:178(317ns)}}}}}
2025-11-19T13:58:49.455711Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435655930712644:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.456150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435655942628169:2069] [node 4] ICP01 ready to work
2025-11-19T13:58:49.456247Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435655942628214:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.456191Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435659003417721:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(51697761ns) {Configure:104(51696280ns) {ProcessConfigured:135(51318724ns) {StartInitialHandshake:157(51318143ns) {DropHandshakes:469(402ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:476(1317ns) {SwitchToState:178(737ns)}}}}}
2025-11-19T13:58:49.468041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435659003417726:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:49.472859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435655930712626:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(24792671ns) {Configure:104(24791525ns) {ProcessConfigured:135(16399098ns) {StartInitialHandshake:157(16398567ns) {DropHandshakes:469(970ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(193ns)} GenerateSessionVirtualId:988(365ns) PrepareNewSessionHandshake:476(1810ns) {SwitchToState:178(617ns)}}}}}
2025-11-19T13:58:49.473117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435659003417727:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.473606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435657990893574:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6268
2025-11-19T13:58:49.474388Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435659003417735:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.474485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435655783558911:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.479735Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435656861029461:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.482954Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435659003417734:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.485369Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435655930712641:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.485417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435655930712563:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:49.486742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435655783558831:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(58816570ns) {Configure:104(58815593ns) {ProcessConfigured:135(58478862ns) {StartInitialHandshake:157(58477765ns) {DropHandshakes:469(458ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(47ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:476(2196ns) {SwitchToState:178(1129ns)}}}}}
2025-11-19T13:58:49.486822Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435657990893572:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.486867Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435657990893574:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.486910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435657990893491:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:49.486922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435657990893490:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:49.487494Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435657990893573:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.487524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435657990893489:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:49.487006Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435655930712643:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.487055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435655930712564:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:49.489290Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435655930712630:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.489298Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435655930712630:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.490691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435659003417727:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17502432ns) {Configure:104(17501502ns) {ProcessConfigured:135(17186818ns) {StartInitialHandshake:157(17185851ns) {DropHandshakes:469(1291ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(344ns)} GenerateSessionVirtualId:988(375ns) PrepareNewSessionHandshake:476(674ns) {SwitchToState:178(507ns)}}}}}
2025-11-19T13:58:49.491462Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435655783558898:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.491479Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435655783558898:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.491552Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435655783558905:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.491558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435655783558905:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.491280Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435659003417730:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.491301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435659003417730:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.491518Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435659003417737:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.491524Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435659003417737:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.491870Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435655783558907:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.493120Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435655930712630:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:49.493176Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435655930712642:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.493450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435655930712628:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.493473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:49.493480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435655930712628:2109] [node 4] ICP22 created new session: [3:7574435655930712653:2109]
2025-11-19T13:58:49.493488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435655930712653:2109] [node 4] ICS09 handshake done sender: [3:7574435655930712630:2110] self: [3:7574435655930712629:0] peer: [4:7574435657990893575:0] socket: 105 qp: -1
2025-11-19T13:58:49.493496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435655930712653:2109] [node 4] ICS10 traffic start
2025-11-19T13:58:49.493516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:49.491292Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435655942628169:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(35065344ns) {Configure:104(35064582ns) {ProcessConfigured:135(34297473ns) {StartInitialHandshake:157(34297183ns) {DropHandshakes:469(628ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(116ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:476(631ns) {SwitchToState:178(266ns)}}}}}
2025-11-19T13:58:49.491339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435655942628169:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.493981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435656861029470:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29038
2025-11-19T13:58:49.494214Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435659003417730:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:49.494466Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435655783558905:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:49.494358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435659003417661:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.494377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:49.494382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435659003417661:2049] [node 3] ICP22 created new session: [2:7574435659003417748:2049]
2025-11-19T13:58:49.494387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435659003417748:2049] [node 3] ICS09 handshake done sender: [2:7574435659003417730:2111] self: [2:7574435659003417729:0] peer: [3:7574435655930712650:0] socket: 119 qp: -1
2025-11-19T13:58:49.494393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435659003417748:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:49.494409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:49.494881Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435656861029463:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.494963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435656861029383:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:49.496033Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435657990893544:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.496051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435657990893544:2093] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.496391Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435655783558898:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:49.496604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435655783558832:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.496634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:49.496642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435655783558832:2049] [node 3] ICP22 created new session: [1:7574435655783558918:2049]
2025-11-19T13:58:49.496650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435655783558918:2049] [node 3] ICS09 handshake done sender: [1:7574435655783558898:2111] self: [1:7574435655783558897:0] peer: [3:7574435655930712649:0] socket: 98 qp: -1
2025-11-19T13:58:49.496659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435655783558918:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:49.496668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:49.498677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435655783558899:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.498694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:49.498702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435655783558899:2110] [node 4] ICP22 created new session: [1:7574435655783558920:2110]
2025-11-19T13:58:49.498709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435655783558920:2110] [node 4] ICS09 handshake done sender: [1:7574435655783558905:2112] self: [1:7574435655783558902:0] peer: [4:7574435657990893577:0] socket: 113 qp: -1
2025-11-19T13:58:49.498716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435655783558920:2110] [node 4] ICS10 traffic start
2025-11-19T13:58:49.498736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:49.498255Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435655930712564:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65021) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435655930712643:2116] held: no
2025-11-19T13:58:49.498593Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435659003417737:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:49.499075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435659003417721:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.499089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:49.499095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435659003417721:2107] [node 4] ICP22 created new session: [2:7574435659003417752:2107]
2025-11-19T13:58:49.499100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435659003417752:2107] [node 4] ICS09 handshake done sender: [2:7574435659003417737:2116] self: [2:7574435659003417722:0] peer: [4:7574435657990893576:0] socket: 129 qp: -1
2025-11-19T13:58:49.499105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435659003417752:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:49.499121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:49.500111Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435655783558906:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.500125Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435655783558906:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.500361Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435656861029470:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.500394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435656861029385:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.502317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435659003417755:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65021
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.504023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435655930712656:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48683
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.504340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435655930712657:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48683
2025-11-19T13:58:49.504557Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435655930712641:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.504678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435657990893491:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11198) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435657990893572:2116] held: no
2025-11-19T13:58:49.504773Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435657990893544:2093] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:49.506908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435657990893579:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6268
2025-11-19T13:58:49.508132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435657990893580:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6268
2025-11-19T13:58:49.504795Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435655930712645:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.504802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435655930712645:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.508400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435657990893581:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6268
2025-11-19T13:58:49.509257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435657990893490:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65021) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435657990893574:2118] held: no
2025-11-19T13:58:49.509279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435657990893542:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.509290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:49.509294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435657990893542:2090] [node 6] ICP22 created new session: [4:7574435657990893582:2090]
2025-11-19T13:58:49.509300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435657990893582:2090] [node 6] ICS09 handshake done sender: [4:7574435657990893544:2093] self: [4:7574435657990893543:0] peer: [6:7574435655942628223:0] socket: 106 qp: -1
2025-11-19T13:58:49.509308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435657990893582:2090] [node 6] ICS10 traffic start
2025-11-19T13:58:49.509326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:49.509455Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435657990893573:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:49.510316Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435655930712645:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:49.510441Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435657990893572:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:49.510563Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435655930712563:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24254) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435655930712641:2114] held: no
2025-11-19T13:58:49.510583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435655930712563:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.510592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:49.510597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435655930712563:2048] [node 1] ICP22 created new session: [3:7574435655930712659:2048]
2025-11-19T13:58:49.510601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435655930712659:2048] [node 1] ICS09 handshake done sender: [3:7574435655930712641:2114] self: [3:7574435655930712649:0] peer: [1:7574435655783558897:0] socket: 100 qp: -1
2025-11-19T13:58:49.510606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435655930712659:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:49.510620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:49.510716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435655930712633:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.510727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:49.510730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435655930712633:2111] [node 5] ICP22 created new session: [3:7574435655930712661:2111]
2025-11-19T13:58:49.510733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435655930712661:2111] [node 5] ICS09 handshake done sender: [3:7574435655930712645:2118] self: [3:7574435655930712636:0] peer: [5:7574435656861029472:0] socket: 133 qp: -1
2025-11-19T13:58:49.510736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435655930712661:2111] [node 5] ICS10 traffic start
2025-11-19T13:58:49.510740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:49.510806Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435655930712643:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:49.506718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655942628224:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6692
2025-11-19T13:58:49.506738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655942628225:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6692
2025-11-19T13:58:49.506782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655942628226:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6692
2025-11-19T13:58:49.510967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435657990893491:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.510979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:49.510983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435657990893491:2050] [node 3] ICP22 created new session: [4:7574435657990893584:2050]
2025-11-19T13:58:49.510987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435657990893584:2050] [node 3] ICS09 handshake done sender: [4:7574435657990893572:2116] self: [4:7574435657990893575:0] peer: [3:7574435655930712629:0] socket: 127 qp: -1
2025-11-19T13:58:49.510991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435657990893584:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:49.511003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:49.511098Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435657990893574:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:49.511485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435657990893490:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.511496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:49.511501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435657990893490:2049] [node 2] ICP22 created new session: [4:7574435657990893587:2049]
2025-11-19T13:58:49.511505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435657990893587:2049] [node 2] ICS09 handshake done sender: [4:7574435657990893574:2118] self: [4:7574435657990893576:0] peer: [2:7574435659003417722:0] socket: 103 qp: -1
2025-11-19T13:58:49.511510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435657990893587:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:49.511513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:49.512527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435656861029385:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11198) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435656861029470:2119] held: no
2025-11-19T13:58:49.512770Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435655783558906:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.513008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435655783558925:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48734
2025-11-19T13:58:49.513213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435655783558895:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.513231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:49.513237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435655783558895:2109] [node 5] ICP22 created new session: [1:7574435655783558926:2109]
2025-11-19T13:58:49.513243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435655783558926:2109] [node 5] ICS09 handshake done sender: [1:7574435655783558906:2113] self: [1:7574435655783558903:0] peer: [5:7574435656861029471:0] socket: 114 qp: -1
2025-11-19T13:58:49.513248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435655783558926:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:49.513254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:49.514507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435656861029383:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24254) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435656861029463:2116] held: no
2025-11-19T13:58:49.514939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435655930712564:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.514951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:49.514956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435655930712564:2049] [node 2] ICP22 created new session: [3:7574435655930712664:2049]
2025-11-19T13:58:49.514961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435655930712664:2049] [node 2] ICS09 handshake done sender: [3:7574435655930712643:2116] self: [3:7574435655930712650:0] peer: [2:7574435659003417729:0] socket: 125 qp: -1
2025-11-19T13:58:49.514965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435655930712664:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:49.514971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:49.516290Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435655783558925:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.516324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435655783558899:2110] [node 4] ICP18 handshake (actor [1:7574435655783558925:2118]) is too old
2025-11-19T13:58:49.517989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435657990893489:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.518006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:49.518012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435657990893489:2048] [node 1] ICP22 created new session: [4:7574435657990893589:2048]
2025-11-19T13:58:49.518017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435657990893589:2048] [node 1] ICS09 handshake done sender: [4:7574435657990893573:2117] self: [4:7574435657990893577:0] peer: [1:7574435655783558902:0] socket: 128 qp: -1
2025-11-19T13:58:49.518023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435657990893589:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:49.518029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:49.518097Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435657990893489:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.519430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435656861029474:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29038
2025-11-19T13:58:49.519469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435656861029475:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29038
2025-11-19T13:58:49.519511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435656861029476:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29038
2025-11-19T13:58:49.519687Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435656861029463:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:49.519825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435656861029383:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.519841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:49.519846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435656861029383:2048] [node 1] ICP22 created new session: [5:7574435656861029477:2048]
2025-11-19T13:58:49.519852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435656861029477:2048] [node 1] ICS09 handshake done sender: [5:7574435656861029463:2116] self: [5:7574435656861029471:0] peer: [1:7574435655783558903:0] socket: 117 qp: -1
2025-11-19T13:58:49.519857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435656861029477:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:49.519883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:49.512198Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435655942628214:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:49.512325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435655942628169:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.512353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:49.512369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435655942628169:2069] [node 4] ICP22 created new session: [6:7574435655942628227:2069]
2025-11-19T13:58:49.512375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435655942628227:2069] [node 4] ICS09 handshake done sender: [6:7574435655942628214:2108] self: [6:7574435655942628223:0] peer: [4:7574435657990893543:0] socket: 107 qp: -1
2025-11-19T13:58:49.512384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435655942628227:2069] [node 4] ICS10 traffic start
2025-11-19T13:58:49.512410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:49.520324Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435656861029470:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:49.520467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435656861029385:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.520480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:49.520485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435656861029385:2050] [node 3] ICP22 created new session: [5:7574435656861029479:2050]
2025-11-19T13:58:49.520491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435656861029479:2050] [node 3] ICS09 handshake done sender: [5:7574435656861029470:2119] self: [5:7574435656861029472:0] peer: [3:7574435655930712636:0] socket: 135 qp: -1
2025-11-19T13:58:49.520497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435656861029479:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:49.520503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:49.520902Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435656861029474:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.520934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435656861029384:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.521889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435656861029482:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29038
2025-11-19T13:58:49.521215Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435659003417743:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.521228Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435659003417743:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.525804Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435656861029474:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:49.525951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435656861029384:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.525972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:49.525977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435656861029384:2049] [node 2] ICP22 created new session: [5:7574435656861029483:2049]
2025-11-19T13:58:49.525983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435656861029483:2049] [node 2] ICS09 handshake done sender: [5:7574435656861029474:2120] self: [5:7574435656861029481:0] peer: [2:7574435659003417740:0] socket: 137 qp: -1
2025-11-19T13:58:49.525988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435656861029483:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:49.525997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:49.525313Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435659003417743:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:49.526997Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435659003417755:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.527049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435659003417660:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:49.527187Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435655783558913:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.527197Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435655783558913:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.528284Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435655783558913:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:49.528381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435655783558831:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.528397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:49.528402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435655783558831:2048] [node 2] ICP22 created new session: [1:7574435655783558929:2048]
2025-11-19T13:58:49.528408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435655783558929:2048] [node 2] ICS09 handshake done sender: [1:7574435655783558913:2117] self: [1:7574435655783558901:0] peer: [2:7574435659003417757:0] socket: 142 qp: -1
2025-11-19T13:58:49.528412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435655783558929:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:49.528417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:49.528608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435659003417726:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.528630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:49.528633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435659003417726:2109] [node 5] ICP22 created new session: [2:7574435659003417758:2109]
2025-11-19T13:58:49.528645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435659003417758:2109] [node 5] ICS09 handshake done sender: [2:7574435659003417743:2117] self: [2:7574435659003417740:0] peer: [5:7574435656861029481:0] socket: 136 qp: -1
2025-11-19T13:58:49.528649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435659003417758:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:49.528657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:49.535066Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435655783558910:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.535079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435655783558910:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.534295Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435655942628225:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.534435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435655942628144:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.536258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435655783558931:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48734
2025-11-19T13:58:49.537345Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435655783558910:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:49.537539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435655783558894:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.537554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:49.537562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435655783558894:2108] [node 6] ICP22 created new session: [1:7574435655783558933:2108]
2025-11-19T13:58:49.537569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435655783558933:2108] [node 6] ICS09 handshake done sender: [1:7574435655783558910:2115] self: [1:7574435655783558904:0] peer: [6:7574435655942628230:0] socket: 141 qp: -1
2025-11-19T13:58:49.537577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435655783558933:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:49.537606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:49.536078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655942628231:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6692
2025-11-19T13:58:49.536163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655942628232:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6692
2025-11-19T13:58:49.536206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655942628233:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6692
2025-11-19T13:58:49.538245Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435655942628231:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.538273Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435655942628232:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.538442Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435655942628224:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.538529Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435655942628225:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:49.542197Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435655783558931:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.541837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435655942628163:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:49.541868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435655942628145:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:49.541929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435655942628144:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.541942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:49.541948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435655942628144:2048] [node 1] ICP22 created new session: [6:7574435655942628237:2048]
2025-11-19T13:58:49.541953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435655942628237:2048] [node 1] ICS09 handshake done sender: [6:7574435655942628225:2117] self: [6:7574435655942628230:0] peer: [1:7574435655783558904:0] socket: 124 qp: -1
2025-11-19T13:58:49.541960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435655942628237:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:49.541970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.543217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435659003417660:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24254) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435659003417755:2119] held: no
2025-11-19T13:58:49.543283Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435659003417745:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.543287Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435659003417745:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.544220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435659003417761:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65021
2025-11-19T13:58:49.544681Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435659003417745:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:49.544887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435659003417727:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.544904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:49.544914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435659003417727:2108] [node 6] ICP22 created new session: [2:7574435659003417762:2108]
2025-11-19T13:58:49.544920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435659003417762:2108] [node 6] ICS09 handshake done sender: [2:7574435659003417745:2118] self: [2:7574435659003417741:0] peer: [6:7574435655942628236:0] socket: 94 qp: -1
2025-11-19T13:58:49.544925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435659003417762:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:49.544934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:49.545038Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435659003417755:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:49.545293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435659003417660:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.545303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:49.545307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435659003417660:2048] [node 1] ICP22 created new session: [2:7574435659003417764:2048]
2025-11-19T13:58:49.545312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435659003417764:2048] [node 1] ICS09 handshake done sender: [2:7574435659003417755:2119] self: [2:7574435659003417757:0] peer: [1:7574435655783558901:0] socket: 144 qp: -1
2025-11-19T13:58:49.545317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435659003417764:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:49.545322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:49.548336Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435656861029446:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.548354Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435656861029446:2100] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.547984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435655942628169:2069] [node 4] ICP35 event processing took too much time PendingConnection:278(35614955ns) {HandleHandshakeStatus:329(35614016ns) {DropIncomingHandshake:441(418ns) DropOutgoingHandshake:456(10704ns) SwitchToState:178(157ns) ProcessPendingSessionEvents:495(19491280ns) {ForwardSessionEventToSession:612(19474377ns) ForwardSessionEventToSession:612(2591ns) ForwardSessionEventToSession:612(1648ns) ForwardSessionEventToSession:612(669ns) ForwardSessionEventToSession:612(827ns) ForwardSessionEventToSession:612(1352ns) ForwardSessionEventToSession:612(568ns) ForwardSessionEventToSession:612(593ns) ForwardSessionEventToSession:612(606ns) ForwardSessionEventToSession:612(1705ns)}}}
2025-11-19T13:58:49.552018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435655942628146:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:49.553238Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435655942628145:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65021) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435655942628224:2116] held: no
2025-11-19T13:58:49.553868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435655783558894:2108] [node 6] ICP35 event processing took too much time PendingConnection:278(16296550ns) {HandleHandshakeStatus:329(16295773ns) {DropIncomingHandshake:441(114ns) DropOutgoingHandshake:456(249ns) SwitchToState:178(155ns) ProcessPendingSessionEvents:495(16187085ns) {ForwardSessionEventToSession:612(16172340ns) ForwardSessionEventToSession:612(1705ns) ForwardSessionEventToSession:612(880ns) ForwardSessionEventToSession:612(1444ns) ForwardSessionEventToSession:612(614ns) ForwardSessionEventToSession:612(567ns) ForwardSessionEventToSession:612(715ns) ForwardSessionEventToSession:612(781ns) ForwardSessionEventToSession:612(634ns) ForwardSessionEventToSession:612(1587ns)}}}
2025-11-19T13:58:49.553948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435655783558894:2108] [node 6] ICP18 handshake (actor [1:7574435655783558931:2119]) is too old
2025-11-19T13:58:49.556534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435659003417726:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(27896395ns) {HandleHandshakeStatus:329(27895643ns) {DropIncomingHandshake:441(5566ns) DropOutgoingHandshake:456(80ns) SwitchToState:178(73ns) ProcessPendingSessionEvents:495(27821260ns) {ForwardSessionEventToSession:612(27807757ns) ForwardSessionEventToSession:612(2081ns) ForwardSessionEventToSession:612(827ns) ForwardSessionEventToSession:612(739ns) ForwardSessionEventToSession:612(714ns) ForwardSessionEventToSession:612(1729ns) ForwardSessionEventToSession:612(827ns) ForwardSessionEventToSession:612(790ns) ForwardSessionEventToSession:612(789ns) ForwardSessionEventToSession:612(1102ns)}}}
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-19T13:58:49.557496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655942628240:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6692
2025-11-19T13:58:49.558748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655942628239:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6692
2025-11-19T13:58:49.556494Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435655930712648:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:49.556511Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435655930712648:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:49.557376Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435655930712648:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:49.557623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435655930712626:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.557639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:49.557651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435655930712626:2108] [node 6] ICP22 created new session: [3:7574435655930712667:2108]
2025-11-19T13:58:49.557659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435655930712667:2108] [node 6] ICS09 handshake done sender: [3:7574435655930712648:2119] self: [3:7574435655930712647:0] peer: [6:7574435655942628234:0] socket: 126 qp: -1
2025-11-19T13:58:49.557665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435655930712667:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:49.557674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:49.559531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435655942628146:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11198) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435655942628231:2119] held: no
2025-11-19T13:58:49.559555Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435655942628224:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:49.559701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435655942628145:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.559716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:49.559721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435655942628145:2049] [node 2] ICP22 created new session: [6:7574435655942628241:2049]
2025-11-19T13:58:49.559725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435655942628241:2049] [node 2] ICS09 handshake done sender: [6:7574435655942628224:2116] self: [6:7574435655942628236:0] peer: [2:7574435659003417741:0] socket: 123 qp: -1
2025-11-19T13:58:49.559732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435655942628241:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:49.559741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:49.560290Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435655942628231:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:49.560381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435655942628146:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.560394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:49.560399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435655942628146:2050] [node 3] ICP22 created new session: [6:7574435655942628243:2050]
2025-11-19T13:58:49.560404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435655942628243:2050] [node 3] ICS09 handshake done sender: [6:7574435655942628231:2119] self: [6:7574435655942628234:0] peer: [3:7574435655930712647:0] socket: 150 qp: -1
2025-11-19T13:58:49.560409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435655942628243:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:49.560418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:49.561201Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435656861029446:2100] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:49.561322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435656861029409:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:49.561339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:49.561343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435656861029409:2068] [node 6] ICP22 created new session: [5:7574435656861029486:2068]
2025-11-19T13:58:49.561349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435656861029486:2068] [node 6] ICS09 handshake done sender: [5:7574435656861029446:2100] self: [5:7574435656861029410:0] peer: [6:7574435655942628235:0] socket: 151 qp: -1
2025-11-19T13:58:49.561354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435656861029486:2068] [node 6] ICS10 traffic start
2025-11-19T13:58:49.561360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:49.561783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435655942628245:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6692
2025-11-19T13:58:49.561949Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435655942628232:2120] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:49.562736Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435655942628163:2064] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28273) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435655942628232:2120] held: no
2025-11-19T13:58:49.562881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435655942628163:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:49.562894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:49.562898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435655942628163:2064] [node 5] ICP22 created new session: [6:7574435655942628246:2064]
2025-11-19T13:58:49.562903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435655942628246:2064] [node 5] ICS09 handshake done sender: [6:7574435655942628232:2120] self: [6:7574435655942628235:0] peer: [5:7574435656861029410:0] socket: 153 qp: -1
2025-11-19T13:58:49.562908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435655942628246:2064] [node 5] ICS10 traffic start
2025-11-19T13:58:49.562914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:49.581255Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:49.581351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435659003417764:2048] [node 1] ICS01 socket: 144 reason# EndOfStream
2025-11-19T13:58:49.581360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435659003417660:2048] [node 1] ICP30 unregister session Session# [2:7574435659003417764:2048] VirtualId# [2:7574435659003417757:0]
2025-11-19T13:58:49.581367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435659003417764:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.581404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:49.581426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:49.583332Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:49.583412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435655930712659:2048] [node 1] ICS01 socket: 100 reason# EndOfStream
2025-11-19T13:58:49.583422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435655930712563:2048] [node 1] ICP30 unregister session Session# [3:7574435655930712659:2048] VirtualId# [3:7574435655930712649:0]
2025-11-19T13:58:49.583428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435655930712659:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.583456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:49.583476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:49.592654Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:49.592753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435657990893589:2048] [node 1] ICS01 socket: 128 reason# EndOfStream
2025-11-19T13:58:49.592762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435657990893489:2048] [node 1] ICP30 unregister session Session# [4:7574435657990893589:2048] VirtualId# [4:7574435657990893577:0]
2025-11-19T13:58:49.592770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435657990893589:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.592840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:49.592867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:49.593991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:49.594086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435655942628237:2048] [node 1] ICS01 socket: 124 reason# EndOfStream
2025-11-19T13:58:49.594094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435655942628144:2048] [node 1] ICP30 unregister session Session# [6:7574435655942628237:2048] VirtualId# [6:7574435655942628230:0]
2025-11-19T13:58:49.594101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435655942628237:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.594154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:49.594178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:49.594219Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:49.594284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435656861029477:2048] [node 1] ICS01 socket: 117 reason# EndOfStream
2025-11-19T13:58:49.594289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435656861029383:2048] [node 1] ICP30 unregister session Session# [5:7574435656861029477:2048] VirtualId# [5:7574435656861029471:0]
2025-11-19T13:58:49.594295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435656861029477:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.594323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:49.594346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:49.596341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:49.596432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435655930712664:2049] [node 2] ICS01 socket: 125 reason# EndOfStream
2025-11-19T13:58:49.596440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435655930712564:2049] [node 2] ICP30 unregister session Session# [3:7574435655930712664:2049] VirtualId# [3:7574435655930712650:0]
2025-11-19T13:58:49.596445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435655930712664:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.596483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:49.596502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:49.603255Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:49.603372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435655942628241:2049] [node 2] ICS01 socket: 123 reason# EndOfStream
2025-11-19T13:58:49.603384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435655942628145:2049] [node 2] ICP30 unregister session Session# [6:7574435655942628241:2049] VirtualId# [6:7574435655942628236:0]
2025-11-19T13:58:49.603392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435655942628241:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.603454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:49.603486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:49.607051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:49.607146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435657990893587:2049] [node 2] ICS01 socket: 103 reason# EndOfStream
2025-11-19T13:58:49.607155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435657990893490:2049] [node 2] ICP30 unregister session Session# [4:7574435657990893587:2049] VirtualId# [4:7574435657990893576:0]
2025-11-19T13:58:49.607162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435657990893587:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.607224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:49.607250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:49.607867Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:49.607951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435656861029483:2049] [node 2] ICS01 socket: 137 reason# EndOfStream
2025-11-19T13:58:49.607959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435656861029384:2049] [node 2] ICP30 unregister session Session# [5:7574435656861029483:2049] VirtualId# [5:7574435656861029481:0]
2025-11-19T13:58:49.607965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435656861029483:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.608035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:49.608061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:49.617110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:49.617218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435656861029479:2050] [node 3] ICS01 socket: 135 reason# EndOfStream
2025-11-19T13:58:49.617227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435656861029385:2050] [node 3] ICP30 unregister session Session# [5:7574435656861029479:2050] VirtualId# [5:7574435656861029472:0]
2025-11-19T13:58:49.617234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435656861029479:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.617286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:49.617310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:49.619826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:49.619940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435655942628243:2050] [node 3] ICS01 socket: 150 reason# EndOfStream
2025-11-19T13:58:49.619952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435655942628146:2050] [node 3] ICP30 unregister session Session# [6:7574435655942628243:2050] VirtualId# [6:7574435655942628234:0]
2025-11-19T13:58:49.619960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435655942628243:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.620020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:49.620050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:49.632946Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:49.633052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435655942628227:2069] [node 4] ICS07 socket disconnect 107 reason# EndOfStream
2025-11-19T13:58:49.633058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435655942628227:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:49.633119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:49.633143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435655942628227:2069] [node 4] ICS15 start handshake
2025-11-19T13:58:49.635921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435655942628169:2069] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31768) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:49.635939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435655942628227:2069] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:49.635946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435655942628169:2069] [node 4] ICP30 unregister session Session# [6:7574435655942628227:2069] VirtualId# [6:7574435655942628223:0]
2025-11-19T13:58:49.635966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:58:49.635971Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435655942628169:2069] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31768) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:49.635975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31768) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:49.920268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435658443300999:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:49.920283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435658443301000:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:49.972155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435659351003270:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:49.972180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435659351003271:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:50.021604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435662992782636:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:50.021615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435662992782637:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:50.059363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435662449727676:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:50.059385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435662449727677:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:50.059391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435662449727678:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:50.083542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435661182440517:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:50.083557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435661182440518:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:50.083563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435661182440519:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:50.123946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435663455286133:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:50.123961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435663455286134:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:50.123967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435663455286135:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:50.133487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435662738268345:2093] [node 4] ICP01 ready to work
2025-11-19T13:58:50.133573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435662738268346:2094] [node 6] ICP01 ready to work
2025-11-19T13:58:50.134272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435662992782676:2086] [node 4] ICP01 ready to work
2025-11-19T13:58:50.134736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435662449727706:2063] [node 5] ICP01 ready to work
2025-11-19T13:58:50.134831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435662449727707:2067] [node 6] ICP01 ready to work
2025-11-19T13:58:50.134832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435661182440547:2072] [node 4] ICP01 ready to work
2025-11-19T13:58:50.136026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435663645970611:2086] [node 5] ICP01 ready to work
2025-11-19T13:58:50.136117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435663645970612:2089] [node 4] ICP01 ready to work
2025-11-19T13:58:50.136190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435663645970614:2091] [node 6] ICP01 ready to work
2025-11-19T13:58:50.136201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435662738268361:2099] [node 5] ICP01 ready to work
2025-11-19T13:58:50.136530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435661182440562:2081] [node 6] ICP01 ready to work
2025-11-19T13:58:50.141937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435662992782685:2088] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.155971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435662449727706:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21079534ns) {Configure:104(21079029ns) {ProcessConfigured:135(20822809ns) {StartInitialHandshake:157(20822139ns) {DropHandshakes:469(384ns) {DropIncomingHandshake:441(41ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(1912ns) {SwitchToState:178(104ns)}}}}}
2025-11-19T13:58:50.157401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435662449727744:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27123
2025-11-19T13:58:50.157449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435662449727745:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27123
2025-11-19T13:58:50.157952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435659351003270:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21694386ns) {Configure:104(21693338ns) {ProcessConfigured:135(21326270ns) {StartInitialHandshake:157(21325917ns) {DropHandshakes:469(752ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(88ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:476(1654ns) {SwitchToState:178(1284ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.164454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435662992782706:2092] [node 6] ICP01 ready to work
2025-11-19T13:58:50.165686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435662992782703:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45638
2025-11-19T13:58:50.165780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435662992782704:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45638
2025-11-19T13:58:50.165793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435662992782705:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45638
2025-11-19T13:58:50.168313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435663455286134:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(34467533ns) {Configure:104(34465766ns) {ProcessConfigured:135(34015105ns) {StartInitialHandshake:157(34014387ns) {DropHandshakes:469(468ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(1694ns) {SwitchToState:178(1134ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.172913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435658443300999:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17660827ns) {Configure:104(17659587ns) {ProcessConfigured:135(11817042ns) {StartInitialHandshake:157(11815132ns) {DropHandshakes:469(577ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(1664ns) {SwitchToState:178(851ns)}}}}}
2025-11-19T13:58:50.173570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435662738268370:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44129
2025-11-19T13:58:50.173585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435662738268375:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44129
2025-11-19T13:58:50.173743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435663645970645:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25669
2025-11-19T13:58:50.173763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435663645970646:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25669
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.180444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435663455286199:2065] [node 5] ICP01 ready to work
2025-11-19T13:58:50.180571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435663455286200:2078] [node 4] ICP01 ready to work
2025-11-19T13:58:50.181746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435663455286198:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3871
2025-11-19T13:58:50.182580Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435661182440562:2081] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(45987845ns) {Configure:104(45987234ns) {ProcessConfigured:135(45741660ns) {StartInitialHandshake:157(45741420ns) {DropHandshakes:469(397ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(82ns) PrepareNewSessionHandshake:476(651ns) {SwitchToState:178(343ns)}}}}}
2025-11-19T13:58:50.184235Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435663455286198:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.184328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435663455286133:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.192025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435662738268376:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44129
2025-11-19T13:58:50.192181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435662449727756:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27123
2025-11-19T13:58:50.192225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435662449727757:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27123
2025-11-19T13:58:50.192273Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435662449727744:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.200922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435663645970647:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25669
2025-11-19T13:58:50.201249Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435662738268350:2097] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.201272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435662738268350:2097] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.202240Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435662992782703:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.207035Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435662738268350:2097] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:50.205969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435662992782636:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.207349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435662738268344:2095] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.207362Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435662738268344:2095] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.207815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435662738268346:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.207830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:50.207838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435662738268346:2094] [node 6] ICP22 created new session: [1:7574435662738268378:2094]
2025-11-19T13:58:50.207845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435662738268378:2094] [node 6] ICS09 handshake done sender: [1:7574435662738268350:2097] self: [1:7574435662738268349:0] peer: [6:7574435663455286212:0] socket: 98 qp: -1
2025-11-19T13:58:50.207852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435662738268378:2094] [node 6] ICS10 traffic start
2025-11-19T13:58:50.207863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.208893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435663455286213:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3871
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.209136Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435663645970647:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.209184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435663645970648:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25669
2025-11-19T13:58:50.209404Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435663455286198:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:50.209500Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435662738268344:2095] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:50.209617Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435662738268376:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.209642Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435662738268375:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.209649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435658443301000:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.209659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:50.209667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435658443301000:2049] [node 3] ICP22 created new session: [1:7574435662738268382:2049]
2025-11-19T13:58:50.209672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435662738268382:2049] [node 3] ICS09 handshake done sender: [1:7574435662738268344:2095] self: [1:7574435662738268342:0] peer: [3:7574435662992782718:0] socket: 96 qp: -1
2025-11-19T13:58:50.209677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435662738268382:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:50.209684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:50.211035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435662738268346:2094] [node 6] ICP18 handshake (actor [1:7574435662738268376:2117]) is too old
2025-11-19T13:58:50.211725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435663455286133:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.211758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:50.211764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435663455286133:2048] [node 1] ICP22 created new session: [6:7574435663455286214:2048]
2025-11-19T13:58:50.211770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435663455286214:2048] [node 1] ICS09 handshake done sender: [6:7574435663455286198:2111] self: [6:7574435663455286212:0] peer: [1:7574435662738268349:0] socket: 119 qp: -1
2025-11-19T13:58:50.211777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435663455286214:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:50.211784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:50.217557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435662449727676:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.217991Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435663645970648:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.218019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435659351003270:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.218156Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435662738268348:2096] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.218167Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435662738268348:2096] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.218610Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435662738268372:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.218614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435662738268372:2114] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.220870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435663645970651:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25669
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.221101Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435662992782705:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.221203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435662992782719:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45638
2025-11-19T13:58:50.221335Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435663645970646:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.221530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435662992782720:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45638
2025-11-19T13:58:50.221588Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435662992782703:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.224623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435661182440595:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40296
2025-11-19T13:58:50.224698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435661182440596:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40296
2025-11-19T13:58:50.224742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435661182440597:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40296
2025-11-19T13:58:50.224774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435661182440598:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40296
2025-11-19T13:58:50.224807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435661182440599:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40296
2025-11-19T13:58:50.228632Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435663645970633:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.228644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435663645970633:2109] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.229435Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435663645970633:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:50.230069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435663645970611:2086] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.230095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:50.230100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435663645970611:2086] [node 5] ICP22 created new session: [2:7574435663645970654:2086]
2025-11-19T13:58:50.230105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435663645970654:2086] [node 5] ICS09 handshake done sender: [2:7574435663645970633:2109] self: [2:7574435663645970632:0] peer: [5:7574435661182440600:0] socket: 110 qp: -1
2025-11-19T13:58:50.230112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435663645970654:2086] [node 5] ICS10 traffic start
2025-11-19T13:58:50.230118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:50.228498Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435661182440596:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.228538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435661182440518:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.229354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435661182440601:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40296
2025-11-19T13:58:50.229609Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435661182440596:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:50.229661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435661182440518:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.229675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:50.229681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435661182440518:2049] [node 2] ICP22 created new session: [5:7574435661182440602:2049]
2025-11-19T13:58:50.229687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435661182440602:2049] [node 2] ICS09 handshake done sender: [5:7574435661182440596:2116] self: [5:7574435661182440600:0] peer: [2:7574435663645970632:0] socket: 129 qp: -1
2025-11-19T13:58:50.229693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435661182440602:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:50.229701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.239784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435663455286217:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3871
2025-11-19T13:58:50.239833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435663455286218:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3871
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.240091Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435662992782704:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.240180Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435662992782720:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.240189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435662992782636:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.240206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:50.240213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435662992782636:2048] [node 1] ICP22 created new session: [3:7574435662992782722:2048]
2025-11-19T13:58:50.240220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435662992782722:2048] [node 1] ICS09 handshake done sender: [3:7574435662992782703:2111] self: [3:7574435662992782718:0] peer: [1:7574435662738268342:0] socket: 103 qp: -1
2025-11-19T13:58:50.240228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435662992782722:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:50.240235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:50.240231Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435662449727756:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.240315Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435662449727757:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.240445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435662449727677:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.240457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435662449727678:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.240475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435662449727759:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27123
2025-11-19T13:58:50.240718Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435662738268348:2096] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:50.240695Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435661182440598:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.240778Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435661182440597:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.240829Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435661182440595:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.240864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435661182440519:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.240873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435661182440547:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.241114Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435662738268372:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:50.241144Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435662449727745:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.241177Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435662449727734:2101] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.241180Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435662449727734:2101] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.241885Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435663455286217:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.241923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435663455286199:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.241603Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435662449727744:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:50.242191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435662449727676:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.242201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:50.242205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435662449727676:2048] [node 1] ICP22 created new session: [4:7574435662449727763:2048]
2025-11-19T13:58:50.242212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435662449727763:2048] [node 1] ICS09 handshake done sender: [4:7574435662449727744:2108] self: [4:7574435662449727758:0] peer: [1:7574435662738268347:0] socket: 104 qp: -1
2025-11-19T13:58:50.242216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435662449727763:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:50.242221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:50.241938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435662738268370:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.242116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435662738268345:2093] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.242125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:50.242131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435662738268345:2093] [node 4] ICP22 created new session: [1:7574435662738268388:2093]
2025-11-19T13:58:50.242137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435662738268388:2093] [node 4] ICS09 handshake done sender: [1:7574435662738268348:2096] self: [1:7574435662738268347:0] peer: [4:7574435662449727758:0] socket: 97 qp: -1
2025-11-19T13:58:50.242142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435662738268388:2093] [node 4] ICS10 traffic start
2025-11-19T13:58:50.242161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:50.242280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435658443300999:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.242290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:50.242295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435658443300999:2048] [node 2] ICP22 created new session: [1:7574435662738268390:2048]
2025-11-19T13:58:50.242299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435662738268390:2048] [node 2] ICS09 handshake done sender: [1:7574435662738268372:2114] self: [1:7574435662738268371:0] peer: [2:7574435663645970650:0] socket: 134 qp: -1
2025-11-19T13:58:50.242302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435662738268390:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:50.242306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:50.242547Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435663645970645:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.242587Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435663645970635:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.242590Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435663645970635:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.243308Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435663455286218:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.243333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435663455286134:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.243429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435661182440594:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.243437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435661182440594:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.244101Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435661182440599:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.244272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435661182440517:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.244293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435661182440608:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40296
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.244525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435663455286221:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3871
2025-11-19T13:58:50.244646Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435661182440594:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:50.244832Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435663455286217:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:50.244927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435663455286199:2065] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.244942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:50.244948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435663455286199:2065] [node 5] ICP22 created new session: [6:7574435663455286222:2065]
2025-11-19T13:58:50.244953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435663455286222:2065] [node 5] ICS09 handshake done sender: [6:7574435663455286217:2117] self: [6:7574435663455286219:0] peer: [5:7574435661182440563:0] socket: 147 qp: -1
2025-11-19T13:58:50.244959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435663455286222:2065] [node 5] ICS10 traffic start
2025-11-19T13:58:50.244967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:50.247596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435661182440562:2081] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.247618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:50.247624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435661182440562:2081] [node 6] ICP22 created new session: [5:7574435661182440611:2081]
2025-11-19T13:58:50.247630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435661182440611:2081] [node 6] ICS09 handshake done sender: [5:7574435661182440594:2114] self: [5:7574435661182440563:0] peer: [6:7574435663455286219:0] socket: 145 qp: -1
2025-11-19T13:58:50.247646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435661182440611:2081] [node 6] ICS10 traffic start
2025-11-19T13:58:50.247655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:50.248261Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435662992782684:2093] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.248275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435662992782684:2093] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.248190Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435663645970637:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.248202Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435663645970637:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.248725Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435663645970648:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:50.249091Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435662449727734:2101] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:50.248936Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435662738268374:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.248949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435662738268374:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.251789Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435662738268374:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:50.251890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435662738268361:2099] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.251908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:50.251915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435662738268361:2099] [node 5] ICP22 created new session: [1:7574435662738268394:2099]
2025-11-19T13:58:50.251921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435662738268394:2099] [node 5] ICS09 handshake done sender: [1:7574435662738268374:2115] self: [1:7574435662738268373:0] peer: [5:7574435661182440610:0] socket: 137 qp: -1
2025-11-19T13:58:50.251928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435662738268394:2099] [node 5] ICS10 traffic start
2025-11-19T13:58:50.251939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.250660Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435662992782708:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.250674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435662992782708:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.251571Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435662992782684:2093] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:50.253508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435659351003270:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.253533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:50.253539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435659351003270:2048] [node 1] ICP22 created new session: [2:7574435663645970660:2048]
2025-11-19T13:58:50.253545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435663645970660:2048] [node 1] ICS09 handshake done sender: [2:7574435663645970648:2118] self: [2:7574435663645970650:0] peer: [1:7574435662738268371:0] socket: 136 qp: -1
2025-11-19T13:58:50.253551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435663645970660:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:50.253558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:50.253625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435659351003270:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:50.253746Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435663645970635:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:50.253915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435663645970612:2089] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.253924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:50.253929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435663645970612:2089] [node 4] ICP22 created new session: [2:7574435663645970662:2089]
2025-11-19T13:58:50.253933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435663645970662:2089] [node 4] ICS09 handshake done sender: [2:7574435663645970635:2110] self: [2:7574435663645970634:0] peer: [4:7574435662449727760:0] socket: 111 qp: -1
2025-11-19T13:58:50.253937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435663645970662:2089] [node 4] ICS10 traffic start
2025-11-19T13:58:50.253944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:50.254359Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435663645970637:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:50.254507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435663645970614:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.254518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:50.254522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435663645970614:2091] [node 6] ICP22 created new session: [2:7574435663645970665:2091]
2025-11-19T13:58:50.254526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435663645970665:2091] [node 6] ICS09 handshake done sender: [2:7574435663645970637:2111] self: [2:7574435663645970636:0] peer: [6:7574435663455286220:0] socket: 146 qp: -1
2025-11-19T13:58:50.254530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435663645970665:2091] [node 6] ICS10 traffic start
2025-11-19T13:58:50.254536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.256885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435663455286225:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3871
2025-11-19T13:58:50.256929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435663455286226:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3871
2025-11-19T13:58:50.257132Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435662992782708:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:50.258725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435662992782676:2086] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.258747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:50.258755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435662992782676:2086] [node 4] ICP22 created new session: [3:7574435662992782729:2086]
2025-11-19T13:58:50.258762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435662992782729:2086] [node 4] ICS09 handshake done sender: [3:7574435662992782684:2093] self: [3:7574435662992782677:0] peer: [4:7574435662449727761:0] socket: 116 qp: -1
2025-11-19T13:58:50.258768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435662992782729:2086] [node 4] ICS10 traffic start
2025-11-19T13:58:50.258777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:50.258957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435662992782685:2088] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.258967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:50.258971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435662992782685:2088] [node 5] ICP22 created new session: [3:7574435662992782732:2088]
2025-11-19T13:58:50.258975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435662992782732:2088] [node 5] ICS09 handshake done sender: [3:7574435662992782708:2114] self: [3:7574435662992782707:0] peer: [5:7574435661182440606:0] socket: 117 qp: -1
2025-11-19T13:58:50.258979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435662992782732:2088] [node 5] ICS10 traffic start
2025-11-19T13:58:50.258986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:50.260478Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435663455286134:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17764) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435663455286218:2118] held: no
2025-11-19T13:58:50.260519Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435663455286218:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:50.260641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435663455286134:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.260656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:50.260662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435663455286134:2049] [node 2] ICP22 created new session: [6:7574435663455286227:2049]
2025-11-19T13:58:50.260668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435663455286227:2049] [node 2] ICS09 handshake done sender: [6:7574435663455286218:2118] self: [6:7574435663455286220:0] peer: [2:7574435663645970636:0] socket: 148 qp: -1
2025-11-19T13:58:50.260674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435663455286227:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:50.260682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:50.261824Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435661182440517:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25004) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435661182440599:2119] held: no
2025-11-19T13:58:50.261858Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435661182440595:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.261996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435661182440547:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.262008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:50.262013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435661182440547:2072] [node 4] ICP22 created new session: [5:7574435661182440615:2072]
2025-11-19T13:58:50.262018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435661182440615:2072] [node 4] ICS09 handshake done sender: [5:7574435661182440595:2115] self: [5:7574435661182440607:0] peer: [4:7574435662449727708:0] socket: 128 qp: -1
2025-11-19T13:58:50.262024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435661182440615:2072] [node 4] ICS10 traffic start
2025-11-19T13:58:50.262031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:50.262165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435661182440613:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40296
2025-11-19T13:58:50.262503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435662449727706:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.262525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:50.262530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435662449727706:2063] [node 5] ICP22 created new session: [4:7574435662449727769:2063]
2025-11-19T13:58:50.262536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435662449727769:2063] [node 5] ICS09 handshake done sender: [4:7574435662449727734:2101] self: [4:7574435662449727708:0] peer: [5:7574435661182440607:0] socket: 107 qp: -1
2025-11-19T13:58:50.262541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435662449727769:2063] [node 5] ICS10 traffic start
2025-11-19T13:58:50.262549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.263662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435661182440614:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40296
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.267181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435662449727677:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17764) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435662449727756:2117] held: no
2025-11-19T13:58:50.267229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435662449727678:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435662449727757:2118] held: no
2025-11-19T13:58:50.267285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435662449727766:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27123
2025-11-19T13:58:50.267340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435662449727767:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27123
2025-11-19T13:58:50.266315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435662992782731:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45638
2025-11-19T13:58:50.267636Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435662992782731:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.267667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435662992782637:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.267799Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435663455286225:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.267830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435663455286135:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.267922Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435663645970639:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.267933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435663645970639:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.268819Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435663645970639:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.270855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435661182440519:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18098) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435661182440597:2117] held: no
2025-11-19T13:58:50.271001Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435661182440599:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:50.271100Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435661182440597:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:50.271174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435661182440517:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.271189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:50.271194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435661182440517:2048] [node 1] ICP22 created new session: [5:7574435661182440617:2048]
2025-11-19T13:58:50.271200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435661182440617:2048] [node 1] ICS09 handshake done sender: [5:7574435661182440599:2119] self: [5:7574435661182440610:0] peer: [1:7574435662738268373:0] socket: 142 qp: -1
2025-11-19T13:58:50.271206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435661182440617:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:50.271213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:50.271288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435661182440519:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.271292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:50.271295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435661182440519:2050] [node 3] ICP22 created new session: [5:7574435661182440619:2050]
2025-11-19T13:58:50.271299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435661182440619:2050] [node 3] ICS09 handshake done sender: [5:7574435661182440597:2117] self: [5:7574435661182440606:0] peer: [3:7574435662992782707:0] socket: 130 qp: -1
2025-11-19T13:58:50.271303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435661182440619:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:50.271309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:50.269504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435659351003271:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.269529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:50.269537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435659351003271:2049] [node 3] ICP22 created new session: [2:7574435663645970668:2049]
2025-11-19T13:58:50.269544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435663645970668:2049] [node 3] ICS09 handshake done sender: [2:7574435663645970639:2112] self: [2:7574435663645970638:0] peer: [3:7574435662992782734:0] socket: 109 qp: -1
2025-11-19T13:58:50.269550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435663645970668:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:50.269561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:50.269821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435662449727768:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27123
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.276048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435663455286230:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3871
2025-11-19T13:58:50.276336Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435662992782714:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.276353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435662992782714:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.278580Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435662992782637:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17764) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435662992782731:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.278888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435663455286231:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3871
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.283192Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435662449727766:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.283298Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435662449727756:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:50.283440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435662992782735:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45638
2025-11-19T13:58:50.288771Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435663455286225:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:50.288550Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435662992782714:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:50.288961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435663455286135:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.288985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:50.288993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435663455286135:2050] [node 3] ICP22 created new session: [6:7574435663455286232:2050]
2025-11-19T13:58:50.289000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435663455286232:2050] [node 3] ICS09 handshake done sender: [6:7574435663455286225:2120] self: [6:7574435663455286229:0] peer: [3:7574435662992782713:0] socket: 131 qp: -1
2025-11-19T13:58:50.289008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435663455286232:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:50.289017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:50.290108Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435662449727757:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:50.290230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435662449727677:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.290247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:50.290253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435662449727677:2049] [node 2] ICP22 created new session: [4:7574435662449727772:2049]
2025-11-19T13:58:50.290261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435662449727772:2049] [node 2] ICS09 handshake done sender: [4:7574435662449727756:2117] self: [4:7574435662449727760:0] peer: [2:7574435663645970634:0] socket: 113 qp: -1
2025-11-19T13:58:50.290267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435662449727772:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:50.290276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:50.290393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435662449727678:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.290401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:50.290405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435662449727678:2050] [node 3] ICP22 created new session: [4:7574435662449727774:2050]
2025-11-19T13:58:50.290411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435662449727774:2050] [node 3] ICS09 handshake done sender: [4:7574435662449727757:2118] self: [4:7574435662449727761:0] peer: [3:7574435662992782677:0] socket: 122 qp: -1
2025-11-19T13:58:50.290415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435662449727774:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:50.290421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:50.292212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435662992782706:2092] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.292236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:50.292242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435662992782706:2092] [node 6] ICP22 created new session: [3:7574435662992782737:2092]
2025-11-19T13:58:50.292248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435662992782737:2092] [node 6] ICS09 handshake done sender: [3:7574435662992782714:2117] self: [3:7574435662992782713:0] peer: [6:7574435663455286229:0] socket: 157 qp: -1
2025-11-19T13:58:50.292254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435662992782737:2092] [node 6] ICS10 traffic start
2025-11-19T13:58:50.292261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:50.292404Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435662992782731:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:50.292756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435662992782637:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.292769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:50.292775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435662992782637:2049] [node 2] ICP22 created new session: [3:7574435662992782739:2049]
2025-11-19T13:58:50.292780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435662992782739:2049] [node 2] ICS09 handshake done sender: [3:7574435662992782731:2120] self: [3:7574435662992782734:0] peer: [2:7574435663645970638:0] socket: 127 qp: -1
2025-11-19T13:58:50.292795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435662992782739:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:50.292802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:50.293632Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435663455286230:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.293666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435663455286200:2078] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.293821Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435662449727741:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.293833Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435662449727741:2106] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.294569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435663455286235:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3871
2025-11-19T13:58:50.294688Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435662449727741:2106] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:50.294935Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435663455286230:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:50.295018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435663455286200:2078] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.295042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:50.295048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435663455286200:2078] [node 4] ICP22 created new session: [6:7574435663455286236:2078]
2025-11-19T13:58:50.295053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435663455286236:2078] [node 4] ICS09 handshake done sender: [6:7574435663455286230:2122] self: [6:7574435663455286234:0] peer: [4:7574435662449727740:0] socket: 101 qp: -1
2025-11-19T13:58:50.295061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435663455286236:2078] [node 4] ICS10 traffic start
2025-11-19T13:58:50.295070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:50.296396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435662449727707:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.296413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:50.296420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435662449727707:2067] [node 6] ICP22 created new session: [4:7574435662449727778:2067]
2025-11-19T13:58:50.296426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435662449727778:2067] [node 6] ICS09 handshake done sender: [4:7574435662449727741:2106] self: [4:7574435662449727740:0] peer: [6:7574435663455286234:0] socket: 99 qp: -1
2025-11-19T13:58:50.296431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435662449727778:2067] [node 6] ICS10 traffic start
2025-11-19T13:58:50.296439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:50.300560Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:50.300642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435663645970660:2048] [node 1] ICS01 socket: 136 reason# EndOfStream
2025-11-19T13:58:50.300649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435659351003270:2048] [node 1] ICP30 unregister session Session# [2:7574435663645970660:2048] VirtualId# [2:7574435663645970650:0]
2025-11-19T13:58:50.300655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435663645970660:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.300690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:50.300720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:50.301319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:50.301386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435662992782722:2048] [node 1] ICS01 socket: 103 reason# EndOfStream
2025-11-19T13:58:50.301393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435662992782636:2048] [node 1] ICP30 unregister session Session# [3:7574435662992782722:2048] VirtualId# [3:7574435662992782718:0]
2025-11-19T13:58:50.301398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435662992782722:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.301422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:50.301439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:50.316071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:50.316173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435662449727763:2048] [node 1] ICS07 socket disconnect 104 reason# EndOfStream
2025-11-19T13:58:50.316182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435662449727763:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.316259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:50.316283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435662449727763:2048] [node 1] ICS15 start handshake
2025-11-19T13:58:50.316525Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:50.316609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435663455286214:2048] [node 1] ICS01 socket: 119 reason# EndOfStream
2025-11-19T13:58:50.316616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435663455286133:2048] [node 1] ICP30 unregister session Session# [6:7574435663455286214:2048] VirtualId# [6:7574435663455286212:0]
2025-11-19T13:58:50.316623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435663455286214:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.316664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:50.316690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:50.316734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:50.316794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435661182440617:2048] [node 1] ICS01 socket: 142 reason# EndOfStream
2025-11-19T13:58:50.316800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435661182440517:2048] [node 1] ICP30 unregister session Session# [5:7574435661182440617:2048] VirtualId# [5:7574435661182440610:0]
2025-11-19T13:58:50.316805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435661182440617:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.316831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:50.316853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:50.317477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435662449727676:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25004) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:50.317490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435662449727763:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:50.317494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435662449727676:2048] [node 1] ICP30 unregister session Session# [4:7574435662449727763:2048] VirtualId# [4:7574435662449727758:0]
2025-11-19T13:58:50.317506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:50.317511Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435662449727676:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25004) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:50.317514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25004) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:50.320425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:50.320514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435662992782739:2049] [node 2] ICS01 socket: 127 reason# EndOfStream
2025-11-19T13:58:50.320522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435662992782637:2049] [node 2] ICP30 unregister session Session# [3:7574435662992782739:2049] VirtualId# [3:7574435662992782734:0]
2025-11-19T13:58:50.320530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435662992782739:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.320578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:50.320599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:50.328018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435662449727676:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:58:50.343854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:50.343964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435661182440602:2049] [node 2] ICS01 socket: 129 reason# EndOfStream
2025-11-19T13:58:50.343976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435661182440518:2049] [node 2] ICP30 unregister session Session# [5:7574435661182440602:2049] VirtualId# [5:7574435661182440600:0]
2025-11-19T13:58:50.343984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435661182440602:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.344045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:50.344072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:50.344233Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:50.344303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435662449727772:2049] [node 2] ICS01 socket: 113 reason# EndOfStream
2025-11-19T13:58:50.344309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435662449727677:2049] [node 2] ICP30 unregister session Session# [4:7574435662449727772:2049] VirtualId# [4:7574435662449727760:0]
2025-11-19T13:58:50.344314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435662449727772:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.344355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:50.344378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:50.345597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:50.345691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435663455286227:2049] [node 2] ICS01 socket: 148 reason# EndOfStream
2025-11-19T13:58:50.345699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435663455286134:2049] [node 2] ICP30 unregister session Session# [6:7574435663455286227:2049] VirtualId# [6:7574435663455286220:0]
2025-11-19T13:58:50.345706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435663455286227:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.345768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:50.345797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:50.370584Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:50.370688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435661182440619:2050] [node 3] ICS01 socket: 130 reason# EndOfStream
2025-11-19T13:58:50.370699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435661182440519:2050] [node 3] ICP30 unregister session Session# [5:7574435661182440619:2050] VirtualId# [5:7574435661182440606:0]
2025-11-19T13:58:50.370706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435661182440619:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.370766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:50.370790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:50.370698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:50.370815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435663455286232:2050] [node 3] ICS01 socket: 131 reason# EndOfStream
2025-11-19T13:58:50.370824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435663455286135:2050] [node 3] ICP30 unregister session Session# [6:7574435663455286232:2050] VirtualId# [6:7574435663455286229:0]
2025-11-19T13:58:50.370831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435663455286232:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.370904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:50.370940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:50.393175Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:50.393270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435663455286236:2078] [node 4] ICS01 socket: 101 reason# EndOfStream
2025-11-19T13:58:50.393281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435663455286200:2078] [node 4] ICP30 unregister session Session# [6:7574435663455286236:2078] VirtualId# [6:7574435663455286234:0]
2025-11-19T13:58:50.393289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435663455286236:2078] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.393344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:50.393368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:50.476058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435659925968941:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:50.476072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435659925968942:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:50.519978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435660194239603:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:50.520000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435660194239604:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:50.549810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435663927496787:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:50.549824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435663927496788:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:50.603981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435664105340247:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:50.603997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435664105340248:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:50.604002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435664105340249:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:50.640121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435662056140209:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:50.640137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435662056140210:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:50.640142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435662056140211:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:50.673731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435661691462332:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:50.673743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435661691462333:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:50.673748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435661691462334:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:50.724070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435659925969003:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:50.724303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435662056140246:2083] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.725138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435660194239664:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:50.726064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435659925969011:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:50.726685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435660194239667:2108] [node 5] ICP01 ready to work
2025-11-19T13:58:50.730142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435659925969001:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60021
2025-11-19T13:58:50.732919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435659925969018:2114] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.737211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435664105340314:2108] [node 5] ICP01 ready to work
2025-11-19T13:58:50.737298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435664105340315:2110] [node 6] ICP01 ready to work
2025-11-19T13:58:50.735998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435662056140256:2086] [node 4] ICP01 ready to work
2025-11-19T13:58:50.738548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435660194239672:2110] [node 6] ICP01 ready to work
2025-11-19T13:58:50.739338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435660194239668:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12128
2025-11-19T13:58:50.739352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435660194239670:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12128
2025-11-19T13:58:50.739363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435660194239671:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12128
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.743654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435663927496855:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.743229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435662056140281:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5220
2025-11-19T13:58:50.743248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435662056140282:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5220
2025-11-19T13:58:50.744274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435663927496851:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19219
2025-11-19T13:58:50.744298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435663927496852:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19219
2025-11-19T13:58:50.744311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435663927496853:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19219
2025-11-19T13:58:50.744324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435663927496854:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19219
2025-11-19T13:58:50.744413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435663927496862:2113] [node 5] ICP01 ready to work
2025-11-19T13:58:50.744481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435663927496863:2115] [node 6] ICP01 ready to work
2025-11-19T13:58:50.746375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435661691462356:2066] [node 5] ICP01 ready to work
2025-11-19T13:58:50.746514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435661691462399:2070] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.752448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435659925969018:2114] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19472969ns) {Configure:104(19471897ns) {ProcessConfigured:135(19133954ns) {StartInitialHandshake:157(19133139ns) {DropHandshakes:469(696ns) {DropIncomingHandshake:441(100ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(112ns) PrepareNewSessionHandshake:476(1609ns) {SwitchToState:178(835ns)}}}}}
2025-11-19T13:58:50.752681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435659925969021:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60021
2025-11-19T13:58:50.752695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435659925969022:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60021
2025-11-19T13:58:50.752704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435659925969023:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60021
2025-11-19T13:58:50.752715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435659925969024:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60021
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.757402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435664105340325:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4929
2025-11-19T13:58:50.757428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435664105340326:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4929
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.759609Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435661691462333:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(31425982ns) {Configure:104(31424595ns) {ProcessConfigured:135(31132162ns) {StartInitialHandshake:157(31131292ns) {DropHandshakes:469(497ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(110ns)} GenerateSessionVirtualId:988(270ns) PrepareNewSessionHandshake:476(1025ns) {SwitchToState:178(682ns)}}}}}
2025-11-19T13:58:50.760202Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435659925969022:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.760323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435661691462404:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.774219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435662056140288:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5220
2025-11-19T13:58:50.774358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435662056140256:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(38262730ns) {Configure:104(38261893ns) {ProcessConfigured:135(37806486ns) {StartInitialHandshake:157(37805819ns) {DropHandshakes:469(840ns) {DropIncomingHandshake:441(108ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(181ns) PrepareNewSessionHandshake:476(748ns) {SwitchToState:178(291ns)}}}}}
2025-11-19T13:58:50.775074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435662056140282:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.775110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435662056140210:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.775232Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435660194239671:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.778866Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435662056140281:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.778899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435662056140256:2086] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.779004Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435664105340326:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.779037Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435664105340318:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.779041Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435664105340318:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.779772Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435661691462404:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.780925Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435660194239668:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.781077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435664105340249:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.778840Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435663927496854:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.778903Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435663927496853:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.778940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435663927496788:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.781234Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435663927496861:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.781238Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435663927496861:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.790831Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435660194239674:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.790863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435660194239674:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.790971Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435660194239678:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.790978Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435660194239678:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.791776Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435660194239678:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:50.792951Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435659925969023:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.793122Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435664105340318:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:50.793363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435664105340314:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.793375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:50.793383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435664105340314:2108] [node 5] ICP22 created new session: [4:7574435664105340329:2108]
2025-11-19T13:58:50.793389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435664105340329:2108] [node 5] ICS09 handshake done sender: [4:7574435664105340318:2112] self: [4:7574435664105340317:0] peer: [5:7574435662056140291:0] socket: 116 qp: -1
2025-11-19T13:58:50.793397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435664105340329:2108] [node 5] ICS10 traffic start
2025-11-19T13:58:50.793407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:50.793434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435661691462356:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.794398Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435663927496861:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:50.794144Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435659925969021:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.794209Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435659925969001:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.794256Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435659925969024:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.794277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435662056140248:2084] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.794290Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435662056140248:2084] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.797130Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435664105340325:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.797168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435664105340247:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.798557Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435662056140248:2084] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:50.799997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435662056140292:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5220
2025-11-19T13:58:50.801104Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435662056140288:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.801131Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435662056140281:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:50.803265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435662056140256:2086] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.803286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:50.803296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435662056140256:2086] [node 4] ICP22 created new session: [5:7574435662056140294:2086]
2025-11-19T13:58:50.803303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435662056140294:2086] [node 4] ICS09 handshake done sender: [5:7574435662056140281:2111] self: [5:7574435662056140291:0] peer: [4:7574435664105340317:0] socket: 121 qp: -1
2025-11-19T13:58:50.803312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435662056140294:2086] [node 4] ICS10 traffic start
2025-11-19T13:58:50.803323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:50.809419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435662056140246:2083] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.809443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:50.809452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435662056140246:2083] [node 6] ICP22 created new session: [5:7574435662056140298:2083]
2025-11-19T13:58:50.809459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435662056140298:2083] [node 6] ICS09 handshake done sender: [5:7574435662056140248:2084] self: [5:7574435662056140247:0] peer: [6:7574435661691462411:0] socket: 104 qp: -1
2025-11-19T13:58:50.809465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435662056140298:2083] [node 6] ICS10 traffic start
2025-11-19T13:58:50.809474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:50.810933Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435661691462356:2066] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25620) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435661691462404:2112] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.808814Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435660194239670:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.808834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435660194239604:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.808855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:50.808866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435660194239604:2049] [node 3] ICP22 created new session: [2:7574435660194239688:2049]
2025-11-19T13:58:50.808874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435660194239688:2049] [node 3] ICS09 handshake done sender: [2:7574435660194239678:2115] self: [2:7574435660194239677:0] peer: [3:7574435663927496871:0] socket: 120 qp: -1
2025-11-19T13:58:50.808881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435660194239688:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:50.808892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.811920Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435663927496852:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.811946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435663927496855:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.811957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:50.811966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435663927496855:2108] [node 4] ICP22 created new session: [3:7574435663927496875:2108]
2025-11-19T13:58:50.811973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435663927496875:2108] [node 4] ICS09 handshake done sender: [3:7574435663927496861:2116] self: [3:7574435663927496860:0] peer: [4:7574435664105340327:0] socket: 129 qp: -1
2025-11-19T13:58:50.811980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435663927496875:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:50.811991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:50.812392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435663927496874:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19219
2025-11-19T13:58:50.812444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435663927496787:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.813100Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435659925969010:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.813116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435659925969010:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.812966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435660194239603:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.813066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435660194239690:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12128
2025-11-19T13:58:50.813179Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435659925969006:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.813187Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435659925969006:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.813274Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435663927496874:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.813231Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435660194239674:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:50.813321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435663927496873:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19219
2025-11-19T13:58:50.813412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435660194239667:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.813426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:50.813432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435660194239667:2108] [node 5] ICP22 created new session: [2:7574435660194239694:2108]
2025-11-19T13:58:50.813437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435660194239694:2108] [node 5] ICS09 handshake done sender: [2:7574435660194239674:2113] self: [2:7574435660194239673:0] peer: [5:7574435662056140290:0] socket: 118 qp: -1
2025-11-19T13:58:50.813443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435660194239694:2108] [node 5] ICS10 traffic start
2025-11-19T13:58:50.813451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:50.815390Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435660194239690:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.817266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435660194239667:2108] [node 5] ICP18 handshake (actor [2:7574435660194239690:2118]) is too old
2025-11-19T13:58:50.820619Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435663927496788:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24623) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435663927496853:2111] held: no
2025-11-19T13:58:50.820706Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435663927496851:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.821959Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435663927496853:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:50.822427Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435659925969008:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.822445Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435659925969008:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.823526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435660194239697:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12128
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.824081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435664105340333:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4929
2025-11-19T13:58:50.824587Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435659925969010:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.828485Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435659925969008:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.829123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435664105340335:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4929
2025-11-19T13:58:50.829169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435664105340336:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4929
2025-11-19T13:58:50.829435Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435660194239670:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:50.829790Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435660194239680:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.829796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435660194239680:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.830511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435660194239603:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.830528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:50.830534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435660194239603:2048] [node 1] ICP22 created new session: [2:7574435660194239698:2048]
2025-11-19T13:58:50.830540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435660194239698:2048] [node 1] ICS09 handshake done sender: [2:7574435660194239670:2111] self: [2:7574435660194239693:0] peer: [1:7574435659925969007:0] socket: 111 qp: -1
2025-11-19T13:58:50.830547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435660194239698:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:50.830555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:50.828594Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435662056140210:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24623) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435662056140282:2112] held: no
2025-11-19T13:58:50.828648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435662056140300:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5220
2025-11-19T13:58:50.828711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435662056140301:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5220
2025-11-19T13:58:50.828873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435661691462413:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-19T13:58:50.828923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435661691462414:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-19T13:58:50.829517Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435661691462414:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.829542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435661691462333:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.829689Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435661691462404:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:50.831461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435661691462356:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.831470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:50.831476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435661691462356:2066] [node 5] ICP22 created new session: [6:7574435661691462416:2066]
2025-11-19T13:58:50.831481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435661691462416:2066] [node 5] ICS09 handshake done sender: [6:7574435661691462404:2112] self: [6:7574435661691462411:0] peer: [5:7574435662056140247:0] socket: 130 qp: -1
2025-11-19T13:58:50.831486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435661691462416:2066] [node 5] ICS10 traffic start
2025-11-19T13:58:50.831493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:50.831780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435659925969003:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.831799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:50.831804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435659925969003:2108] [node 4] ICP22 created new session: [1:7574435659925969032:2108]
2025-11-19T13:58:50.831808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435659925969032:2108] [node 4] ICS09 handshake done sender: [1:7574435659925969010:2112] self: [1:7574435659925969009:0] peer: [4:7574435664105340332:0] socket: 109 qp: -1
2025-11-19T13:58:50.831812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435659925969032:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:50.831820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:50.831917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435659925968941:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.831926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:50.831929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435659925968941:2048] [node 2] ICP22 created new session: [1:7574435659925969035:2048]
2025-11-19T13:58:50.831933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435659925969035:2048] [node 2] ICS09 handshake done sender: [1:7574435659925969008:2111] self: [1:7574435659925969007:0] peer: [2:7574435660194239693:0] socket: 108 qp: -1
2025-11-19T13:58:50.831936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435659925969035:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:50.831940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:50.833404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435664105340334:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4929
2025-11-19T13:58:50.833494Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435664105340335:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.835019Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435664105340326:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:50.835102Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435664105340247:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30186) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435664105340325:2115] held: no
2025-11-19T13:58:50.835148Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435664105340334:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.835158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435664105340249:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.835169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:50.835174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435664105340249:2050] [node 3] ICP22 created new session: [4:7574435664105340337:2050]
2025-11-19T13:58:50.835178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435664105340337:2050] [node 3] ICS09 handshake done sender: [4:7574435664105340326:2116] self: [4:7574435664105340327:0] peer: [3:7574435663927496860:0] socket: 135 qp: -1
2025-11-19T13:58:50.835183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435664105340337:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:50.835190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:50.834303Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435662056140282:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:50.834682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435662056140210:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.834694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:50.834703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435662056140210:2049] [node 2] ICP22 created new session: [5:7574435662056140302:2049]
2025-11-19T13:58:50.834710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435662056140302:2049] [node 2] ICS09 handshake done sender: [5:7574435662056140282:2112] self: [5:7574435662056140290:0] peer: [2:7574435660194239673:0] socket: 122 qp: -1
2025-11-19T13:58:50.834716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435662056140302:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:50.834724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:50.834507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435661691462418:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-19T13:58:50.836170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435663927496855:2108] [node 4] ICP18 handshake (actor [3:7574435663927496874:2120]) is too old
2025-11-19T13:58:50.836348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435663927496788:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.836358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:50.836363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435663927496788:2049] [node 2] ICP22 created new session: [3:7574435663927496879:2049]
2025-11-19T13:58:50.836368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435663927496879:2049] [node 2] ICS09 handshake done sender: [3:7574435663927496853:2111] self: [3:7574435663927496871:0] peer: [2:7574435660194239677:0] socket: 126 qp: -1
2025-11-19T13:58:50.836374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435663927496879:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:50.836380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:50.836760Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435660194239680:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:50.837031Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435662056140300:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.837062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435662056140211:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.840043Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435663927496865:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.840055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435663927496865:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.840888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435663927496881:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19219
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.841878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435660194239672:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.841895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:50.841902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435660194239672:2110] [node 6] ICP22 created new session: [2:7574435660194239701:2110]
2025-11-19T13:58:50.841909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435660194239701:2110] [node 6] ICS09 handshake done sender: [2:7574435660194239680:2116] self: [2:7574435660194239679:0] peer: [6:7574435661691462415:0] socket: 151 qp: -1
2025-11-19T13:58:50.841916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435660194239701:2110] [node 6] ICS10 traffic start
2025-11-19T13:58:50.841925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.842812Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435659925969006:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:50.843123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435659925968942:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.843143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:50.843150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435659925968942:2049] [node 3] ICP22 created new session: [1:7574435659925969039:2049]
2025-11-19T13:58:50.843157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435659925969039:2049] [node 3] ICS09 handshake done sender: [1:7574435659925969006:2110] self: [1:7574435659925969005:0] peer: [3:7574435663927496878:0] socket: 107 qp: -1
2025-11-19T13:58:50.843163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435659925969039:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:50.843171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:50.841161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435662056140305:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5220
2025-11-19T13:58:50.842269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435662056140306:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5220
2025-11-19T13:58:50.841660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435661691462419:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-19T13:58:50.841680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435661691462420:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-19T13:58:50.841904Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435661691462414:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:50.842007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435661691462333:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.842021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:50.842029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435661691462333:2049] [node 2] ICP22 created new session: [6:7574435661691462421:2049]
2025-11-19T13:58:50.842035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435661691462421:2049] [node 2] ICS09 handshake done sender: [6:7574435661691462414:2117] self: [6:7574435661691462415:0] peer: [2:7574435660194239679:0] socket: 159 qp: -1
2025-11-19T13:58:50.842054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435661691462421:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:50.842063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:50.846645Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435663927496865:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:50.846868Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435662056140300:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:50.846929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435662056140211:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.846943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:50.846949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435662056140211:2050] [node 3] ICP22 created new session: [5:7574435662056140307:2050]
2025-11-19T13:58:50.846955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435662056140307:2050] [node 3] ICS09 handshake done sender: [5:7574435662056140300:2119] self: [5:7574435662056140304:0] peer: [3:7574435663927496864:0] socket: 150 qp: -1
2025-11-19T13:58:50.846962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435662056140307:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:50.846970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:50.848969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435663927496862:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.848993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:50.848999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435663927496862:2113] [node 5] ICP22 created new session: [3:7574435663927496885:2113]
2025-11-19T13:58:50.849004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435663927496885:2113] [node 5] ICS09 handshake done sender: [3:7574435663927496865:2117] self: [3:7574435663927496864:0] peer: [5:7574435662056140304:0] socket: 149 qp: -1
2025-11-19T13:58:50.849009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435663927496885:2113] [node 5] ICS10 traffic start
2025-11-19T13:58:50.849017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:50.849172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435663927496787:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30186) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435663927496852:2110] held: no
2025-11-19T13:58:50.854537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435664105340248:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.854669Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435664105340325:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:50.854778Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435660194239684:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.854795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435660194239684:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.856971Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435661691462420:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.857029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435661691462423:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-19T13:58:50.857044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435661691462332:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.856832Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435663927496852:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:50.857711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435664105340342:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4929
2025-11-19T13:58:50.858085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435664105340247:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.858126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:50.858133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435664105340247:2048] [node 1] ICP22 created new session: [4:7574435664105340343:2048]
2025-11-19T13:58:50.858141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435664105340343:2048] [node 1] ICS09 handshake done sender: [4:7574435664105340325:2115] self: [4:7574435664105340332:0] peer: [1:7574435659925969009:0] socket: 112 qp: -1
2025-11-19T13:58:50.858148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435664105340343:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:50.858157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:50.857272Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435660194239684:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:50.858496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435660194239664:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.858513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:50.858520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435660194239664:2107] [node 4] ICP22 created new session: [2:7574435660194239704:2107]
2025-11-19T13:58:50.858526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435660194239704:2107] [node 4] ICS09 handshake done sender: [2:7574435660194239684:2117] self: [2:7574435660194239665:0] peer: [4:7574435664105340339:0] socket: 152 qp: -1
2025-11-19T13:58:50.858532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435660194239704:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:50.858539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:50.858790Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435661691462419:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.858820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435661691462334:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.858929Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435659925969020:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.858942Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435659925969020:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.859108Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435661691462423:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.859133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435661691462399:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:50.859253Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435664105340322:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.859261Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435664105340322:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.860619Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435663927496867:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.860632Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435663927496867:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.861559Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435663927496867:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:50.861648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435663927496787:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.861664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:50.861671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435663927496787:2048] [node 1] ICP22 created new session: [3:7574435663927496888:2048]
2025-11-19T13:58:50.861676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435663927496888:2048] [node 1] ICS09 handshake done sender: [3:7574435663927496852:2110] self: [3:7574435663927496878:0] peer: [1:7574435659925969005:0] socket: 125 qp: -1
2025-11-19T13:58:50.861684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435663927496888:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:50.861693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:50.861784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435663927496863:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.861795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:50.861798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435663927496863:2115] [node 6] ICP22 created new session: [3:7574435663927496890:2115]
2025-11-19T13:58:50.861801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435663927496890:2115] [node 6] ICS09 handshake done sender: [3:7574435663927496867:2118] self: [3:7574435663927496866:0] peer: [6:7574435661691462425:0] socket: 131 qp: -1
2025-11-19T13:58:50.861805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435663927496890:2115] [node 6] ICS10 traffic start
2025-11-19T13:58:50.861810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:50.857838Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435662056140306:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.857882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435662056140209:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.863591Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435659925969013:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:50.863606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435659925969013:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:50.864163Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435659925969020:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.864454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435661691462427:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-19T13:58:50.864536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435661691462428:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-19T13:58:50.864727Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435661691462419:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:50.864936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435661691462334:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.864952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:50.864958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435661691462334:2050] [node 3] ICP22 created new session: [6:7574435661691462429:2050]
2025-11-19T13:58:50.864963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435661691462429:2050] [node 3] ICS09 handshake done sender: [6:7574435661691462419:2119] self: [6:7574435661691462425:0] peer: [3:7574435663927496866:0] socket: 162 qp: -1
2025-11-19T13:58:50.864970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435661691462429:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:50.864976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:50.866122Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435661691462420:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:50.867137Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435664105340335:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:50.867231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435664105340248:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24623) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435664105340335:2119] held: no
2025-11-19T13:58:50.867252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435664105340248:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.867260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:50.867265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435664105340248:2049] [node 2] ICP22 created new session: [4:7574435664105340346:2049]
2025-11-19T13:58:50.867270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435664105340346:2049] [node 2] ICS09 handshake done sender: [4:7574435664105340335:2119] self: [4:7574435664105340339:0] peer: [2:7574435660194239665:0] socket: 157 qp: -1
2025-11-19T13:58:50.867277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435664105340346:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:50.867284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.868997Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435659925969013:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:50.867853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435661691462332:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.867867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:50.867874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435661691462332:2048] [node 1] ICP22 created new session: [6:7574435661691462432:2048]
2025-11-19T13:58:50.867879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435661691462432:2048] [node 1] ICS09 handshake done sender: [6:7574435661691462420:2120] self: [6:7574435661691462424:0] peer: [1:7574435659925969019:0] socket: 163 qp: -1
2025-11-19T13:58:50.867884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435661691462432:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:50.867891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:50.869167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435659925969011:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.869184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:50.869189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435659925969011:2109] [node 5] ICP22 created new session: [1:7574435659925969043:2109]
2025-11-19T13:58:50.869194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435659925969043:2109] [node 5] ICS09 handshake done sender: [1:7574435659925969013:2113] self: [1:7574435659925969012:0] peer: [5:7574435662056140309:0] socket: 132 qp: -1
2025-11-19T13:58:50.869199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435659925969043:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:50.869206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:50.869650Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435664105340322:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:50.869751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435664105340315:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.869767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:50.869772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435664105340315:2110] [node 6] ICP22 created new session: [4:7574435664105340348:2110]
2025-11-19T13:58:50.869776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435664105340348:2110] [node 6] ICS09 handshake done sender: [4:7574435664105340322:2114] self: [4:7574435664105340321:0] peer: [6:7574435661691462426:0] socket: 123 qp: -1
2025-11-19T13:58:50.869780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435664105340348:2110] [node 6] ICS10 traffic start
2025-11-19T13:58:50.869786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:50.870043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435659925969018:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:50.870059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:50.870063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435659925969018:2114] [node 6] ICP22 created new session: [1:7574435659925969045:2114]
2025-11-19T13:58:50.870067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435659925969045:2114] [node 6] ICS09 handshake done sender: [1:7574435659925969020:2115] self: [1:7574435659925969019:0] peer: [6:7574435661691462424:0] socket: 147 qp: -1
2025-11-19T13:58:50.870080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435659925969045:2114] [node 6] ICS10 traffic start
2025-11-19T13:58:50.870086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:50.870913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435661691462431:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54326
2025-11-19T13:58:50.868835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435662056140310:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5220
2025-11-19T13:58:50.873258Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435662056140209:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30186) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435662056140306:2122] held: no
2025-11-19T13:58:50.873313Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435662056140306:2122] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:50.873404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435662056140209:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.873418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:50.873424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435662056140209:2048] [node 1] ICP22 created new session: [5:7574435662056140311:2048]
2025-11-19T13:58:50.873428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435662056140311:2048] [node 1] ICS09 handshake done sender: [5:7574435662056140306:2122] self: [5:7574435662056140309:0] peer: [1:7574435659925969012:0] socket: 164 qp: -1
2025-11-19T13:58:50.873434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435662056140311:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:50.873442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:50.875310Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435661691462399:2070] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16659) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435661691462423:2121] held: no
2025-11-19T13:58:50.875372Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435661691462423:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:50.876624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435661691462399:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:50.876643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:50.876652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435661691462399:2070] [node 4] ICP22 created new session: [6:7574435661691462434:2070]
2025-11-19T13:58:50.876658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435661691462434:2070] [node 4] ICS09 handshake done sender: [6:7574435661691462423:2121] self: [6:7574435661691462426:0] peer: [4:7574435664105340321:0] socket: 141 qp: -1
2025-11-19T13:58:50.876666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435661691462434:2070] [node 4] ICS10 traffic start
2025-11-19T13:58:50.876674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:50.881325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:50.881413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435660194239698:2048] [node 1] ICS01 socket: 111 reason# EndOfStream
2025-11-19T13:58:50.881422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435660194239603:2048] [node 1] ICP30 unregister session Session# [2:7574435660194239698:2048] VirtualId# [2:7574435660194239693:0]
2025-11-19T13:58:50.881429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435660194239698:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.881459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:50.881479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:50.884388Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:50.884469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435663927496888:2048] [node 1] ICS01 socket: 125 reason# EndOfStream
2025-11-19T13:58:50.884476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435663927496787:2048] [node 1] ICP30 unregister session Session# [3:7574435663927496888:2048] VirtualId# [3:7574435663927496878:0]
2025-11-19T13:58:50.884484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435663927496888:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.884517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:50.884536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:50.887698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:50.887799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435664105340343:2048] [node 1] ICS01 socket: 112 reason# EndOfStream
2025-11-19T13:58:50.887806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435664105340247:2048] [node 1] ICP30 unregister session Session# [4:7574435664105340343:2048] VirtualId# [4:7574435664105340332:0]
2025-11-19T13:58:50.887813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435664105340343:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.887862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:50.887884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:50.887917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:50.887978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435662056140311:2048] [node 1] ICS01 socket: 164 reason# EndOfStream
2025-11-19T13:58:50.887982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435662056140209:2048] [node 1] ICP30 unregister session Session# [5:7574435662056140311:2048] VirtualId# [5:7574435662056140309:0]
2025-11-19T13:58:50.887986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435662056140311:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.888014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:50.888033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:50.891344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:50.891423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435661691462432:2048] [node 1] ICS01 socket: 163 reason# EndOfStream
2025-11-19T13:58:50.891430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435661691462332:2048] [node 1] ICP30 unregister session Session# [6:7574435661691462432:2048] VirtualId# [6:7574435661691462424:0]
2025-11-19T13:58:50.891436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435661691462432:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.891487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:50.891512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:50.892386Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:50.892457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435663927496879:2049] [node 2] ICS01 socket: 126 reason# EndOfStream
2025-11-19T13:58:50.892463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435663927496788:2049] [node 2] ICP30 unregister session Session# [3:7574435663927496879:2049] VirtualId# [3:7574435663927496871:0]
2025-11-19T13:58:50.892467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435663927496879:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.892495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:50.892511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:50.902465Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:50.902566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435661691462421:2049] [node 2] ICS01 socket: 159 reason# EndOfStream
2025-11-19T13:58:50.902575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435661691462333:2049] [node 2] ICP30 unregister session Session# [6:7574435661691462421:2049] VirtualId# [6:7574435661691462415:0]
2025-11-19T13:58:50.902582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435661691462421:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.902628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:50.902651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:50.905849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:50.905939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435664105340346:2049] [node 2] ICS01 socket: 157 reason# EndOfStream
2025-11-19T13:58:50.905946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435664105340248:2049] [node 2] ICP30 unregister session Session# [4:7574435664105340346:2049] VirtualId# [4:7574435664105340339:0]
2025-11-19T13:58:50.905952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435664105340346:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.906002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:50.906030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:50.906264Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:50.906327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435662056140302:2049] [node 2] ICS01 socket: 122 reason# EndOfStream
2025-11-19T13:58:50.906333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435662056140210:2049] [node 2] ICP30 unregister session Session# [5:7574435662056140302:2049] VirtualId# [5:7574435662056140290:0]
2025-11-19T13:58:50.906337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435662056140302:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.906368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:50.906387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:50.916173Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:50.916283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435661691462429:2050] [node 3] ICS01 socket: 162 reason# EndOfStream
2025-11-19T13:58:50.916293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435661691462334:2050] [node 3] ICP30 unregister session Session# [6:7574435661691462429:2050] VirtualId# [6:7574435661691462425:0]
2025-11-19T13:58:50.916300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435661691462429:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.916357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:50.916382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:50.917027Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:50.917100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435662056140307:2050] [node 3] ICS01 socket: 150 reason# EndOfStream
2025-11-19T13:58:50.917107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435662056140211:2050] [node 3] ICP30 unregister session Session# [5:7574435662056140307:2050] VirtualId# [5:7574435662056140304:0]
2025-11-19T13:58:50.917113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435662056140307:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.917157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:50.917179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:50.928145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:50.928240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435661691462434:2070] [node 4] ICS01 socket: 141 reason# EndOfStream
2025-11-19T13:58:50.928251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435661691462399:2070] [node 4] ICP30 unregister session Session# [6:7574435661691462434:2070] VirtualId# [6:7574435661691462426:0]
2025-11-19T13:58:50.928259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435661691462434:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:50.928316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:50.928341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:51.148922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435667808692936:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:51.148941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435667808692937:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:51.244218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435666849062546:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:51.244234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435666849062547:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:51.292525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435667085552254:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:51.292540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435667085552255:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:51.336425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435665981730879:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:51.336441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435665981730880:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:51.336446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435665981730881:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:51.367886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435667858744730:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:51.367896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435667858744731:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:51.367902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435667858744732:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:51.459828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435664217351935:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:51.464622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435664217351937:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:51.468006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435664217351936:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:51.518359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435664217351953:2064] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.519332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435665981730940:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13849
2025-11-19T13:58:51.521191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435664217351964:2070] [node 5] ICP01 ready to work
2025-11-19T13:58:51.528636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435667858744794:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:51.529880Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435665981730940:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.530071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435665981730942:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:51.530459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435665981730942:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.534289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435667808692998:2107] [node 5] ICP01 ready to work
2025-11-19T13:58:51.534477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435667808693002:2109] [node 4] ICP01 ready to work
2025-11-19T13:58:51.535198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435667808693000:2108] [node 6] ICP01 ready to work
2025-11-19T13:58:51.540210Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435664217351935:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17653449ns) {Configure:104(17651809ns) {ProcessConfigured:135(17340513ns) {StartInitialHandshake:157(17339796ns) {DropHandshakes:469(756ns) {DropIncomingHandshake:441(78ns) DropOutgoingHandshake:456(201ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(1442ns) {SwitchToState:178(618ns)}}}}}
2025-11-19T13:58:51.540677Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435664217351955:2065] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.540683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435664217351955:2065] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:51.543678Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435667858744732:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(22569884ns) {Configure:104(22567954ns) {ProcessConfigured:135(22185302ns) {StartInitialHandshake:157(22184729ns) {DropHandshakes:469(1227ns) {DropIncomingHandshake:441(252ns) DropOutgoingHandshake:456(227ns)} GenerateSessionVirtualId:988(170ns) PrepareNewSessionHandshake:476(1990ns) {SwitchToState:178(1292ns)}}}}}
2025-11-19T13:58:51.544053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435667858744798:2105] [node 4] ICP01 ready to work
2025-11-19T13:58:51.548023Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435664217351955:2065] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.549241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435667808693010:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48165
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.551415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435667858744802:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.552915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435666849062610:2107] [node 6] ICP01 ready to work
2025-11-19T13:58:51.553031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435666849062611:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:51.553639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435666849062609:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50964
2025-11-19T13:58:51.549545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435664217351964:2070] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(26684759ns) {Configure:104(26682703ns) {ProcessConfigured:135(26460787ns) {StartInitialHandshake:157(26460618ns) {DropHandshakes:469(212ns) {DropIncomingHandshake:441(32ns) DropOutgoingHandshake:456(27ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(644ns) {SwitchToState:178(191ns)}}}}}
2025-11-19T13:58:51.551334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435667808692937:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(16091733ns) {Configure:104(16091085ns) {ProcessConfigured:135(15879104ns) {StartInitialHandshake:157(15878852ns) {DropHandshakes:469(390ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(763ns) {SwitchToState:178(588ns)}}}}}
2025-11-19T13:58:51.554711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435664217351953:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.554729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:51.554735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435664217351953:2064] [node 4] ICP22 created new session: [6:7574435664217352012:2064]
2025-11-19T13:58:51.554741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435664217352012:2064] [node 4] ICS09 handshake done sender: [6:7574435664217351955:2065] self: [6:7574435664217351954:0] peer: [4:7574435665981730943:0] socket: 108 qp: -1
2025-11-19T13:58:51.554749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435664217352012:2064] [node 4] ICS10 traffic start
2025-11-19T13:58:51.554759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:51.556548Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435667808693010:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.559949Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435666849062609:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.563274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435667858744798:2105] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19144084ns) {Configure:104(19143179ns) {ProcessConfigured:135(18820096ns) {StartInitialHandshake:157(18819491ns) {DropHandshakes:469(946ns) {DropIncomingHandshake:441(135ns) DropOutgoingHandshake:456(147ns)} GenerateSessionVirtualId:988(241ns) PrepareNewSessionHandshake:476(1884ns) {SwitchToState:178(1015ns)}}}}}
2025-11-19T13:58:51.563584Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435667808692936:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(39746800ns) {Configure:104(39745325ns) {ProcessConfigured:135(39372712ns) {StartInitialHandshake:157(39372276ns) {DropHandshakes:469(616ns) {DropIncomingHandshake:441(128ns) DropOutgoingHandshake:456(90ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:476(1496ns) {SwitchToState:178(892ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.563997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435665981730946:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13849
2025-11-19T13:58:51.564098Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435666849062546:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16786381ns) {Configure:104(16784611ns) {ProcessConfigured:135(16368909ns) {StartInitialHandshake:157(16368125ns) {DropHandshakes:469(811ns) {DropIncomingHandshake:441(237ns) DropOutgoingHandshake:456(145ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(771ns) {SwitchToState:178(493ns)}}}}}
2025-11-19T13:58:51.567892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435664217352016:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55113
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.572102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435664217352018:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55113
2025-11-19T13:58:51.572126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435664217352019:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55113
2025-11-19T13:58:51.570329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435666849062623:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.577562Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435665981730879:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17147780ns) {Configure:104(17147110ns) {ProcessConfigured:135(16853247ns) {StartInitialHandshake:157(16852743ns) {DropHandshakes:469(654ns) {DropIncomingHandshake:441(109ns) DropOutgoingHandshake:456(151ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:476(1297ns) {SwitchToState:178(780ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.577614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435665981730947:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13849
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.578726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435667085552317:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:51.578940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435667085552324:2112] [node 5] ICP01 ready to work
2025-11-19T13:58:51.577632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435665981730948:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13849
2025-11-19T13:58:51.577697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435665981730949:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13849
2025-11-19T13:58:51.577741Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435665981730940:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:51.577989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435665981730880:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17252493ns) {Configure:104(17251734ns) {ProcessConfigured:135(17030488ns) {StartInitialHandshake:157(17030032ns) {DropHandshakes:469(143ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(15ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:476(1107ns) {SwitchToState:178(102ns)}}}}}
2025-11-19T13:58:51.579414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435667085552316:2108] [node 6] ICP01 ready to work
2025-11-19T13:58:51.580213Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435664217352016:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.580267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435664217351964:2070] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.583940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435665981730942:2109] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.583942Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435665981730949:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.583963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:51.583970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435665981730942:2109] [node 6] ICP22 created new session: [4:7574435665981730954:2109]
2025-11-19T13:58:51.583977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435665981730954:2109] [node 6] ICS09 handshake done sender: [4:7574435665981730940:2108] self: [4:7574435665981730943:0] peer: [6:7574435664217351954:0] socket: 109 qp: -1
2025-11-19T13:58:51.583986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435665981730954:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:51.584005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:51.584353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435665981730957:2116] [node 5] ICP01 ready to work
2025-11-19T13:58:51.584666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435665981730957:2116] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.582766Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435667085552255:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(18037978ns) {Configure:104(18037108ns) {ProcessConfigured:135(17826423ns) {StartInitialHandshake:157(17826077ns) {DropHandshakes:469(257ns) {DropIncomingHandshake:441(30ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:476(1447ns) {SwitchToState:178(64ns)}}}}}
2025-11-19T13:58:51.583181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435667085552318:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12821
2025-11-19T13:58:51.583271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435667085552319:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12821
2025-11-19T13:58:51.583570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435666849062627:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50964
2025-11-19T13:58:51.587096Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435667858744803:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.587111Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435667858744803:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:51.587981Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435664217352019:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.588105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435664217351936:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.588351Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435667858744803:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:51.588448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435667858744798:2105] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.588463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:51.588473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435667858744798:2105] [node 4] ICP22 created new session: [5:7574435667858744806:2105]
2025-11-19T13:58:51.588480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435667858744806:2105] [node 4] ICS09 handshake done sender: [5:7574435667858744803:2113] self: [5:7574435667858744801:0] peer: [4:7574435665981730958:0] socket: 128 qp: -1
2025-11-19T13:58:51.588486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435667858744806:2105] [node 4] ICS10 traffic start
2025-11-19T13:58:51.588519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:51.589112Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435667085552318:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.589145Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435667085552319:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.592101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435667808693017:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48165
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.592150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435667808693018:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48165
2025-11-19T13:58:51.592181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435667808693019:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48165
2025-11-19T13:58:51.592406Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435665981730948:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.592645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435665981730880:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.595778Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435667858744796:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.595806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435667858744796:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:51.596067Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435666849062613:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.596090Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435666849062613:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:51.596965Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435667085552324:2112] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17482579ns) {Configure:104(17481714ns) {ProcessConfigured:135(17273398ns) {StartInitialHandshake:157(17272839ns) {DropHandshakes:469(264ns) {DropIncomingHandshake:441(24ns) DropOutgoingHandshake:456(49ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:476(1615ns) {SwitchToState:178(1194ns)}}}}}
2025-11-19T13:58:51.597786Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435667808693017:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.600836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435666849062628:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50964
2025-11-19T13:58:51.601270Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435666849062615:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.601277Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435666849062615:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.602171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435665981730961:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13849
2025-11-19T13:58:51.602279Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435667808693018:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.602614Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435665981730947:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.602633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435665981730879:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.602721Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435667808693009:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.602726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435667808693009:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:51.603296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435667808693007:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.603299Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435667808693007:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:51.604105Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435667808693009:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:51.604124Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435667808693007:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:51.604250Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435666849062615:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:51.604314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435667808693000:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.604337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:51.604345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435667808693000:2108] [node 6] ICP22 created new session: [1:7574435667808693024:2108]
2025-11-19T13:58:51.604351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435667808693024:2108] [node 6] ICS09 handshake done sender: [1:7574435667808693009:2112] self: [1:7574435667808693008:0] peer: [6:7574435664217352023:0] socket: 122 qp: -1
2025-11-19T13:58:51.604359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435667808693024:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:51.604379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.607014Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435665981730948:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.601087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435664217352022:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55113
2025-11-19T13:58:51.602486Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435664217352018:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.602510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435664217351935:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.605646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435664217351935:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9660) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435664217352018:2116] held: no
2025-11-19T13:58:51.605708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435664217352024:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55113
2025-11-19T13:58:51.605922Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435664217352018:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:51.606025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435664217351935:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.606036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:51.606044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435664217351935:2048] [node 1] ICP22 created new session: [6:7574435664217352025:2048]
2025-11-19T13:58:51.606050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435664217352025:2048] [node 1] ICS09 handshake done sender: [6:7574435664217352018:2116] self: [6:7574435664217352023:0] peer: [1:7574435667808693008:0] socket: 127 qp: -1
2025-11-19T13:58:51.606059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435664217352025:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:51.606066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:51.607965Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435667858744802:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.608653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435667858744730:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.608692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435667858744808:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.609830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435666849062611:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.609850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:51.609857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435666849062611:2108] [node 4] ICP22 created new session: [2:7574435666849062630:2108]
2025-11-19T13:58:51.609863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435666849062630:2108] [node 4] ICS09 handshake done sender: [2:7574435666849062615:2111] self: [2:7574435666849062614:0] peer: [4:7574435665981730960:0] socket: 125 qp: -1
2025-11-19T13:58:51.609872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435666849062630:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:51.609881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:51.610380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435665981730880:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.610402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:51.610408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435665981730880:2049] [node 2] ICP22 created new session: [4:7574435665981730966:2049]
2025-11-19T13:58:51.610415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435665981730966:2049] [node 2] ICS09 handshake done sender: [4:7574435665981730948:2112] self: [4:7574435665981730960:0] peer: [2:7574435666849062614:0] socket: 130 qp: -1
2025-11-19T13:58:51.610421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435665981730966:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:51.610440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:51.610794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435665981730959:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13849
2025-11-19T13:58:51.611829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435665981730963:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13849
2025-11-19T13:58:51.615220Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435667858744808:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.615899Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435667858744796:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:51.616032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435667858744794:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.616053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:51.616061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435667858744794:2109] [node 6] ICP22 created new session: [5:7574435667858744817:2109]
2025-11-19T13:58:51.616068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435667858744817:2109] [node 6] ICS09 handshake done sender: [5:7574435667858744796:2110] self: [5:7574435667858744795:0] peer: [6:7574435664217352020:0] socket: 111 qp: -1
2025-11-19T13:58:51.616074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435667858744817:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:51.616083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.616580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435667808693002:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.616610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:51.616617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435667808693002:2109] [node 4] ICP22 created new session: [1:7574435667808693027:2109]
2025-11-19T13:58:51.616624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435667808693027:2109] [node 4] ICS09 handshake done sender: [1:7574435667808693007:2111] self: [1:7574435667808693006:0] peer: [4:7574435665981730962:0] socket: 121 qp: -1
2025-11-19T13:58:51.616630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435667808693027:2109] [node 4] ICS10 traffic start
2025-11-19T13:58:51.616638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:51.616729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435667858744819:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:599
2025-11-19T13:58:51.617702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435667858744798:2105] [node 4] ICP35 event processing took too much time PendingConnection:278(29230419ns) {HandleHandshakeStatus:329(29229141ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(481ns) SwitchToState:178(208ns) ProcessPendingSessionEvents:495(21717318ns) {ForwardSessionEventToSession:612(21711309ns) ForwardSessionEventToSession:612(1874ns) ForwardSessionEventToSession:612(1040ns)}}}
2025-11-19T13:58:51.618150Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435667808693005:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.618170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435667808693005:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:51.618000Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435666849062613:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:51.624162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435667085552317:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(45161572ns) {Configure:104(45160723ns) {ProcessConfigured:135(44834233ns) {StartInitialHandshake:157(44833640ns) {DropHandshakes:469(804ns) {DropIncomingHandshake:441(146ns) DropOutgoingHandshake:456(141ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(1801ns) {SwitchToState:178(1104ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.627115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435666849062611:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(17251806ns) {HandleHandshakeStatus:329(17250832ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(136ns) SwitchToState:178(183ns) ProcessPendingSessionEvents:495(16044617ns) {ForwardSessionEventToSession:612(16028672ns) ForwardSessionEventToSession:612(2129ns) ForwardSessionEventToSession:612(1760ns) ForwardSessionEventToSession:612(860ns) ForwardSessionEventToSession:612(901ns) ForwardSessionEventToSession:612(1199ns) ForwardSessionEventToSession:612(651ns) ForwardSessionEventToSession:612(874ns) ForwardSessionEventToSession:612(914ns) ForwardSessionEventToSession:612(1092ns)}}}
2025-11-19T13:58:51.627154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435666849062610:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.627175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:51.627183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435666849062610:2107] [node 6] ICP22 created new session: [2:7574435666849062634:2107]
2025-11-19T13:58:51.627188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435666849062634:2107] [node 6] ICS09 handshake done sender: [2:7574435666849062613:2110] self: [2:7574435666849062612:0] peer: [6:7574435664217352021:0] socket: 124 qp: -1
2025-11-19T13:58:51.627195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435666849062634:2107] [node 6] ICS10 traffic start
2025-11-19T13:58:51.627205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:51.627908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435667808693029:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48165
2025-11-19T13:58:51.630626Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435667808693019:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.631136Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435666849062628:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.632087Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435667858744819:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.632128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435667858744731:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.632225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435666849062625:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.632234Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435666849062625:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.632937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435667085552337:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12821
2025-11-19T13:58:51.632985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435667085552338:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12821
2025-11-19T13:58:51.633258Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435667085552337:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.633292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435667085552254:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.633430Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435667808693014:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.633439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435667808693014:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.634460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435667085552340:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12821
2025-11-19T13:58:51.634615Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435667808693014:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:51.634806Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435667085552337:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:51.634916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435667085552254:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.634936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:51.634941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435667085552254:2048] [node 1] ICP22 created new session: [3:7574435667085552341:2048]
2025-11-19T13:58:51.634947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435667085552341:2048] [node 1] ICS09 handshake done sender: [3:7574435667085552337:2117] self: [3:7574435667085552339:0] peer: [1:7574435667808693004:0] socket: 156 qp: -1
2025-11-19T13:58:51.634954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435667085552341:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:51.634960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:51.637519Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435666849062625:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:51.637559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435667808692937:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.637583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:51.637590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435667808692937:2049] [node 3] ICP22 created new session: [1:7574435667808693033:2049]
2025-11-19T13:58:51.637596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435667808693033:2049] [node 3] ICS09 handshake done sender: [1:7574435667808693014:2114] self: [1:7574435667808693004:0] peer: [3:7574435667085552339:0] socket: 119 qp: -1
2025-11-19T13:58:51.637602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435667808693033:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:51.637610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:51.637553Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435664217351964:2070] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435664217352016:2114] held: no
2025-11-19T13:58:51.638985Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435664217351936:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5319) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435664217352019:2117] held: no
2025-11-19T13:58:51.639045Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435664217352022:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.651055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435666849062623:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.651085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:51.651099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435666849062623:2113] [node 5] ICP22 created new session: [2:7574435666849062639:2113]
2025-11-19T13:58:51.651105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435666849062639:2113] [node 5] ICS09 handshake done sender: [2:7574435666849062625:2114] self: [2:7574435666849062624:0] peer: [5:7574435667858744821:0] socket: 161 qp: -1
2025-11-19T13:58:51.651112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435666849062639:2113] [node 5] ICS10 traffic start
2025-11-19T13:58:51.651122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:51.651301Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435667085552338:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.651347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435667085552255:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.651525Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435667808693005:2110] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:51.651455Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435666849062626:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.651466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435666849062626:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:51.651675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435667808692998:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.651686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:51.651695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435667808692998:2107] [node 5] ICP22 created new session: [1:7574435667808693036:2107]
2025-11-19T13:58:51.651700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435667808693036:2107] [node 5] ICS09 handshake done sender: [1:7574435667808693005:2110] self: [1:7574435667808693003:0] peer: [5:7574435667858744813:0] socket: 120 qp: -1
2025-11-19T13:58:51.651705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435667808693036:2107] [node 5] ICS10 traffic start
2025-11-19T13:58:51.651712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.652439Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435665981730880:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(42026275ns) {HandleHandshakeStatus:329(42025348ns) {DropIncomingHandshake:441(180ns) DropOutgoingHandshake:456(2078ns) SwitchToState:178(83ns) ProcessPendingSessionEvents:495(34027540ns) {ForwardSessionEventToSession:612(34015747ns) ForwardSessionEventToSession:612(2241ns) ForwardSessionEventToSession:612(778ns) ForwardSessionEventToSession:612(1119ns) ForwardSessionEventToSession:612(629ns) ForwardSessionEventToSession:612(1481ns) ForwardSessionEventToSession:612(664ns) ForwardSessionEventToSession:612(719ns) ForwardSessionEventToSession:612(648ns)}}}
2025-11-19T13:58:51.652472Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435665981730949:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:51.652622Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435665981730947:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:51.652688Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435665981730879:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9660) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435665981730947:2111] held: no
2025-11-19T13:58:51.652710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435665981730879:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.652729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:51.652735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435665981730879:2048] [node 1] ICP22 created new session: [4:7574435665981730968:2048]
2025-11-19T13:58:51.652740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435665981730968:2048] [node 1] ICS09 handshake done sender: [4:7574435665981730947:2111] self: [4:7574435665981730962:0] peer: [1:7574435667808693006:0] socket: 129 qp: -1
2025-11-19T13:58:51.652746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435665981730968:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:51.652755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:51.652951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435665981730957:2116] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.652957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:51.652961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435665981730957:2116] [node 5] ICP22 created new session: [4:7574435665981730971:2116]
2025-11-19T13:58:51.652966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435665981730971:2116] [node 5] ICS09 handshake done sender: [4:7574435665981730949:2113] self: [4:7574435665981730958:0] peer: [5:7574435667858744801:0] socket: 131 qp: -1
2025-11-19T13:58:51.652970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435665981730971:2116] [node 5] ICS10 traffic start
2025-11-19T13:58:51.652975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:51.652171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435666849062641:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50964
2025-11-19T13:58:51.652233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435666849062642:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50964
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.655984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435667858744823:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:599
2025-11-19T13:58:51.656068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435667858744822:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:599
2025-11-19T13:58:51.656396Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435667858744819:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:51.656124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435667085552345:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12821
2025-11-19T13:58:51.656142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435667085552346:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12821
2025-11-19T13:58:51.656482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435667858744731:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.656501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:51.656508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435667858744731:2049] [node 2] ICP22 created new session: [5:7574435667858744825:2049]
2025-11-19T13:58:51.656514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435667858744825:2049] [node 2] ICS09 handshake done sender: [5:7574435667858744819:2117] self: [5:7574435667858744821:0] peer: [2:7574435666849062624:0] socket: 154 qp: -1
2025-11-19T13:58:51.656527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435667858744825:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:51.656536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:51.656543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435667858744824:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:599
2025-11-19T13:58:51.656791Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435666849062626:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:51.656982Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435667085552346:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.657056Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435667085552338:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.660247Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435666849062641:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.660345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435666849062611:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:51.660366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435666849062623:2113] [node 5] ICP18 handshake (actor [2:7574435666849062641:2118]) is too old
2025-11-19T13:58:51.660255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435664217352028:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55113
2025-11-19T13:58:51.660398Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435667808693029:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.661157Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435666849062642:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.661170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435666849062547:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.661186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:51.661193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435666849062547:2049] [node 3] ICP22 created new session: [2:7574435666849062644:2049]
2025-11-19T13:58:51.661199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435666849062644:2049] [node 3] ICS09 handshake done sender: [2:7574435666849062626:2115] self: [2:7574435666849062618:0] peer: [3:7574435667085552344:0] socket: 113 qp: -1
2025-11-19T13:58:51.661204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435666849062644:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:51.661212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.661387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435664217352029:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55113
2025-11-19T13:58:51.661496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435664217351937:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.661514Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435664217352016:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:51.664010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435664217351964:2070] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.664021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:51.664026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435664217351964:2070] [node 5] ICP22 created new session: [6:7574435664217352031:2070]
2025-11-19T13:58:51.664032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435664217352031:2070] [node 5] ICS09 handshake done sender: [6:7574435664217352016:2114] self: [6:7574435664217352020:0] peer: [5:7574435667858744795:0] socket: 112 qp: -1
2025-11-19T13:58:51.664039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435664217352031:2070] [node 5] ICS10 traffic start
2025-11-19T13:58:51.664047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:51.664141Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435664217352019:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:51.664289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435664217351936:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.664297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:51.664302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435664217351936:2049] [node 2] ICP22 created new session: [6:7574435664217352033:2049]
2025-11-19T13:58:51.664306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435664217352033:2049] [node 2] ICS09 handshake done sender: [6:7574435664217352019:2117] self: [6:7574435664217352021:0] peer: [2:7574435666849062612:0] socket: 135 qp: -1
2025-11-19T13:58:51.664310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435664217352033:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:51.664315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:51.666382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435667808692998:2107] [node 5] ICP18 handshake (actor [1:7574435667808693029:2119]) is too old
2025-11-19T13:58:51.666721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435667085552255:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.666740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:51.666746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435667085552255:2049] [node 2] ICP22 created new session: [3:7574435667085552348:2049]
2025-11-19T13:58:51.666753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435667085552348:2049] [node 2] ICS09 handshake done sender: [3:7574435667085552338:2118] self: [3:7574435667085552344:0] peer: [2:7574435666849062618:0] socket: 165 qp: -1
2025-11-19T13:58:51.666758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435667085552348:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:51.666777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:51.666918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435667085552329:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.666922Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435667085552329:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:51.673482Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435667858744802:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:51.674317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435667858744730:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.674339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:51.674347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435667858744730:2048] [node 1] ICP22 created new session: [5:7574435667858744827:2048]
2025-11-19T13:58:51.674353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435667858744827:2048] [node 1] ICS09 handshake done sender: [5:7574435667858744802:2112] self: [5:7574435667858744813:0] peer: [1:7574435667808693003:0] socket: 123 qp: -1
2025-11-19T13:58:51.674361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435667858744827:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:51.674370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:51.681765Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435667858744731:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(25239304ns) {HandleHandshakeStatus:329(25238321ns) {DropIncomingHandshake:441(364ns) DropOutgoingHandshake:456(1834ns) SwitchToState:178(176ns) ProcessPendingSessionEvents:495(25126779ns) {ForwardSessionEventToSession:612(25109561ns) ForwardSessionEventToSession:612(2083ns) ForwardSessionEventToSession:612(528ns) ForwardSessionEventToSession:612(1190ns) ForwardSessionEventToSession:612(599ns) ForwardSessionEventToSession:612(942ns) ForwardSessionEventToSession:612(755ns) ForwardSessionEventToSession:612(606ns) ForwardSessionEventToSession:612(699ns) ForwardSessionEventToSession:612(1223ns)}}}
2025-11-19T13:58:51.682139Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435667858744823:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.682195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435667858744732:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.684042Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435667085552329:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:51.684119Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435667085552333:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.684123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435667085552333:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.686301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435664217352035:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55113
2025-11-19T13:58:51.686728Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435664217352022:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:51.686842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435664217351937:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.686862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:51.686870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435664217351937:2050] [node 3] ICP22 created new session: [6:7574435664217352036:2050]
2025-11-19T13:58:51.686877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435664217352036:2050] [node 3] ICS09 handshake done sender: [6:7574435664217352022:2118] self: [6:7574435664217352030:0] peer: [3:7574435667085552328:0] socket: 144 qp: -1
2025-11-19T13:58:51.686883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435664217352036:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:51.686893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.688364Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435667085552333:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:51.688346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435667858744830:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:599
2025-11-19T13:58:51.688474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435667085552316:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.688493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:51.688499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435667085552316:2108] [node 6] ICP22 created new session: [3:7574435667085552353:2108]
2025-11-19T13:58:51.688505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435667085552353:2108] [node 6] ICS09 handshake done sender: [3:7574435667085552329:2114] self: [3:7574435667085552328:0] peer: [6:7574435664217352030:0] socket: 141 qp: -1
2025-11-19T13:58:51.688510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435667085552353:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:51.688530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:51.688674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435667085552324:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.688689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:51.688695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435667085552324:2112] [node 5] ICP22 created new session: [3:7574435667085552355:2112]
2025-11-19T13:58:51.688699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435667085552355:2112] [node 5] ICS09 handshake done sender: [3:7574435667085552333:2115] self: [3:7574435667085552326:0] peer: [5:7574435667858744829:0] socket: 169 qp: -1
2025-11-19T13:58:51.688704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435667085552355:2112] [node 5] ICS10 traffic start
2025-11-19T13:58:51.688710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:51.689029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435666849062546:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.689330Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435667858744823:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:51.689212Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435667808693016:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.689225Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435667808693016:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:51.690195Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435667808693016:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:51.690365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435667808692936:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.690381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:51.690410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435667808692936:2048] [node 2] ICP22 created new session: [1:7574435667808693039:2048]
2025-11-19T13:58:51.690416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435667808693039:2048] [node 2] ICS09 handshake done sender: [1:7574435667808693016:2115] self: [1:7574435667808692996:0] peer: [2:7574435666849062646:0] socket: 163 qp: -1
2025-11-19T13:58:51.690422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435667808693039:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:51.690439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:51.688135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435665981730973:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13849
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.693980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435667858744732:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.694002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:51.694012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435667858744732:2050] [node 3] ICP22 created new session: [5:7574435667858744831:2050]
2025-11-19T13:58:51.694019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435667858744831:2050] [node 3] ICS09 handshake done sender: [5:7574435667858744823:2119] self: [5:7574435667858744829:0] peer: [3:7574435667085552326:0] socket: 136 qp: -1
2025-11-19T13:58:51.694026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435667858744831:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:51.694036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:51.695172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435666849062647:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50964
2025-11-19T13:58:51.696562Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435666849062642:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:51.697173Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435665981730973:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.697217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435665981730881:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:51.697377Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435667085552336:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:51.697389Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435667085552336:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:51.698059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435665981730975:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13849
2025-11-19T13:58:51.698201Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435667085552336:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:51.698364Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435665981730973:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:51.698466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435665981730881:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.698485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:51.698491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435665981730881:2050] [node 3] ICP22 created new session: [4:7574435665981730976:2050]
2025-11-19T13:58:51.698497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435665981730976:2050] [node 3] ICS09 handshake done sender: [4:7574435665981730973:2121] self: [4:7574435665981730974:0] peer: [3:7574435667085552325:0] socket: 174 qp: -1
2025-11-19T13:58:51.698503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435665981730976:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:51.698509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:51.699528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435667085552317:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:51.699547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:51.699552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435667085552317:2107] [node 4] ICP22 created new session: [3:7574435667085552358:2107]
2025-11-19T13:58:51.699557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435667085552358:2107] [node 4] ICS09 handshake done sender: [3:7574435667085552336:2116] self: [3:7574435667085552325:0] peer: [4:7574435665981730974:0] socket: 145 qp: -1
2025-11-19T13:58:51.699561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435667085552358:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:51.699567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:51.701213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435666849062546:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:51.701251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:51.701260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435666849062546:2048] [node 1] ICP22 created new session: [2:7574435666849062648:2048]
2025-11-19T13:58:51.701265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435666849062648:2048] [node 1] ICS09 handshake done sender: [2:7574435666849062642:2119] self: [2:7574435666849062646:0] peer: [1:7574435667808692996:0] socket: 115 qp: -1
2025-11-19T13:58:51.701273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435666849062648:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:51.701282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:51.701372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435666849062546:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:51.707864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435667808692936:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(17449558ns) {HandleHandshakeStatus:329(17448204ns) {DropIncomingHandshake:441(4157ns) DropOutgoingHandshake:456(82ns) SwitchToState:178(203ns) ProcessPendingSessionEvents:495(17341763ns) {ForwardSessionEventToSession:612(17327196ns) ForwardSessionEventToSession:612(2900ns) ForwardSessionEventToSession:612(855ns) ForwardSessionEventToSession:612(1629ns) ForwardSessionEventToSession:612(869ns) ForwardSessionEventToSession:612(468ns) ForwardSessionEventToSession:612(1259ns) ForwardSessionEventToSession:612(653ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(1041ns)}}}
2025-11-19T13:58:51.741376Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:51.741479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435666849062648:2048] [node 1] ICS01 socket: 115 reason# EndOfStream
2025-11-19T13:58:51.741491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435666849062546:2048] [node 1] ICP30 unregister session Session# [2:7574435666849062648:2048] VirtualId# [2:7574435666849062646:0]
2025-11-19T13:58:51.741499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435666849062648:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:51.741538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:51.741559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:51.743663Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:51.743934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435667085552341:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-11-19T13:58:51.743948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435667085552254:2048] [node 1] ICP30 unregister session Session# [3:7574435667085552341:2048] VirtualId# [3:7574435667085552339:0]
2025-11-19T13:58:51.743959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435667085552341:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:51.744022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:51.744051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:51.751860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:51.751958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435664217352025:2048] [node 1] ICS07 socket disconnect 127 reason# EndOfStream
2025-11-19T13:58:51.751966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435664217352025:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:51.752023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:51.752052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435664217352025:2048] [node 1] ICS15 start handshake
2025-11-19T13:58:51.752569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435664217351935:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9660) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:51.752582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435664217352025:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:51.752587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435664217351935:2048] [node 1] ICP30 unregister session Session# [6:7574435664217352025:2048] VirtualId# [6:7574435664217352023:0]
2025-11-19T13:58:51.752600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:51.752606Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435664217351935:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:51.752609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9660) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:51.752826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:51.752915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435667858744827:2048] [node 1] ICS01 socket: 123 reason# EndOfStream
2025-11-19T13:58:51.752923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435667858744730:2048] [node 1] ICP30 unregister session Session# [5:7574435667858744827:2048] VirtualId# [5:7574435667858744813:0]
2025-11-19T13:58:51.752931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435667858744827:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:51.752984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:51.753010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:51.754768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:51.754853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435667085552348:2049] [node 2] ICS01 socket: 165 reason# EndOfStream
2025-11-19T13:58:51.754864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435667085552255:2049] [node 2] ICP30 unregister session Session# [3:7574435667085552348:2049] VirtualId# [3:7574435667085552344:0]
2025-11-19T13:58:51.754870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435667085552348:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:51.754911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:51.754931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:51.753054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:51.753127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435665981730968:2048] [node 1] ICS01 socket: 129 reason# EndOfStream
2025-11-19T13:58:51.753133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435665981730879:2048] [node 1] ICP30 unregister session Session# [4:7574435665981730968:2048] VirtualId# [4:7574435665981730962:0]
2025-11-19T13:58:51.753137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435665981730968:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:51.753171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:51.753195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:51.764041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:51.764135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435664217352033:2049] [node 2] ICS01 socket: 135 reason# EndOfStream
2025-11-19T13:58:51.764145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435664217351936:2049] [node 2] ICP30 unregister session Session# [6:7574435664217352033:2049] VirtualId# [6:7574435664217352021:0]
2025-11-19T13:58:51.764152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435664217352033:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:51.764216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:51.764240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:51.764332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435664217351935:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:58:51.764590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:51.764676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435667858744825:2049] [node 2] ICS07 socket disconnect 154 reason# EndOfStream
2025-11-19T13:58:51.764682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435667858744825:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:51.764736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:51.764761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435667858744825:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:51.766018Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435667858744731:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5319) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:51.766031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435667858744825:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:51.766035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435667858744731:2049] [node 2] ICP30 unregister session Session# [5:7574435667858744825:2049] VirtualId# [5:7574435667858744821:0]
2025-11-19T13:58:51.766045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:51.766049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435667858744731:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:51.766052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:51.765116Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:51.765181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435665981730966:2049] [node 2] ICS07 socket disconnect 130 reason# EndOfStream
2025-11-19T13:58:51.765186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435665981730966:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:51.765221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:51.765242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435665981730966:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:51.765821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435665981730880:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5319) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:51.765830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435665981730966:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:51.765836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435665981730880:2049] [node 2] ICP30 unregister session Session# [4:7574435665981730966:2049] VirtualId# [4:7574435665981730960:0]
2025-11-19T13:58:51.765860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:51.765866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435665981730880:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:51.765869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:51.775886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435665981730880:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:58:51.776561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435667858744731:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:58:51.776593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:51.776684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435667858744831:2050] [node 3] ICS01 socket: 136 reason# EndOfStream
2025-11-19T13:58:51.776692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435667858744732:2050] [node 3] ICP30 unregister session Session# [5:7574435667858744831:2050] VirtualId# [5:7574435667858744829:0]
2025-11-19T13:58:51.776700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435667858744831:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:51.776757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:51.776781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:51.776592Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:51.776694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435664217352036:2050] [node 3] ICS01 socket: 144 reason# EndOfStream
2025-11-19T13:58:51.776701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435664217351937:2050] [node 3] ICP30 unregister session Session# [6:7574435664217352036:2050] VirtualId# [6:7574435664217352030:0]
2025-11-19T13:58:51.776707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435664217352036:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:51.776763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:51.776788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:51.792505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:51.792608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435664217352012:2064] [node 4] ICS01 socket: 108 reason# EndOfStream
2025-11-19T13:58:51.792617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435664217351953:2064] [node 4] ICP30 unregister session Session# [6:7574435664217352012:2064] VirtualId# [6:7574435664217351954:0]
2025-11-19T13:58:51.792626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435664217352012:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:51.792687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:51.792720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:51.924009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435667417660892:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:51.924067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435667417660891:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:51.981236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435664460035631:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:51.981254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435664460035632:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:52.025461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435672279808092:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:52.025483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435672279808093:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:52.043983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435668855456594:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:52.044000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435668855456595:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:52.044005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435668855456596:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:52.067968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435669671493705:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:52.067984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435669671493706:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:52.067988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435669671493707:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:52.103891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435668978964888:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:52.103906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435668978964889:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:52.103911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435668978964890:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:52.176247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435668978964914:2071] [node 4] ICP01 ready to work
2025-11-19T13:58:52.176492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435672279808152:2106] [node 5] ICP01 ready to work
2025-11-19T13:58:52.176811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435671712628253:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:52.176895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435671712628254:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:52.177521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435671712628261:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.178576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435668978964926:2078] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.178660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435668855456655:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13339
2025-11-19T13:58:52.183605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435671712628266:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60690
2025-11-19T13:58:52.183672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435671712628267:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60690
2025-11-19T13:58:52.184350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435672279808156:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62267
2025-11-19T13:58:52.184520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435672279808160:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.188301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435668755002995:2110] [node 4] ICP01 ready to work
2025-11-19T13:58:52.188420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435668755002996:2111] [node 6] ICP01 ready to work
2025-11-19T13:58:52.188449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435668755002992:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15383
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.189687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435668755003003:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15383
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.193158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435668978964951:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38187
2025-11-19T13:58:52.193432Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435668978964914:2071] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(17099525ns) {Configure:104(17098978ns) {ProcessConfigured:135(13648905ns) {StartInitialHandshake:157(13647985ns) {DropHandshakes:469(483ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(183ns) PrepareNewSessionHandshake:476(754ns) {SwitchToState:178(234ns)}}}}}
2025-11-19T13:58:52.195981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435668978964889:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20004882ns) {Configure:104(19998723ns) {ProcessConfigured:135(16108103ns) {StartInitialHandshake:157(16107084ns) {DropHandshakes:469(430ns) {DropIncomingHandshake:441(12ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:476(1167ns) {SwitchToState:178(961ns)}}}}}
2025-11-19T13:58:52.199301Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435668755002992:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.199342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435664460035631:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.199345Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435668855456655:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.200014Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435671712628267:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.203982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435672279808093:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20127899ns) {Configure:104(20126901ns) {ProcessConfigured:135(19836711ns) {StartInitialHandshake:157(19836008ns) {DropHandshakes:469(593ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(21ns)} GenerateSessionVirtualId:988(413ns) PrepareNewSessionHandshake:476(1024ns) {SwitchToState:178(359ns)}}}}}
2025-11-19T13:58:52.204036Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435672279808156:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.204558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435672279808166:2113] [node 6] ICP01 ready to work
2025-11-19T13:58:52.204877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435672279808166:2113] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.205280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435672279808160:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(20723824ns) {Configure:104(20723300ns) {ProcessConfigured:135(20488012ns) {StartInitialHandshake:157(20487702ns) {DropHandshakes:469(424ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(83ns) PrepareNewSessionHandshake:476(550ns) {SwitchToState:178(154ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.208098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435671712628269:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60690
2025-11-19T13:58:52.208127Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435668978964916:2073] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.208141Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435668978964916:2073] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.208014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435668755003006:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15383
2025-11-19T13:58:52.212184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435664460035632:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(27192499ns) {Configure:104(27191076ns) {ProcessConfigured:135(26914908ns) {StartInitialHandshake:157(26914504ns) {DropHandshakes:469(498ns) {DropIncomingHandshake:441(90ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:476(817ns) {SwitchToState:178(646ns)}}}}}
2025-11-19T13:58:52.212889Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435668978964916:2073] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:52.213155Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435668978964951:2101] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.213192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435668978964889:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.207839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435672279808171:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62267
2025-11-19T13:58:52.213678Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435671712628269:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.213762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435668978964890:2050] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.213779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:52.213786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435668978964890:2050] [node 3] ICP22 created new session: [6:7574435668978964968:2050]
2025-11-19T13:58:52.213792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435668978964968:2050] [node 3] ICS09 handshake done sender: [6:7574435668978964916:2073] self: [6:7574435668978964915:0] peer: [3:7574435672279808168:0] socket: 115 qp: -1
2025-11-19T13:58:52.213799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435668978964968:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:52.213815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:52.214851Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435672279808171:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.214880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435672279808092:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.212340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435669671493767:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58735
2025-11-19T13:58:52.215877Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435668755003006:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.215930Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435671712628248:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.215943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435671712628248:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.219017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435668855456594:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.219062Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435668855456594:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(19658200ns) {Configure:104(19657613ns) {ProcessConfigured:135(19346710ns) {IncomingHandshake:261(19339142ns) {DropIncomingHandshake:441(133ns) PrepareNewSessionHandshake:476(249ns) {SwitchToState:178(78ns)} GenerateSessionVirtualId:988(323ns) IssueIncomingHandshakeReply:193(19335283ns) {SwitchToState:178(693ns)}}}}}
2025-11-19T13:58:52.216592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435669671493769:2108] [node 6] ICP01 ready to work
2025-11-19T13:58:52.219699Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435668755003000:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.219708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435668755003000:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.220370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435668855456662:2110] [node 5] ICP01 ready to work
2025-11-19T13:58:52.219547Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435671712628252:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.219559Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435671712628252:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.219626Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435671712628266:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.220834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435668755003005:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.223959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435668978964926:2078] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(44001556ns) {Configure:104(44000607ns) {ProcessConfigured:135(43773705ns) {StartInitialHandshake:157(43773031ns) {DropHandshakes:469(282ns) {DropIncomingHandshake:441(30ns) DropOutgoingHandshake:456(30ns)} GenerateSessionVirtualId:988(263ns) PrepareNewSessionHandshake:476(499ns) {SwitchToState:178(102ns)}}}}}
2025-11-19T13:58:52.224085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435668978964972:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38187
2025-11-19T13:58:52.224380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435672279808173:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62267
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.224161Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435668755003003:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.224965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435668755003011:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15383
2025-11-19T13:58:52.225128Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435668755003011:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.225314Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435672279808156:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:52.225451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435672279808166:2113] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.225467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:52.225473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435672279808166:2113] [node 6] ICP22 created new session: [3:7574435672279808174:2113]
2025-11-19T13:58:52.225479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435672279808174:2113] [node 6] ICS09 handshake done sender: [3:7574435672279808156:2108] self: [3:7574435672279808168:0] peer: [6:7574435668978964915:0] socket: 116 qp: -1
2025-11-19T13:58:52.225486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435672279808174:2113] [node 6] ICS10 traffic start
2025-11-19T13:58:52.225496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.227610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435668855456668:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13339
2025-11-19T13:58:52.227656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435668855456669:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13339
2025-11-19T13:58:52.228468Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435668855456668:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.228164Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435671712628257:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.228178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435671712628257:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.228204Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435671712628248:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:52.235985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435667417660891:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.236024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:52.236034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435667417660891:2048] [node 2] ICP22 created new session: [1:7574435671712628273:2048]
2025-11-19T13:58:52.236042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435671712628273:2048] [node 2] ICS09 handshake done sender: [1:7574435671712628248:2107] self: [1:7574435671712628247:0] peer: [2:7574435668755003004:0] socket: 117 qp: -1
2025-11-19T13:58:52.236052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435671712628273:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:52.236076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.235961Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435669671493705:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16865314ns) {Configure:104(16863694ns) {ProcessConfigured:135(16600953ns) {StartInitialHandshake:157(16600318ns) {DropHandshakes:469(403ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(1536ns) {SwitchToState:178(900ns)}}}}}
2025-11-19T13:58:52.236771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435668978964890:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(22977301ns) {HandleHandshakeStatus:329(22976275ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(332ns) SwitchToState:178(40ns) ProcessPendingSessionEvents:495(17203268ns) {ForwardSessionEventToSession:612(17191332ns) ForwardSessionEventToSession:612(2046ns) ForwardSessionEventToSession:612(1099ns) ForwardSessionEventToSession:612(567ns) ForwardSessionEventToSession:612(503ns) ForwardSessionEventToSession:612(1243ns) ForwardSessionEventToSession:612(535ns) ForwardSessionEventToSession:612(559ns) ForwardSessionEventToSession:612(566ns) ForwardSessionEventToSession:612(837ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.238975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435664460035631:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435668755002992:2109] held: no
2025-11-19T13:58:52.239949Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435668755003000:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:52.240045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435668755002996:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.240063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:52.240070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435668755002996:2111] [node 6] ICP22 created new session: [2:7574435668755003014:2111]
2025-11-19T13:58:52.240077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435668755003014:2111] [node 6] ICS09 handshake done sender: [2:7574435668755003000:2113] self: [2:7574435668755002999:0] peer: [6:7574435668978964967:0] socket: 129 qp: -1
2025-11-19T13:58:52.240084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435668755003014:2111] [node 6] ICS10 traffic start
2025-11-19T13:58:52.240094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:52.242843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435668855456662:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(22361676ns) {Configure:104(22360953ns) {ProcessConfigured:135(22120545ns) {StartInitialHandshake:157(22119496ns) {DropHandshakes:469(399ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(45ns)} GenerateSessionVirtualId:988(141ns) PrepareNewSessionHandshake:476(956ns) {SwitchToState:178(460ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.242618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435669671493781:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58735
2025-11-19T13:58:52.243708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435668978964889:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5948) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435668978964951:2101] held: no
2025-11-19T13:58:52.243826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435668978964973:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38187
2025-11-19T13:58:52.244080Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435668978964951:2101] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:52.245080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435668755003005:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(24179713ns) {Configure:104(24179091ns) {ProcessConfigured:135(23906456ns) {StartInitialHandshake:157(23905874ns) {DropHandshakes:469(466ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:476(781ns) {SwitchToState:178(337ns)}}}}}
2025-11-19T13:58:52.245596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435668978964889:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.245611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:52.245616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435668978964889:2049] [node 2] ICP22 created new session: [6:7574435668978964974:2049]
2025-11-19T13:58:52.245622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435668978964974:2049] [node 2] ICS09 handshake done sender: [6:7574435668978964951:2101] self: [6:7574435668978964967:0] peer: [2:7574435668755002999:0] socket: 133 qp: -1
2025-11-19T13:58:52.245627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435668978964974:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:52.245648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:52.246681Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435669671493767:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.246985Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435669671493781:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.247010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435669671493707:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.247019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435669671493705:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.244030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435671712628275:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60690
2025-11-19T13:58:52.244133Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435671712628252:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:52.244258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435667417660892:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.244279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:52.244286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435667417660892:2049] [node 3] ICP22 created new session: [1:7574435671712628278:2049]
2025-11-19T13:58:52.244292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435671712628278:2049] [node 3] ICS09 handshake done sender: [1:7574435671712628252:2110] self: [1:7574435671712628251:0] peer: [3:7574435672279808172:0] socket: 118 qp: -1
2025-11-19T13:58:52.244297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435671712628278:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:52.244305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:52.242941Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435668855456669:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.248068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435668855456665:2111] [node 6] ICP01 ready to work
2025-11-19T13:58:52.247931Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435672279808092:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435672279808171:2115] held: no
2025-11-19T13:58:52.248431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435669671493774:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.252470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435668755003018:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15383
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.252282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435672279808176:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62267
2025-11-19T13:58:52.252298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435672279808177:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62267
2025-11-19T13:58:52.252350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435672279808178:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62267
2025-11-19T13:58:52.256287Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435671712628275:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.256290Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435671712628260:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.256295Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435671712628260:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.256650Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435671712628257:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.257127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435668855456674:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13339
2025-11-19T13:58:52.257879Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435672279808154:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.257895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435672279808154:2107] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.258000Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435668755002992:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:52.259981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435668855456595:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.260021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435668855456595:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(30438438ns) {IncomingHandshake:261(30437856ns) {DropIncomingHandshake:441(347ns) PrepareNewSessionHandshake:476(529ns) {SwitchToState:178(269ns)} GenerateSessionVirtualId:988(215ns) IssueIncomingHandshakeReply:193(30432079ns) {SwitchToState:178(443ns)}}}
2025-11-19T13:58:52.260053Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435668855456655:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:52.259930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435672279808179:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62267
2025-11-19T13:58:52.260032Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435672279808177:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.260072Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435672279808171:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:52.260067Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435668978964972:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.262244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435671712628253:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.262268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:52.262276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435671712628253:2108] [node 4] ICP22 created new session: [1:7574435671712628284:2108]
2025-11-19T13:58:52.262284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435671712628284:2108] [node 4] ICS09 handshake done sender: [1:7574435671712628257:2111] self: [1:7574435671712628256:0] peer: [4:7574435668855456659:0] socket: 119 qp: -1
2025-11-19T13:58:52.262293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435671712628284:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:52.262304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:52.262641Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435671712628260:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:52.262930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435668978964926:2078] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.263475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435669671493785:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58735
2025-11-19T13:58:52.263496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435669671493786:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58735
2025-11-19T13:58:52.263536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435669671493787:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58735
2025-11-19T13:58:52.263724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435672279808092:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.263741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:52.263761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435672279808092:2048] [node 1] ICP22 created new session: [3:7574435672279808180:2048]
2025-11-19T13:58:52.263766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435672279808180:2048] [node 1] ICS09 handshake done sender: [3:7574435672279808171:2115] self: [3:7574435672279808172:0] peer: [1:7574435671712628251:0] socket: 125 qp: -1
2025-11-19T13:58:52.263772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435672279808180:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:52.263779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:52.259851Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435668755002998:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.259870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435668755002998:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.263994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435664460035631:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.264018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:52.264023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435664460035631:2048] [node 1] ICP22 created new session: [2:7574435668755003019:2048]
2025-11-19T13:58:52.264029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435668755003019:2048] [node 1] ICS09 handshake done sender: [2:7574435668755002992:2109] self: [2:7574435668755003004:0] peer: [1:7574435671712628247:0] socket: 123 qp: -1
2025-11-19T13:58:52.264034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435668755003019:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:52.264040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:52.264566Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435669671493771:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.264577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435669671493771:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.265094Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435669671493781:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:52.265283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435668855456594:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.265300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:52.265307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435668855456594:2048] [node 1] ICP22 created new session: [4:7574435668855456677:2048]
2025-11-19T13:58:52.265312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435668855456677:2048] [node 1] ICS09 handshake done sender: [4:7574435668855456655:2108] self: [4:7574435668855456659:0] peer: [1:7574435671712628256:0] socket: 121 qp: -1
2025-11-19T13:58:52.265317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435668855456677:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:52.265337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:52.268029Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435669671493771:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:52.271832Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435672279808176:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.271869Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435672279808179:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.271917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435672279808093:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.272108Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435668755003007:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.272119Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435668755003007:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.272026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435669671493705:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.272046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:52.272053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435669671493705:2048] [node 1] ICP22 created new session: [5:7574435669671493790:2048]
2025-11-19T13:58:52.272059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435669671493790:2048] [node 1] ICS09 handshake done sender: [5:7574435669671493781:2114] self: [5:7574435669671493783:0] peer: [1:7574435671712628259:0] socket: 155 qp: -1
2025-11-19T13:58:52.272065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435669671493790:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:52.272081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:52.272230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435669671493769:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.272237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:52.272242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435669671493769:2108] [node 6] ICP22 created new session: [5:7574435669671493792:2108]
2025-11-19T13:58:52.272245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435669671493792:2108] [node 6] ICS09 handshake done sender: [5:7574435669671493771:2111] self: [5:7574435669671493770:0] peer: [6:7574435668978964977:0] socket: 143 qp: -1
2025-11-19T13:58:52.272248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435669671493792:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:52.272252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:52.272434Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435669671493786:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.272502Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435669671493706:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(50569799ns) {Configure:104(50568856ns) {ProcessConfigured:135(50321512ns) {StartInitialHandshake:157(50320786ns) {DropHandshakes:469(423ns) {DropIncomingHandshake:441(78ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:476(866ns) {SwitchToState:178(771ns)}}}}}
2025-11-19T13:58:52.272673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435669671493769:2108] [node 6] ICP18 handshake (actor [5:7574435669671493786:2116]) is too old
2025-11-19T13:58:52.274420Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435672279808154:2107] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:52.274531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435672279808152:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.274549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:52.274555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435672279808152:2106] [node 5] ICP22 created new session: [3:7574435672279808186:2106]
2025-11-19T13:58:52.274561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435672279808186:2106] [node 5] ICS09 handshake done sender: [3:7574435672279808154:2107] self: [3:7574435672279808153:0] peer: [5:7574435669671493782:0] socket: 114 qp: -1
2025-11-19T13:58:52.274566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435672279808186:2106] [node 5] ICS10 traffic start
2025-11-19T13:58:52.274581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.275618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435668855456679:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13339
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.276145Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435668755002998:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:52.276024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435668978964978:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38187
2025-11-19T13:58:52.275940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435671712628254:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.275971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:52.275977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435671712628254:2109] [node 5] ICP22 created new session: [1:7574435671712628286:2109]
2025-11-19T13:58:52.275982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435671712628286:2109] [node 5] ICS09 handshake done sender: [1:7574435671712628260:2113] self: [1:7574435671712628259:0] peer: [5:7574435669671493783:0] socket: 152 qp: -1
2025-11-19T13:58:52.275987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435671712628286:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:52.275993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:52.276268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435668755002995:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.276283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:52.276289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435668755002995:2110] [node 4] ICP22 created new session: [2:7574435668755003023:2110]
2025-11-19T13:58:52.276295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435668755003023:2110] [node 4] ICS09 handshake done sender: [2:7574435668755002998:2112] self: [2:7574435668755002997:0] peer: [4:7574435668855456670:0] socket: 128 qp: -1
2025-11-19T13:58:52.276300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435668755003023:2110] [node 4] ICS10 traffic start
2025-11-19T13:58:52.276306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:52.276806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435668978964979:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38187
2025-11-19T13:58:52.279808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435668855456680:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13339
2025-11-19T13:58:52.279840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435668855456681:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13339
2025-11-19T13:58:52.280238Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435668855456681:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.280269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435668855456596:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.280360Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435668855456668:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:52.280419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435668855456595:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.280429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:52.280433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435668855456595:2049] [node 2] ICP22 created new session: [4:7574435668855456683:2049]
2025-11-19T13:58:52.280438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435668855456683:2049] [node 2] ICS09 handshake done sender: [4:7574435668855456668:2113] self: [4:7574435668855456670:0] peer: [2:7574435668755002997:0] socket: 131 qp: -1
2025-11-19T13:58:52.280442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435668855456683:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:52.280460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:52.283905Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435668978964972:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:52.284342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435668978964926:2078] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.284364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:52.284370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435668978964926:2078] [node 5] ICP22 created new session: [6:7574435668978964980:2078]
2025-11-19T13:58:52.284376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435668978964980:2078] [node 5] ICS09 handshake done sender: [6:7574435668978964972:2116] self: [6:7574435668978964977:0] peer: [5:7574435669671493770:0] socket: 144 qp: -1
2025-11-19T13:58:52.284382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435668978964980:2078] [node 5] ICS10 traffic start
2025-11-19T13:58:52.284390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:52.289046Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435672279808170:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.289070Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435672279808170:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.289537Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435668755003007:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:52.289600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435664460035632:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.289619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:52.289627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435664460035632:2049] [node 3] ICP22 created new session: [2:7574435668755003026:2049]
2025-11-19T13:58:52.289634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435668755003026:2049] [node 3] ICS09 handshake done sender: [2:7574435668755003007:2116] self: [2:7574435668755002991:0] peer: [3:7574435672279808184:0] socket: 142 qp: -1
2025-11-19T13:58:52.289639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435668755003026:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:52.289646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:52.292600Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435668978964978:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.292634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435668978964888:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.295522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435668755002995:2110] [node 4] ICP35 event processing took too much time PendingConnection:278(19222067ns) {HandleHandshakeStatus:329(19221353ns) {DropIncomingHandshake:441(2990ns) DropOutgoingHandshake:456(108ns) SwitchToState:178(71ns) ProcessPendingSessionEvents:495(6737250ns) {ForwardSessionEventToSession:612(6725204ns) ForwardSessionEventToSession:612(1265ns) ForwardSessionEventToSession:612(1012ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(443ns) ForwardSessionEventToSession:612(968ns) ForwardSessionEventToSession:612(839ns) ForwardSessionEventToSession:612(682ns) ForwardSessionEventToSession:612(705ns) ForwardSessionEventToSession:612(904ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.294780Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435669671493785:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.295975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435669671493774:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.296246Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435668855456679:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.297052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435668855456687:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13339
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.297468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435668755003028:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15383
2025-11-19T13:58:52.296669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435672279808190:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62267
2025-11-19T13:58:52.300472Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435668855456681:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:52.299909Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435672279808170:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:52.300066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435672279808093:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5948) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435672279808179:2120] held: no
2025-11-19T13:58:52.300106Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435672279808179:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:52.300704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435672279808160:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.300726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:52.300735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435672279808160:2110] [node 4] ICP22 created new session: [3:7574435672279808191:2110]
2025-11-19T13:58:52.300741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435672279808191:2110] [node 4] ICS09 handshake done sender: [3:7574435672279808170:2114] self: [3:7574435672279808161:0] peer: [4:7574435668855456682:0] socket: 173 qp: -1
2025-11-19T13:58:52.300749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435672279808191:2110] [node 4] ICS10 traffic start
2025-11-19T13:58:52.300780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:52.300871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435669671493797:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58735
2025-11-19T13:58:52.300981Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435668855456671:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.300991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435668855456671:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.301382Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435668755003028:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.301660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435668855456596:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.301679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:52.301685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435668855456596:2050] [node 3] ICP22 created new session: [4:7574435668855456688:2050]
2025-11-19T13:58:52.301690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435668855456688:2050] [node 3] ICS09 handshake done sender: [4:7574435668855456681:2120] self: [4:7574435668855456682:0] peer: [3:7574435672279808161:0] socket: 176 qp: -1
2025-11-19T13:58:52.301696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435668855456688:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:52.301703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:52.300809Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435671712628264:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.300827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435671712628264:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.303870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435669671493707:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435669671493767:2109] held: no
2025-11-19T13:58:52.303937Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435669671493797:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.303976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435669671493706:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.304289Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435668755003017:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.304301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435668755003017:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.305558Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435668755003017:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:52.305717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435668755003005:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.305762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:52.305769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435668755003005:2108] [node 5] ICP22 created new session: [2:7574435668755003031:2108]
2025-11-19T13:58:52.305777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435668755003031:2108] [node 5] ICS09 handshake done sender: [2:7574435668755003017:2118] self: [2:7574435668755003009:0] peer: [5:7574435669671493799:0] socket: 167 qp: -1
2025-11-19T13:58:52.305783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435668755003031:2108] [node 5] ICS10 traffic start
2025-11-19T13:58:52.305809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:52.308812Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435669671493706:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5948) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435669671493797:2120] held: no
2025-11-19T13:58:52.312148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435672279808093:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.312176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:52.312185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435672279808093:2049] [node 2] ICP22 created new session: [3:7574435672279808193:2049]
2025-11-19T13:58:52.312194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435672279808193:2049] [node 2] ICS09 handshake done sender: [3:7574435672279808179:2120] self: [3:7574435672279808184:0] peer: [2:7574435668755002991:0] socket: 160 qp: -1
2025-11-19T13:58:52.312203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435672279808193:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:52.312212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.314877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435669671493800:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58735
2025-11-19T13:58:52.315185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435669671493801:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58735
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.317127Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435671712628264:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:52.317269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435669671493802:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58735
2025-11-19T13:58:52.317276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435672279808160:2110] [node 4] ICP35 event processing took too much time PendingConnection:278(16531299ns) {HandleHandshakeStatus:329(16529946ns) {DropIncomingHandshake:441(4051ns) DropOutgoingHandshake:456(102ns) SwitchToState:178(230ns) ProcessPendingSessionEvents:495(5375163ns) {ForwardSessionEventToSession:612(5362530ns) ForwardSessionEventToSession:612(1781ns) ForwardSessionEventToSession:612(1079ns) ForwardSessionEventToSession:612(771ns) ForwardSessionEventToSession:612(570ns) ForwardSessionEventToSession:612(996ns) ForwardSessionEventToSession:612(1009ns) ForwardSessionEventToSession:612(564ns) ForwardSessionEventToSession:612(591ns) ForwardSessionEventToSession:612(1208ns)}}}
2025-11-19T13:58:52.317582Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435669671493797:2120] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.317687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435671712628261:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.317711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:52.317718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435671712628261:2112] [node 6] ICP22 created new session: [1:7574435671712628289:2112]
2025-11-19T13:58:52.317723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435671712628289:2112] [node 6] ICS09 handshake done sender: [1:7574435671712628264:2114] self: [1:7574435671712628263:0] peer: [6:7574435668978964982:0] socket: 161 qp: -1
2025-11-19T13:58:52.317933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435668978964983:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38187
2025-11-19T13:58:52.317976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435668978964984:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38187
2025-11-19T13:58:52.317730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435671712628289:2112] [node 6] ICS10 traffic start
2025-11-19T13:58:52.317740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:52.318229Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435669671493785:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:52.318296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435669671493706:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.318307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:52.318311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435669671493706:2049] [node 2] ICP22 created new session: [5:7574435669671493803:2049]
2025-11-19T13:58:52.318316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435669671493803:2049] [node 2] ICS09 handshake done sender: [5:7574435669671493797:2120] self: [5:7574435669671493799:0] peer: [2:7574435668755003009:0] socket: 168 qp: -1
2025-11-19T13:58:52.318323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435669671493803:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:52.318330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:52.318712Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435669671493767:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:52.318656Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435668978964983:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.318700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435668978964914:2071] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.318777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435669671493774:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.318788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:52.318793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435669671493774:2110] [node 4] ICP22 created new session: [5:7574435669671493805:2110]
2025-11-19T13:58:52.318798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435669671493805:2110] [node 4] ICS09 handshake done sender: [5:7574435669671493785:2115] self: [5:7574435669671493798:0] peer: [4:7574435668855456663:0] socket: 141 qp: -1
2025-11-19T13:58:52.318803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435669671493805:2110] [node 4] ICS10 traffic start
2025-11-19T13:58:52.318822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:52.319149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435669671493707:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.319160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:52.319165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435669671493707:2050] [node 3] ICP22 created new session: [5:7574435669671493807:2050]
2025-11-19T13:58:52.319170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435669671493807:2050] [node 3] ICS09 handshake done sender: [5:7574435669671493767:2109] self: [5:7574435669671493782:0] peer: [3:7574435672279808153:0] socket: 124 qp: -1
2025-11-19T13:58:52.319174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435669671493807:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:52.319183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.320316Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435668978964888:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435668978964978:2118] held: no
2025-11-19T13:58:52.320375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435668978964986:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38187
2025-11-19T13:58:52.317659Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435668855456671:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:52.318228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435668855456662:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.318242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:52.318248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435668855456662:2110] [node 5] ICP22 created new session: [4:7574435668855456691:2110]
2025-11-19T13:58:52.318252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435668855456691:2110] [node 5] ICS09 handshake done sender: [4:7574435668855456671:2115] self: [4:7574435668855456663:0] peer: [5:7574435669671493798:0] socket: 156 qp: -1
2025-11-19T13:58:52.318257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435668855456691:2110] [node 5] ICS10 traffic start
2025-11-19T13:58:52.318264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:52.318852Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435668855456675:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.318857Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435668855456675:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.320241Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435668855456675:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:52.320529Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435668978964978:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:52.320964Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435668978964983:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:52.320635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435668855456665:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.320654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:52.320659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435668855456665:2111] [node 6] ICP22 created new session: [4:7574435668855456694:2111]
2025-11-19T13:58:52.320664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435668855456694:2111] [node 6] ICS09 handshake done sender: [4:7574435668855456675:2117] self: [4:7574435668855456673:0] peer: [6:7574435668978964985:0] socket: 132 qp: -1
2025-11-19T13:58:52.320669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435668855456694:2111] [node 6] ICS10 traffic start
2025-11-19T13:58:52.320675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:52.321076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435668978964888:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.321095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:52.321101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435668978964888:2048] [node 1] ICP22 created new session: [6:7574435668978964987:2048]
2025-11-19T13:58:52.321107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435668978964987:2048] [node 1] ICS09 handshake done sender: [6:7574435668978964978:2118] self: [6:7574435668978964982:0] peer: [1:7574435671712628263:0] socket: 120 qp: -1
2025-11-19T13:58:52.321113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435668978964987:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:52.321125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:52.321126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435668978964914:2071] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.321136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:52.321140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435668978964914:2071] [node 4] ICP22 created new session: [6:7574435668978964989:2071]
2025-11-19T13:58:52.321145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435668978964989:2071] [node 4] ICS09 handshake done sender: [6:7574435668978964983:2120] self: [6:7574435668978964985:0] peer: [4:7574435668855456673:0] socket: 158 qp: -1
2025-11-19T13:58:52.321159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435668978964989:2071] [node 4] ICS10 traffic start
2025-11-19T13:58:52.321164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:52.329861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:52.329958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435668755003019:2048] [node 1] ICS01 socket: 123 reason# EndOfStream
2025-11-19T13:58:52.329968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435664460035631:2048] [node 1] ICP30 unregister session Session# [2:7574435668755003019:2048] VirtualId# [2:7574435668755003004:0]
2025-11-19T13:58:52.329976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435668755003019:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:52.330007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:52.330029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:52.331826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:52.331913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435672279808180:2048] [node 1] ICS01 socket: 125 reason# EndOfStream
2025-11-19T13:58:52.331920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435672279808092:2048] [node 1] ICP30 unregister session Session# [3:7574435672279808180:2048] VirtualId# [3:7574435672279808172:0]
2025-11-19T13:58:52.331926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435672279808180:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:52.331961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:52.331985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:52.334592Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:52.334662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435668978964987:2048] [node 1] ICS01 socket: 120 reason# EndOfStream
2025-11-19T13:58:52.334668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435668978964888:2048] [node 1] ICP30 unregister session Session# [6:7574435668978964987:2048] VirtualId# [6:7574435668978964982:0]
2025-11-19T13:58:52.334673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435668978964987:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:52.334681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:52.334721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:52.335867Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:52.335936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435669671493790:2048] [node 1] ICS01 socket: 155 reason# EndOfStream
2025-11-19T13:58:52.335941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435669671493705:2048] [node 1] ICP30 unregister session Session# [5:7574435669671493790:2048] VirtualId# [5:7574435669671493783:0]
2025-11-19T13:58:52.335947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435669671493790:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:52.335986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:52.336008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:52.337246Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:52.337314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435668855456677:2048] [node 1] ICS07 socket disconnect 121 reason# EndOfStream
2025-11-19T13:58:52.337318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435668855456677:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:52.337359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:52.337379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435668855456677:2048] [node 1] ICS15 start handshake
2025-11-19T13:58:52.338008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435668855456594:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4845) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:52.338029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435668855456677:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:52.338034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435668855456594:2048] [node 1] ICP30 unregister session Session# [4:7574435668855456677:2048] VirtualId# [4:7574435668855456659:0]
2025-11-19T13:58:52.338047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:52.338053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435668855456594:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:52.338056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:52.343030Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:52.343112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435672279808193:2049] [node 2] ICS01 socket: 160 reason# EndOfStream
2025-11-19T13:58:52.343120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435672279808093:2049] [node 2] ICP30 unregister session Session# [3:7574435672279808193:2049] VirtualId# [3:7574435672279808184:0]
2025-11-19T13:58:52.343125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435672279808193:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:52.343166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:52.343186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:52.347361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435668855456594:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:58:52.354627Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:52.354731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435668978964974:2049] [node 2] ICS01 socket: 133 reason# EndOfStream
2025-11-19T13:58:52.354742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435668978964889:2049] [node 2] ICP30 unregister session Session# [6:7574435668978964974:2049] VirtualId# [6:7574435668978964967:0]
2025-11-19T13:58:52.354750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435668978964974:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:52.354792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:52.354815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:52.357494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:52.357581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435668855456683:2049] [node 2] ICS01 socket: 131 reason# EndOfStream
2025-11-19T13:58:52.357588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435668855456595:2049] [node 2] ICP30 unregister session Session# [4:7574435668855456683:2049] VirtualId# [4:7574435668855456670:0]
2025-11-19T13:58:52.357594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435668855456683:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:52.357639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:52.357660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:52.357593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:52.357664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435669671493803:2049] [node 2] ICS01 socket: 168 reason# EndOfStream
2025-11-19T13:58:52.357670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435669671493706:2049] [node 2] ICP30 unregister session Session# [5:7574435669671493803:2049] VirtualId# [5:7574435669671493799:0]
2025-11-19T13:58:52.357675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435669671493803:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:52.357719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:52.357745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:52.363909Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:52.363917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T13:58:52.363997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435668855456688:2050] [node 3] ICS01 socket: 176 reason# EndOfStream
2025-11-19T13:58:52.364004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435668855456596:2050] [node 3] ICP30 unregister session Session# [4:7574435668855456688:2050] VirtualId# [4:7574435668855456682:0]
2025-11-19T13:58:52.364010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435668855456688:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:52.364046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:52.364066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:52.364192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435668978964968:2050] [node 3] ICS01 socket: 115 reason# EndOfStream
2025-11-19T13:58:52.364201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435668978964890:2050] [node 3] ICP30 unregister session Session# [6:7574435668978964968:2050] VirtualId# [6:7574435668978964915:0]
2025-11-19T13:58:52.364205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435668978964968:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:52.364229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:52.364254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:52.364117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:52.364200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435669671493807:2050] [node 3] ICS01 socket: 124 reason# EndOfStream
2025-11-19T13:58:52.364207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435669671493707:2050] [node 3] ICP30 unregister session Session# [5:7574435669671493807:2050] VirtualId# [5:7574435669671493782:0]
2025-11-19T13:58:52.364212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435669671493807:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:52.364251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:52.364273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:52.389574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:52.389677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435668978964989:2071] [node 4] ICS01 socket: 158 reason# EndOfStream
2025-11-19T13:58:52.389689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435668978964914:2071] [node 4] ICP30 unregister session Session# [6:7574435668978964989:2071] VirtualId# [6:7574435668978964985:0]
2025-11-19T13:58:52.389697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435668978964989:2071] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:52.389750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:52.389775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:52.552030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435672654303510:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:52.552047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435672654303511:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:52.587916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435671497311509:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:52.587931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435671497311510:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:52.675999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435671105615713:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:52.676016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435671105615714:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:52.771948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435671359049707:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:52.772005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435671359049708:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:52.772020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435671359049709:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:52.815876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435670887331388:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:52.815895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435670887331389:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:52.815899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435670887331390:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:52.867970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435671577727464:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:52.867984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435671577727465:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:52.867989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435671577727466:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:52.916216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435671105615751:2082] [node 6] ICP01 ready to work
2025-11-19T13:58:52.917182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435671105615758:2087] [node 5] ICP01 ready to work
2025-11-19T13:58:52.916722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435672654303572:2104] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.918441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435671105615761:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21538
2025-11-19T13:58:52.919835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435672654303578:2106] [node 5] ICP01 ready to work
2025-11-19T13:58:52.919909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435672654303579:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:52.920376Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435671105615761:2090] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.924140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435671105615713:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.924302Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435672654303569:2103] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.924311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435672654303569:2103] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.924858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435671105615768:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21538
2025-11-19T13:58:52.924968Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435672654303569:2103] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:52.925188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435672654303511:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.925206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:52.925212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435672654303511:2049] [node 3] ICP22 created new session: [1:7574435672654303583:2049]
2025-11-19T13:58:52.925218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435672654303583:2049] [node 3] ICS09 handshake done sender: [1:7574435672654303569:2103] self: [1:7574435672654303568:0] peer: [3:7574435671105615767:0] socket: 121 qp: -1
2025-11-19T13:58:52.925226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435672654303583:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:52.925246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:52.923318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435671359049742:2078] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.933405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435671497311576:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37467
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.934248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435672654303585:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28999
2025-11-19T13:58:52.934268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435672654303586:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28999
2025-11-19T13:58:52.934280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435672654303587:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28999
2025-11-19T13:58:52.932838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435670887331389:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16674922ns) {Configure:104(16673903ns) {ProcessConfigured:135(16348664ns) {StartInitialHandshake:157(16347909ns) {DropHandshakes:469(682ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(76ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:476(832ns) {SwitchToState:178(40ns)}}}}}
2025-11-19T13:58:52.936179Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435671105615761:2090] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:52.936697Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435671497311576:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.937295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435671497311570:2107] [node 5] ICP01 ready to work
2025-11-19T13:58:52.937327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435671497311509:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.944690Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435672654303566:2102] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.944703Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435672654303566:2102] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.944218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435671497311584:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37467
2025-11-19T13:58:52.944279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435671497311585:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37467
2025-11-19T13:58:52.944289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435671497311586:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37467
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.945635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435671497311587:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37467
2025-11-19T13:58:52.945754Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435672654303566:2102] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:52.945964Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435671497311576:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:52.946024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435671497311509:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.946044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:52.946051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435671497311509:2048] [node 1] ICP22 created new session: [2:7574435671497311588:2048]
2025-11-19T13:58:52.946057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435671497311588:2048] [node 1] ICS09 handshake done sender: [2:7574435671497311576:2110] self: [2:7574435671497311582:0] peer: [1:7574435672654303565:0] socket: 123 qp: -1
2025-11-19T13:58:52.946064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435671497311588:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:52.946088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:52.946446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435672654303510:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.946461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:52.946466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435672654303510:2048] [node 2] ICP22 created new session: [1:7574435672654303596:2048]
2025-11-19T13:58:52.946471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435672654303596:2048] [node 2] ICS09 handshake done sender: [1:7574435672654303566:2102] self: [1:7574435672654303565:0] peer: [2:7574435671497311582:0] socket: 120 qp: -1
2025-11-19T13:58:52.946475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435672654303596:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:52.946504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:52.947437Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435671577727466:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(27169982ns) {Configure:104(20731872ns) {ProcessConfigured:135(20462223ns) {StartInitialHandshake:157(20460908ns) {DropHandshakes:469(668ns) {DropIncomingHandshake:441(172ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:476(986ns) {SwitchToState:178(99ns)}}}}}
2025-11-19T13:58:52.947981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435671105615713:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.948000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:52.948006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435671105615713:2048] [node 1] ICP22 created new session: [3:7574435671105615794:2048]
2025-11-19T13:58:52.948012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435671105615794:2048] [node 1] ICS09 handshake done sender: [3:7574435671105615761:2090] self: [3:7574435671105615767:0] peer: [1:7574435672654303568:0] socket: 122 qp: -1
2025-11-19T13:58:52.948017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435671105615794:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:52.948033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.953161Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435671497311584:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.953414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435671105615791:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21538
2025-11-19T13:58:52.953435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435671105615792:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21538
2025-11-19T13:58:52.953447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435671105615793:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21538
2025-11-19T13:58:52.952148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435670887331390:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(24040796ns) {Configure:104(24039777ns) {ProcessConfigured:135(23764852ns) {StartInitialHandshake:157(23764372ns) {DropHandshakes:469(355ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(250ns) PrepareNewSessionHandshake:476(1340ns) {SwitchToState:178(666ns)}}}}}
2025-11-19T13:58:52.952286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435671577727527:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52337
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.955728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435670887331457:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64841
2025-11-19T13:58:52.956241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435671105615752:2079] [node 4] ICP01 ready to work
2025-11-19T13:58:52.957480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435672654303511:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:52.959861Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435671577727527:2102] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.959902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435671577727464:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.960924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435671497311570:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23546534ns) {Configure:104(23546135ns) {ProcessConfigured:135(23287634ns) {StartInitialHandshake:157(23286990ns) {DropHandshakes:469(499ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(136ns) PrepareNewSessionHandshake:476(553ns) {SwitchToState:178(343ns)}}}}}
2025-11-19T13:58:52.960970Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435671497311585:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.961185Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435671105615792:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.961592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435670887331462:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64841
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.961560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435672654303579:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(41482783ns) {Configure:104(41481686ns) {ProcessConfigured:135(41247500ns) {StartInitialHandshake:157(41246996ns) {DropHandshakes:469(288ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(129ns) PrepareNewSessionHandshake:476(729ns) {SwitchToState:178(507ns)}}}}}
2025-11-19T13:58:52.961880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435672654303600:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28999
2025-11-19T13:58:52.961899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435672654303601:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28999
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.964100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435671577727518:2066] [node 5] ICP01 ready to work
2025-11-19T13:58:52.964888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435671497311573:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:52.962224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435671359049783:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17744
2025-11-19T13:58:52.965735Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435672654303574:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.965747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435672654303574:2105] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.965825Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435672654303601:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.965409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435671497311574:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:52.968633Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435671105615758:2087] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(44712524ns) {Configure:104(44711486ns) {ProcessConfigured:135(44483143ns) {StartInitialHandshake:157(44482375ns) {DropHandshakes:469(259ns) {DropIncomingHandshake:441(39ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:476(1168ns) {SwitchToState:178(968ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.970803Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435672654303585:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.971102Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435670887331457:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.971138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435670887331388:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.971261Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435672654303588:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.971266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435672654303588:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.971830Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435672654303588:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:52.971925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435672654303578:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.971945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:52.971951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435672654303578:2106] [node 5] ICP22 created new session: [1:7574435672654303605:2106]
2025-11-19T13:58:52.971963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435672654303605:2106] [node 5] ICS09 handshake done sender: [1:7574435672654303588:2111] self: [1:7574435672654303580:0] peer: [5:7574435670887331464:0] socket: 139 qp: -1
2025-11-19T13:58:52.971971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435672654303605:2106] [node 5] ICS10 traffic start
2025-11-19T13:58:52.971982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:52.972016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435670887331423:2063] [node 6] ICP01 ready to work
2025-11-19T13:58:52.972337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435671577727540:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52337
2025-11-19T13:58:52.975765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435671577727495:2059] [node 4] ICP01 ready to work
2025-11-19T13:58:52.975843Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435671577727540:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.975764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435670887331463:2070] [node 4] ICP01 ready to work
2025-11-19T13:58:52.976055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435671105615802:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21538
2025-11-19T13:58:52.976171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435671577727495:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.979898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435671577727545:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52337
2025-11-19T13:58:52.979882Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435671359049767:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.979897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435671359049767:2100] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.980091Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435671105615802:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.981744Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435671359049783:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.981789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435671359049707:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.981661Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435672654303574:2105] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:52.982037Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435672654303598:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.982046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435672654303598:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.982650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435672654303572:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.982665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:52.982672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435672654303572:2104] [node 6] ICP22 created new session: [1:7574435672654303608:2104]
2025-11-19T13:58:52.982679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435672654303608:2104] [node 6] ICS09 handshake done sender: [1:7574435672654303574:2105] self: [1:7574435672654303573:0] peer: [6:7574435671577727538:0] socket: 124 qp: -1
2025-11-19T13:58:52.982687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435672654303608:2104] [node 6] ICS10 traffic start
2025-11-19T13:58:52.982697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:52.983000Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435671105615798:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.983008Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435671105615798:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.982355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435671359049782:2082] [node 5] ICP01 ready to work
2025-11-19T13:58:52.983155Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435671105615781:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.983160Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435671105615781:2105] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:52.981535Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435671577727545:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.981591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435671577727466:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:52.981473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435670887331466:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64841
2025-11-19T13:58:52.981522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435670887331467:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64841
2025-11-19T13:58:52.982220Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435670887331466:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.982243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435670887331390:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-19T13:58:52.983977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435671577727548:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52337
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.984005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435671577727549:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52337
2025-11-19T13:58:52.984056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435671577727550:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52337
2025-11-19T13:58:52.984209Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435671359049767:2100] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.985512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435671497311599:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37467
2025-11-19T13:58:52.985167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435671359049742:2078] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.985180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:52.985186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435671359049742:2078] [node 6] ICP22 created new session: [4:7574435671359049787:2078]
2025-11-19T13:58:52.985190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435671359049787:2078] [node 6] ICS09 handshake done sender: [4:7574435671359049767:2100] self: [4:7574435671359049743:0] peer: [6:7574435671577727544:0] socket: 142 qp: -1
2025-11-19T13:58:52.985195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435671359049787:2078] [node 6] ICS10 traffic start
2025-11-19T13:58:52.985211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:52.984153Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435670887331457:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:52.984312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435670887331388:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.984327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:52.984332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435670887331388:2048] [node 1] ICP22 created new session: [5:7574435670887331471:2048]
2025-11-19T13:58:52.984338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435670887331471:2048] [node 1] ICS09 handshake done sender: [5:7574435670887331457:2109] self: [5:7574435670887331464:0] peer: [1:7574435672654303580:0] socket: 143 qp: -1
2025-11-19T13:58:52.984344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435670887331471:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:52.984350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:52.987528Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435671577727527:2102] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:52.988045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435672654303578:2106] [node 5] ICP35 event processing took too much time PendingConnection:278(16089514ns) {HandleHandshakeStatus:329(16088431ns) {DropIncomingHandshake:441(3637ns) DropOutgoingHandshake:456(353ns) SwitchToState:178(60ns) ProcessPendingSessionEvents:495(15986382ns) {ForwardSessionEventToSession:612(15975689ns) ForwardSessionEventToSession:612(1387ns) ForwardSessionEventToSession:612(747ns) ForwardSessionEventToSession:612(818ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(526ns) ForwardSessionEventToSession:612(959ns) ForwardSessionEventToSession:612(572ns) ForwardSessionEventToSession:612(525ns) ForwardSessionEventToSession:612(1269ns)}}}
2025-11-19T13:58:52.988048Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435672654303598:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:52.988151Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435671497311599:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:52.988174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435672654303579:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.988188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:52.988192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435672654303579:2107] [node 4] ICP22 created new session: [1:7574435672654303611:2107]
2025-11-19T13:58:52.988195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435672654303611:2107] [node 4] ICS09 handshake done sender: [1:7574435672654303598:2117] self: [1:7574435672654303581:0] peer: [4:7574435671359049784:0] socket: 157 qp: -1
2025-11-19T13:58:52.988199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435672654303611:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:52.988204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:52.990431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435670887331475:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64841
2025-11-19T13:58:52.990400Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435671105615798:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:52.987776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435671577727464:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.987796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:52.987801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435671577727464:2048] [node 1] ICP22 created new session: [6:7574435671577727551:2048]
2025-11-19T13:58:52.987807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435671577727551:2048] [node 1] ICS09 handshake done sender: [6:7574435671577727527:2102] self: [6:7574435671577727538:0] peer: [1:7574435672654303573:0] socket: 126 qp: -1
2025-11-19T13:58:52.987812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435671577727551:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:52.987829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:52.990567Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435670887331466:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:52.990770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435670887331390:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:52.990785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:52.990790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435670887331390:2050] [node 3] ICP22 created new session: [5:7574435670887331476:2050]
2025-11-19T13:58:52.990795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435670887331476:2050] [node 3] ICS09 handshake done sender: [5:7574435670887331466:2115] self: [5:7574435670887331468:0] peer: [3:7574435671105615766:0] socket: 137 qp: -1
2025-11-19T13:58:52.990801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435670887331476:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:52.990815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:52.990596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435671105615758:2087] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.990615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:52.990621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435671105615758:2087] [node 5] ICP22 created new session: [3:7574435671105615805:2087]
2025-11-19T13:58:52.990627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435671105615805:2087] [node 5] ICS09 handshake done sender: [3:7574435671105615798:2118] self: [3:7574435671105615766:0] peer: [5:7574435670887331468:0] socket: 161 qp: -1
2025-11-19T13:58:52.990632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435671105615805:2087] [node 5] ICS10 traffic start
2025-11-19T13:58:52.990639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:52.992451Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435671105615781:2105] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:52.992563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435671105615751:2082] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:52.992576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:52.992581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435671105615751:2082] [node 6] ICP22 created new session: [3:7574435671105615809:2082]
2025-11-19T13:58:52.992586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435671105615809:2082] [node 6] ICS09 handshake done sender: [3:7574435671105615781:2105] self: [3:7574435671105615765:0] peer: [6:7574435671577727547:0] socket: 154 qp: -1
2025-11-19T13:58:52.992591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435671105615809:2082] [node 6] ICS10 traffic start
2025-11-19T13:58:52.992599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:52.998809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435672654303572:2104] [node 6] ICP35 event processing took too much time PendingConnection:278(16118760ns) {HandleHandshakeStatus:329(16117496ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(341ns) SwitchToState:178(49ns) ProcessPendingSessionEvents:495(11155601ns) {ForwardSessionEventToSession:612(11140142ns) ForwardSessionEventToSession:612(2871ns) ForwardSessionEventToSession:612(1241ns) ForwardSessionEventToSession:612(589ns) ForwardSessionEventToSession:612(810ns) ForwardSessionEventToSession:612(1445ns) ForwardSessionEventToSession:612(599ns) ForwardSessionEventToSession:612(760ns) ForwardSessionEventToSession:612(561ns) ForwardSessionEventToSession:612(1258ns)}}}
2025-11-19T13:58:53.000547Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435671577727540:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:53.004103Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435671359049707:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18289) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435671359049783:2114] held: no
2025-11-19T13:58:53.004222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435671577727464:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16411659ns) {HandleHandshakeStatus:329(16411122ns) {DropIncomingHandshake:441(425ns) DropOutgoingHandshake:456(2132ns) SwitchToState:178(57ns) ProcessPendingSessionEvents:495(45252ns) {ForwardSessionEventToSession:612(33854ns) ForwardSessionEventToSession:612(1233ns) ForwardSessionEventToSession:612(934ns) ForwardSessionEventToSession:612(647ns) ForwardSessionEventToSession:612(612ns) ForwardSessionEventToSession:612(968ns) ForwardSessionEventToSession:612(593ns) ForwardSessionEventToSession:612(526ns) ForwardSessionEventToSession:612(684ns) ForwardSessionEventToSession:612(1449ns)}}}
2025-11-19T13:58:53.004920Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435671497311591:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.004935Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435671497311591:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.004588Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435670887331462:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.004653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435670887331389:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.007899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435675654017087:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17744
2025-11-19T13:58:53.007981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435675654017088:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17744
2025-11-19T13:58:53.007993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435675654017089:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17744
2025-11-19T13:58:53.008282Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435671359049783:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:53.008471Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435675654017089:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.008613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435671359049707:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.008631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:53.008649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435671359049707:2048] [node 1] ICP22 created new session: [4:7574435675654017091:2048]
2025-11-19T13:58:53.008656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435675654017091:2048] [node 1] ICS09 handshake done sender: [4:7574435671359049783:2114] self: [4:7574435671359049784:0] peer: [1:7574435672654303581:0] socket: 159 qp: -1
2025-11-19T13:58:53.008664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435675654017091:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:53.008674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:53.010827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435672654303579:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(22614722ns) {HandleHandshakeStatus:329(22614056ns) {DropIncomingHandshake:441(2591ns) DropOutgoingHandshake:456(86ns) SwitchToState:178(57ns) ProcessPendingSessionEvents:495(22565723ns) {ForwardSessionEventToSession:612(22553029ns) ForwardSessionEventToSession:612(1504ns) ForwardSessionEventToSession:612(755ns) ForwardSessionEventToSession:612(540ns) ForwardSessionEventToSession:612(604ns) ForwardSessionEventToSession:612(1388ns) ForwardSessionEventToSession:612(749ns) ForwardSessionEventToSession:612(553ns) ForwardSessionEventToSession:612(612ns) ForwardSessionEventToSession:612(506ns)}}}
2025-11-19T13:58:53.011659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435671577727495:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.011680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:53.011688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435671577727495:2059] [node 4] ICP22 created new session: [6:7574435675872694851:2059]
2025-11-19T13:58:53.011694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435675872694851:2059] [node 4] ICS09 handshake done sender: [6:7574435671577727540:2114] self: [6:7574435671577727544:0] peer: [4:7574435671359049743:0] socket: 151 qp: -1
2025-11-19T13:58:53.011700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435675872694851:2059] [node 4] ICS10 traffic start
2025-11-19T13:58:53.011731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:53.012608Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435671105615791:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.012655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435671105615714:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.012885Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435671497311577:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.012897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435671497311577:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.013137Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435675654017088:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.013168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435671359049709:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.013567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435675872694849:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52337
2025-11-19T13:58:53.013610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435675872694850:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52337
2025-11-19T13:58:53.013790Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435671105615800:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.013797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435671105615800:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.014676Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435675872694849:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.014726Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435671577727545:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:53.014824Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435671105615800:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:53.014927Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435671105615793:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.014944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435671105615752:2079] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.014957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:53.014961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435671105615752:2079] [node 4] ICP22 created new session: [3:7574435675400583109:2079]
2025-11-19T13:58:53.014966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435675400583109:2079] [node 4] ICS09 handshake done sender: [3:7574435671105615800:2119] self: [3:7574435671105615797:0] peer: [4:7574435675654017093:0] socket: 176 qp: -1
2025-11-19T13:58:53.014970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435675400583109:2079] [node 4] ICS10 traffic start
2025-11-19T13:58:53.014985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:53.015211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435671105615751:2082] [node 6] ICP18 handshake (actor [3:7574435671105615793:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.015872Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435671577727548:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.015886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435671577727466:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.015897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:53.015903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435671577727466:2050] [node 3] ICP22 created new session: [6:7574435675872694854:2050]
2025-11-19T13:58:53.015908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435675872694854:2050] [node 3] ICS09 handshake done sender: [6:7574435671577727545:2116] self: [6:7574435671577727547:0] peer: [3:7574435671105615765:0] socket: 160 qp: -1
2025-11-19T13:58:53.015919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435675872694854:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:53.015933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:53.016003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435671577727466:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:53.016080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435671577727518:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.016164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435671577727465:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.018960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435671359049709:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435675654017088:2117] held: no
2025-11-19T13:58:53.020051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435675182298776:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64841
2025-11-19T13:58:53.020096Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435670887331470:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.020098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435670887331470:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.020083Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435671497311596:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.020096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435671497311596:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.021267Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435671497311596:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:53.021409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435671497311573:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.021424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:53.021429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435671497311573:2109] [node 6] ICP22 created new session: [2:7574435675792278898:2109]
2025-11-19T13:58:53.021435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435675792278898:2109] [node 6] ICS09 handshake done sender: [2:7574435671497311596:2118] self: [2:7574435671497311594:0] peer: [6:7574435675872694857:0] socket: 163 qp: -1
2025-11-19T13:58:53.021440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435675792278898:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:53.021445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.021720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435675872694858:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52337
2025-11-19T13:58:53.021870Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435671577727548:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:53.021929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435671577727465:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.021941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:53.021956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435671577727465:2049] [node 2] ICP22 created new session: [6:7574435675872694859:2049]
2025-11-19T13:58:53.021962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435675872694859:2049] [node 2] ICS09 handshake done sender: [6:7574435671577727548:2118] self: [6:7574435675872694857:0] peer: [2:7574435671497311594:0] socket: 166 qp: -1
2025-11-19T13:58:53.021966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435675872694859:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:53.021971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:53.024992Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435671497311591:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:53.025132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435671497311570:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.025146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:53.025151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435671497311570:2107] [node 5] ICP22 created new session: [2:7574435675792278902:2107]
2025-11-19T13:58:53.025157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435675792278902:2107] [node 5] ICS09 handshake done sender: [2:7574435671497311591:2117] self: [2:7574435671497311583:0] peer: [5:7574435675182298775:0] socket: 155 qp: -1
2025-11-19T13:58:53.025163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435675792278902:2107] [node 5] ICS10 traffic start
2025-11-19T13:58:53.025182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:53.025710Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435671497311573:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:53.029218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435670887331389:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23442) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435670887331462:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.031470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435675654017095:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17744
2025-11-19T13:58:53.031556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435675654017094:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17744
2025-11-19T13:58:53.031225Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435670887331470:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.031686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435675400583111:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21538
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.031722Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435675182298776:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.031736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435670887331423:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.031763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:53.031770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435670887331423:2063] [node 6] ICP22 created new session: [5:7574435675182298778:2063]
2025-11-19T13:58:53.031776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435675182298778:2063] [node 6] ICS09 handshake done sender: [5:7574435670887331470:2117] self: [5:7574435670887331465:0] peer: [6:7574435675872694856:0] socket: 172 qp: -1
2025-11-19T13:58:53.031916Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435671497311577:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:53.031782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435675182298778:2063] [node 6] ICS10 traffic start
2025-11-19T13:58:53.031791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:53.031924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435675182298780:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64841
2025-11-19T13:58:53.031931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435670887331423:2063] [node 6] ICP18 handshake (actor [5:7574435675182298776:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.032089Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435675654017095:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.032129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435671359049708:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.032468Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435670887331462:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:53.032115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435671497311510:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.032139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:53.032145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435671497311510:2049] [node 3] ICP22 created new session: [2:7574435675792278905:2049]
2025-11-19T13:58:53.032151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435675792278905:2049] [node 3] ICS09 handshake done sender: [2:7574435671497311577:2111] self: [2:7574435671497311575:0] peer: [3:7574435675400583107:0] socket: 134 qp: -1
2025-11-19T13:58:53.032157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435675792278905:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:53.032163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:53.032427Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435671497311598:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.032431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435671497311598:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.032606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435670887331389:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.032618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:53.032622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435670887331389:2049] [node 2] ICP22 created new session: [5:7574435675182298781:2049]
2025-11-19T13:58:53.032629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435675182298781:2049] [node 2] ICS09 handshake done sender: [5:7574435670887331462:2114] self: [5:7574435675182298775:0] peer: [2:7574435671497311583:0] socket: 156 qp: -1
2025-11-19T13:58:53.032636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435675182298781:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:53.032643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:53.033908Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435675654017088:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.033961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435675654017097:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17744
2025-11-19T13:58:53.032105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435675872694861:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52337
2025-11-19T13:58:53.032336Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435671577727518:2066] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18941) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435675872694849:2121] held: no
2025-11-19T13:58:53.032885Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435675872694849:2121] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:53.034187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435671577727518:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.034196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:53.034202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435671577727518:2066] [node 5] ICP22 created new session: [6:7574435675872694862:2066]
2025-11-19T13:58:53.034207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435675872694862:2066] [node 5] ICS09 handshake done sender: [6:7574435675872694849:2121] self: [6:7574435675872694856:0] peer: [5:7574435670887331465:0] socket: 152 qp: -1
2025-11-19T13:58:53.034212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435675872694862:2066] [node 5] ICS10 traffic start
2025-11-19T13:58:53.034218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:53.034577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435671359049709:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.034589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:53.034593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435671359049709:2050] [node 3] ICP22 created new session: [4:7574435675654017098:2050]
2025-11-19T13:58:53.034597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435675654017098:2050] [node 3] ICS09 handshake done sender: [4:7574435675654017088:2117] self: [4:7574435675654017093:0] peer: [3:7574435671105615797:0] socket: 127 qp: -1
2025-11-19T13:58:53.034601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435675654017098:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:53.034609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:53.034892Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435671105615791:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:53.035034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435671105615714:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.035050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:53.035056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435671105615714:2049] [node 2] ICP22 created new session: [3:7574435675400583112:2049]
2025-11-19T13:58:53.035062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435675400583112:2049] [node 2] ICS09 handshake done sender: [3:7574435671105615791:2115] self: [3:7574435675400583107:0] peer: [2:7574435671497311575:0] socket: 138 qp: -1
2025-11-19T13:58:53.035067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435675400583112:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:53.035076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:53.035848Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435671497311598:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:53.036200Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435675654017095:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:53.036247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435671497311574:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.036276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435671359049708:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.036304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:53.036309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435671359049708:2049] [node 2] ICP22 created new session: [4:7574435675654017101:2049]
2025-11-19T13:58:53.036315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435675654017101:2049] [node 2] ICS09 handshake done sender: [4:7574435675654017095:2120] self: [4:7574435675654017096:0] peer: [2:7574435671497311595:0] socket: 185 qp: -1
2025-11-19T13:58:53.036320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435675654017101:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:53.036340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.037177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435675182298783:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64841
2025-11-19T13:58:53.037661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:53.037674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435671497311574:2108] [node 4] ICP22 created new session: [2:7574435675792278908:2108]
2025-11-19T13:58:53.037679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435675792278908:2108] [node 4] ICS09 handshake done sender: [2:7574435671497311598:2119] self: [2:7574435671497311595:0] peer: [4:7574435675654017096:0] socket: 180 qp: -1
2025-11-19T13:58:53.037684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435675792278908:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:53.037694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:53.044000Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435675182298783:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.044067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435670887331463:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.051973Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435671359049790:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.051990Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435671359049790:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.055160Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435671359049790:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:53.055336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435671359049782:2082] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.055367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:53.055375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435671359049782:2082] [node 5] ICP22 created new session: [4:7574435675654017104:2082]
2025-11-19T13:58:53.055380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435675654017104:2082] [node 5] ICS09 handshake done sender: [4:7574435671359049790:2115] self: [4:7574435671359049785:0] peer: [5:7574435675182298784:0] socket: 125 qp: -1
2025-11-19T13:58:53.055388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435675654017104:2082] [node 5] ICS10 traffic start
2025-11-19T13:58:53.052839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435675182298785:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64841
2025-11-19T13:58:53.055403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:53.055973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435670887331463:2070] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20549) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435675182298783:2122] held: no
2025-11-19T13:58:53.056023Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435675182298783:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:53.056552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435670887331463:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.056569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:53.056576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435670887331463:2070] [node 4] ICP22 created new session: [5:7574435675182298786:2070]
2025-11-19T13:58:53.056583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435675182298786:2070] [node 4] ICS09 handshake done sender: [5:7574435675182298783:2122] self: [5:7574435675182298784:0] peer: [4:7574435671359049785:0] socket: 128 qp: -1
2025-11-19T13:58:53.056589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435675182298786:2070] [node 4] ICS10 traffic start
2025-11-19T13:58:53.056596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:53.064526Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:53.064630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435671497311588:2048] [node 1] ICS01 socket: 123 reason# EndOfStream
2025-11-19T13:58:53.064640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435671497311509:2048] [node 1] ICP30 unregister session Session# [2:7574435671497311588:2048] VirtualId# [2:7574435671497311582:0]
2025-11-19T13:58:53.064648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435671497311588:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.064680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:53.064701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:53.068754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:53.068813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435675654017091:2048] [node 1] ICS01 socket: 159 reason# EndOfStream
2025-11-19T13:58:53.068819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435671359049707:2048] [node 1] ICP30 unregister session Session# [4:7574435675654017091:2048] VirtualId# [4:7574435671359049784:0]
2025-11-19T13:58:53.068823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435675654017091:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.068855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:53.068876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:53.065096Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:53.065160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435671105615794:2048] [node 1] ICS01 socket: 122 reason# EndOfStream
2025-11-19T13:58:53.065165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435671105615713:2048] [node 1] ICP30 unregister session Session# [3:7574435671105615794:2048] VirtualId# [3:7574435671105615767:0]
2025-11-19T13:58:53.065169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435671105615794:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.065194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:53.065210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:53.072023Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:53.072105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435671577727551:2048] [node 1] ICS07 socket disconnect 126 reason# EndOfStream
2025-11-19T13:58:53.072109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435671577727551:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.072158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:53.072176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435671577727551:2048] [node 1] ICS15 start handshake
2025-11-19T13:58:53.072524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:53.072593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435670887331471:2048] [node 1] ICS01 socket: 143 reason# EndOfStream
2025-11-19T13:58:53.072598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435670887331388:2048] [node 1] ICP30 unregister session Session# [5:7574435670887331471:2048] VirtualId# [5:7574435670887331464:0]
2025-11-19T13:58:53.072602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435670887331471:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.072643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:53.072673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:53.072693Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435671577727464:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18289) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:53.072704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435671577727551:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:53.072708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435671577727464:2048] [node 1] ICP30 unregister session Session# [6:7574435671577727551:2048] VirtualId# [6:7574435671577727538:0]
2025-11-19T13:58:53.072723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:53.072729Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435671577727464:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18289) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:53.072731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18289) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:53.078925Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:53.079017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435675400583112:2049] [node 2] ICS01 socket: 138 reason# EndOfStream
2025-11-19T13:58:53.079024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435671105615714:2049] [node 2] ICP30 unregister session Session# [3:7574435675400583112:2049] VirtualId# [3:7574435675400583107:0]
2025-11-19T13:58:53.079030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435675400583112:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.079064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:53.079083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:53.083688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:53.083501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:53.083785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435675654017101:2049] [node 2] ICS01 socket: 185 reason# EndOfStream
2025-11-19T13:58:53.083585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435675182298781:2049] [node 2] ICS01 socket: 156 reason# EndOfStream
2025-11-19T13:58:53.083592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435670887331389:2049] [node 2] ICP30 unregister session Session# [5:7574435675182298781:2049] VirtualId# [5:7574435675182298775:0]
2025-11-19T13:58:53.083796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435671359049708:2049] [node 2] ICP30 unregister session Session# [4:7574435675654017101:2049] VirtualId# [4:7574435675654017096:0]
2025-11-19T13:58:53.083800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435675654017101:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.083832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:53.083853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:53.083598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435675182298781:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.083637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:53.083659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:53.083877Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:53.083950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435675872694859:2049] [node 2] ICS01 socket: 166 reason# EndOfStream
2025-11-19T13:58:53.083956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435671577727465:2049] [node 2] ICP30 unregister session Session# [6:7574435675872694859:2049] VirtualId# [6:7574435675872694857:0]
2025-11-19T13:58:53.083961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435675872694859:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.084000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:53.084021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:53.085588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435671577727464:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:58:53.111693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:53.111804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435675872694854:2050] [node 3] ICS01 socket: 160 reason# EndOfStream
2025-11-19T13:58:53.111816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435671577727466:2050] [node 3] ICP30 unregister session Session# [6:7574435675872694854:2050] VirtualId# [6:7574435671577727547:0]
2025-11-19T13:58:53.111824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435675872694854:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.111873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:53.111897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:53.112297Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:53.112370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435670887331476:2050] [node 3] ICS01 socket: 137 reason# EndOfStream
2025-11-19T13:58:53.112376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435670887331390:2050] [node 3] ICP30 unregister session Session# [5:7574435670887331476:2050] VirtualId# [5:7574435670887331468:0]
2025-11-19T13:58:53.112381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435670887331476:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.112423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:53.112443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:53.124192Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:53.124305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435675872694851:2059] [node 4] ICS01 socket: 151 reason# EndOfStream
2025-11-19T13:58:53.124314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435671577727495:2059] [node 4] ICP30 unregister session Session# [6:7574435675872694851:2059] VirtualId# [6:7574435671577727544:0]
2025-11-19T13:58:53.124322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435675872694851:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.124377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:53.124403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:53.299974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435675081329985:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:53.299997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435675081329986:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:53.367894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435675916087824:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:53.367910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435675916087825:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:53.460425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435675140578079:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:53.460438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435675140578080:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:53.520754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435676168846113:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:53.520778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435676168846114:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:53.520782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435676168846115:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:53.571807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435675051532468:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:53.571822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435675051532469:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:53.571827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435675051532470:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:53.656482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435676782126227:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:53.656496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435676782126228:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:53.656501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435676782126229:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.718577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435675916087884:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23614
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.719393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435675916087885:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23614
2025-11-19T13:58:53.720209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435675140578126:2090] [node 6] ICP01 ready to work
2025-11-19T13:58:53.721276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435675081330046:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:53.720821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435676782126258:2068] [node 4] ICP01 ready to work
2025-11-19T13:58:53.723931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435675916087889:2109] [node 4] ICP01 ready to work
2025-11-19T13:58:53.724032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435675916087890:2110] [node 5] ICP01 ready to work
2025-11-19T13:58:53.724109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435675916087891:2111] [node 6] ICP01 ready to work
2025-11-19T13:58:53.724899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435676168846144:2072] [node 6] ICP01 ready to work
2025-11-19T13:58:53.724987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435676168846146:2074] [node 5] ICP01 ready to work
2025-11-19T13:58:53.727462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435675140578138:2095] [node 4] ICP01 ready to work
2025-11-19T13:58:53.727556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435675140578139:2097] [node 5] ICP01 ready to work
2025-11-19T13:58:53.729630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435675051532516:2068] [node 6] ICP01 ready to work
2025-11-19T13:58:53.729735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435675051532517:2076] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.747819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435675051532542:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29540
2025-11-19T13:58:53.747847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435675051532543:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29540
2025-11-19T13:58:53.743900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435675916087889:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19717691ns) {Configure:104(19716895ns) {ProcessConfigured:135(19342029ns) {StartInitialHandshake:157(19341396ns) {DropHandshakes:469(561ns) {DropIncomingHandshake:441(86ns) DropOutgoingHandshake:456(85ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:476(1939ns) {SwitchToState:178(1332ns)}}}}}
2025-11-19T13:58:53.748033Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435675916087884:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.750576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435676168846186:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23893
2025-11-19T13:58:53.750601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435676168846187:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23893
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.756480Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435676782126228:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(35592453ns) {Configure:104(35591755ns) {ProcessConfigured:135(35288128ns) {StartInitialHandshake:157(35287413ns) {DropHandshakes:469(398ns) {DropIncomingHandshake:441(46ns) DropOutgoingHandshake:456(77ns)} GenerateSessionVirtualId:988(97ns) PrepareNewSessionHandshake:476(1736ns) {SwitchToState:178(1263ns)}}}}}
2025-11-19T13:58:53.757225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435676782126296:2080] [node 5] ICP01 ready to work
2025-11-19T13:58:53.757858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435676782126297:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36151
2025-11-19T13:58:53.757924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435676782126298:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36151
2025-11-19T13:58:53.758363Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435675081330046:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(37039096ns) {Configure:104(37038713ns) {ProcessConfigured:135(36750067ns) {StartInitialHandshake:157(36749690ns) {DropHandshakes:469(574ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(89ns) PrepareNewSessionHandshake:476(725ns) {SwitchToState:178(288ns)}}}}}
2025-11-19T13:58:53.758494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435675081330051:2108] [node 5] ICP01 ready to work
2025-11-19T13:58:53.758561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435675081330052:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:53.757936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435676782126299:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36151
2025-11-19T13:58:53.757967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435676782126300:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36151
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.768559Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435676782126297:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.768606Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435676782126299:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.769093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435675081330053:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24135
2025-11-19T13:58:53.769157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435675081330054:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24135
2025-11-19T13:58:53.769809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435676168846193:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23893
2025-11-19T13:58:53.769831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435676168846194:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23893
2025-11-19T13:58:53.772003Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435675081330053:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.779098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435675140578158:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4633
2025-11-19T13:58:53.779119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435675140578159:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4633
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.779934Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435675916087885:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.782047Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435676168846186:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.782112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435676168846115:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.782068Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435675081330054:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.783603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435676782126258:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.783626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435676782126296:2080] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.787840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435675916087906:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23614
2025-11-19T13:58:53.787878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435675916087907:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23614
2025-11-19T13:58:53.788098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435676168846187:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.788133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435676168846114:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.788214Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435675051532543:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.788243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435675051532469:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.789830Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435676168846194:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.789854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435676168846113:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.790063Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435675140578141:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.790076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435675140578141:2104] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.790769Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435675051532542:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.790794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435675051532470:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.790970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435676168846198:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23893
2025-11-19T13:58:53.793479Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435675051532519:2092] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.793488Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435675051532519:2092] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.794309Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435676168846159:2087] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.794317Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435676168846159:2087] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.795178Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435675051532519:2092] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:53.795287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435675051532516:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.795308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:53.795317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435675051532516:2068] [node 6] ICP22 created new session: [5:7574435675051532551:2068]
2025-11-19T13:58:53.795323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435675051532551:2068] [node 6] ICS09 handshake done sender: [5:7574435675051532519:2092] self: [5:7574435675051532518:0] peer: [6:7574435676782126310:0] socket: 136 qp: -1
2025-11-19T13:58:53.795332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435675051532551:2068] [node 6] ICS10 traffic start
2025-11-19T13:58:53.795343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:53.795753Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435676168846159:2087] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:53.793276Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435676782126298:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.793314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435676782126229:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.795851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435676168846144:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.795868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:53.795874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435676168846144:2072] [node 6] ICP22 created new session: [4:7574435676168846200:2072]
2025-11-19T13:58:53.795879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435676168846200:2072] [node 6] ICS09 handshake done sender: [4:7574435676168846159:2087] self: [4:7574435676168846157:0] peer: [6:7574435676782126309:0] socket: 131 qp: -1
2025-11-19T13:58:53.795885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435676168846200:2072] [node 6] ICS10 traffic start
2025-11-19T13:58:53.795891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.797367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435675051532554:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29540
2025-11-19T13:58:53.797489Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435675140578141:2104] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.797631Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435675140578143:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.797640Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435675140578143:2105] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.798042Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435675140578137:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.798046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435675140578137:2103] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.798546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435675140578138:2095] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.798556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:53.798561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435675140578138:2095] [node 4] ICP22 created new session: [3:7574435675140578162:2095]
2025-11-19T13:58:53.798567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435675140578162:2095] [node 4] ICS09 handshake done sender: [3:7574435675140578141:2104] self: [3:7574435675140578140:0] peer: [4:7574435676168846195:0] socket: 133 qp: -1
2025-11-19T13:58:53.798572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435675140578162:2095] [node 4] ICS10 traffic start
2025-11-19T13:58:53.798580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:53.798840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435675140578161:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4633
2025-11-19T13:58:53.799828Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435675081330050:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.799844Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435675081330050:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.800262Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435675140578143:2105] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:53.800547Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435676168846186:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:53.800652Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435675081330050:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:53.800748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435675081330046:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.800766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:53.800772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435675081330046:2107] [node 4] ICP22 created new session: [1:7574435675081330069:2107]
2025-11-19T13:58:53.800778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435675081330069:2107] [node 4] ICS09 handshake done sender: [1:7574435675081330050:2110] self: [1:7574435675081330047:0] peer: [4:7574435676168846197:0] socket: 154 qp: -1
2025-11-19T13:58:53.800783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435675081330069:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:53.800790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.802367Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435675916087893:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.802384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435675916087893:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.802736Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435675916087895:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.802739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435675916087895:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.803454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435675916087909:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23614
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.803801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435675051532555:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29540
2025-11-19T13:58:53.803902Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435675916087893:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:53.804043Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435675916087895:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.804210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435675051532556:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29540
2025-11-19T13:58:53.804248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435675051532557:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29540
2025-11-19T13:58:53.805329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435675916087889:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.805350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:53.805356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435675916087889:2109] [node 4] ICP22 created new session: [2:7574435675916087912:2109]
2025-11-19T13:58:53.805361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435675916087912:2109] [node 4] ICS09 handshake done sender: [2:7574435675916087893:2112] self: [2:7574435675916087892:0] peer: [4:7574435676168846196:0] socket: 139 qp: -1
2025-11-19T13:58:53.805366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435675916087912:2109] [node 4] ICS10 traffic start
2025-11-19T13:58:53.805375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:53.805475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435675916087890:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.805483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:53.805487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435675916087890:2110] [node 5] ICP22 created new session: [2:7574435675916087914:2110]
2025-11-19T13:58:53.805491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435675916087914:2110] [node 5] ICS09 handshake done sender: [2:7574435675916087895:2113] self: [2:7574435675916087894:0] peer: [5:7574435675051532548:0] socket: 140 qp: -1
2025-11-19T13:58:53.805494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435675916087914:2110] [node 5] ICS10 traffic start
2025-11-19T13:58:53.805497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:53.805584Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435675916087907:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.805821Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435675140578137:2103] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:53.806003Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435675051532556:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.806015Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435675051532542:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:53.806076Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435675051532543:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:53.806132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435675051532470:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.806156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:53.806161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435675051532470:2050] [node 3] ICP22 created new session: [5:7574435675051532558:2050]
2025-11-19T13:58:53.806165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435675051532558:2050] [node 3] ICS09 handshake done sender: [5:7574435675051532542:2111] self: [5:7574435675051532549:0] peer: [3:7574435675140578142:0] socket: 137 qp: -1
2025-11-19T13:58:53.806170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435675051532558:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:53.806176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:53.806256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435675051532469:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.806262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:53.806265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435675051532469:2049] [node 2] ICP22 created new session: [5:7574435675051532560:2049]
2025-11-19T13:58:53.806268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435675051532560:2049] [node 2] ICS09 handshake done sender: [5:7574435675051532543:2112] self: [5:7574435675051532548:0] peer: [2:7574435675916087894:0] socket: 142 qp: -1
2025-11-19T13:58:53.806271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435675051532560:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:53.806275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.806940Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435676782126300:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.807004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435676782126312:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36151
2025-11-19T13:58:53.807131Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435675140578159:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.807154Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435675140578161:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.807160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435675140578139:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.807168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:53.807174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435675140578139:2097] [node 5] ICP22 created new session: [3:7574435675140578167:2097]
2025-11-19T13:58:53.807178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435675140578167:2097] [node 5] ICS09 handshake done sender: [3:7574435675140578143:2105] self: [3:7574435675140578142:0] peer: [5:7574435675051532549:0] socket: 134 qp: -1
2025-11-19T13:58:53.807182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435675140578167:2097] [node 5] ICS10 traffic start
2025-11-19T13:58:53.807188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:53.807318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435675140578126:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.807323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:53.807328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435675140578126:2090] [node 6] ICP22 created new session: [3:7574435675140578169:2090]
2025-11-19T13:58:53.807331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435675140578169:2090] [node 6] ICS09 handshake done sender: [3:7574435675140578137:2103] self: [3:7574435675140578131:0] peer: [6:7574435676782126311:0] socket: 132 qp: -1
2025-11-19T13:58:53.807334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435675140578169:2090] [node 6] ICS10 traffic start
2025-11-19T13:58:53.807339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:53.807408Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435675140578158:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.808603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435675140578139:2097] [node 5] ICP18 handshake (actor [3:7574435675140578161:2117]) is too old
2025-11-19T13:58:53.808626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435675140578126:2090] [node 6] ICP18 handshake (actor [3:7574435675140578158:2115]) is too old
2025-11-19T13:58:53.808649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435675140578079:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.809703Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435675916087906:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.809883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435675916087824:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.810162Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435675081330056:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.810173Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435675081330056:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.810855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435675081330072:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24135
2025-11-19T13:58:53.811083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435676168846115:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.811103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:53.811110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435676168846115:2050] [node 3] ICP22 created new session: [4:7574435676168846205:2050]
2025-11-19T13:58:53.811115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435676168846205:2050] [node 3] ICS09 handshake done sender: [4:7574435676168846186:2110] self: [4:7574435676168846195:0] peer: [3:7574435675140578140:0] socket: 138 qp: -1
2025-11-19T13:58:53.811120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435676168846205:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:53.811126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:53.811272Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435676168846113:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18270) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435676168846194:2118] held: no
2025-11-19T13:58:53.812168Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435675081330056:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.812376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435676782126313:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36151
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.812523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435676782126314:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36151
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.814669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435675916087920:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23614
2025-11-19T13:58:53.814881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435676782126228:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.814950Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435676782126299:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:53.815527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435675081329985:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.815544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:53.815551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435675081329985:2048] [node 2] ICP22 created new session: [1:7574435675081330074:2048]
2025-11-19T13:58:53.815557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435675081330074:2048] [node 2] ICS09 handshake done sender: [1:7574435675081330056:2113] self: [1:7574435675081330055:0] peer: [2:7574435675916087918:0] socket: 155 qp: -1
2025-11-19T13:58:53.815563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435675081330074:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:53.815570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:53.815920Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435676782126297:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:53.815988Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435676782126298:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:53.816073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435676782126296:2080] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.816086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:53.816091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435676782126296:2080] [node 5] ICP22 created new session: [6:7574435676782126317:2080]
2025-11-19T13:58:53.816096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435676782126317:2080] [node 5] ICS09 handshake done sender: [6:7574435676782126299:2114] self: [6:7574435676782126310:0] peer: [5:7574435675051532518:0] socket: 146 qp: -1
2025-11-19T13:58:53.816101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435676782126317:2080] [node 5] ICS10 traffic start
2025-11-19T13:58:53.816106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:53.816218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435676782126258:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.816224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:53.816228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435676782126258:2068] [node 4] ICP22 created new session: [6:7574435676782126319:2068]
2025-11-19T13:58:53.816232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435676782126319:2068] [node 4] ICS09 handshake done sender: [6:7574435676782126297:2112] self: [6:7574435676782126309:0] peer: [4:7574435676168846157:0] socket: 144 qp: -1
2025-11-19T13:58:53.816236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435676782126319:2068] [node 4] ICS10 traffic start
2025-11-19T13:58:53.816242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:53.816322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435676782126229:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.816329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:53.816332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435676782126229:2050] [node 3] ICP22 created new session: [6:7574435676782126321:2050]
2025-11-19T13:58:53.816336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435676782126321:2050] [node 3] ICS09 handshake done sender: [6:7574435676782126298:2113] self: [6:7574435676782126311:0] peer: [3:7574435675140578131:0] socket: 145 qp: -1
2025-11-19T13:58:53.816339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435676782126321:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:53.816344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.818264Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435676168846193:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.818416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435676168846114:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15964) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435676168846187:2111] held: no
2025-11-19T13:58:53.818556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435676168846203:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23893
2025-11-19T13:58:53.818722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435675051532468:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.818767Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435675051532554:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.819036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435675051532517:2076] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.819726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435675140578173:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4633
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.819966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435676168846204:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23893
2025-11-19T13:58:53.821301Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435676168846161:2088] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.821310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435676168846161:2088] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.821694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435676168846144:2072] [node 6] ICP18 handshake (actor [4:7574435676168846193:2117]) is too old
2025-11-19T13:58:53.821716Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435676168846194:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.823315Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435675081330060:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.823325Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435675081330060:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.823707Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435675081330058:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.823711Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435675081330058:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.824323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435675081330077:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24135
2025-11-19T13:58:53.824482Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435676168846187:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.824822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435676782126316:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36151
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.825110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435676168846113:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.825123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:53.825129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435676168846113:2048] [node 1] ICP22 created new session: [4:7574435676168846209:2048]
2025-11-19T13:58:53.825135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435676168846209:2048] [node 1] ICS09 handshake done sender: [4:7574435676168846194:2118] self: [4:7574435676168846197:0] peer: [1:7574435675081330047:0] socket: 157 qp: -1
2025-11-19T13:58:53.825141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435676168846209:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:53.825148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:53.825221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435676168846114:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.825227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:53.825230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435676168846114:2049] [node 2] ICP22 created new session: [4:7574435676168846211:2049]
2025-11-19T13:58:53.825234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435676168846211:2049] [node 2] ICS09 handshake done sender: [4:7574435676168846187:2111] self: [4:7574435676168846196:0] peer: [2:7574435675916087892:0] socket: 143 qp: -1
2025-11-19T13:58:53.825238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435676168846211:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:53.825242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:53.825310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435676168846207:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23893
2025-11-19T13:58:53.825586Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435675916087897:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.825596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435675916087897:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.826152Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435675916087909:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.826242Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435675916087907:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.827450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435675051532565:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29540
2025-11-19T13:58:53.827466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435675051532566:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29540
2025-11-19T13:58:53.827565Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435676168846161:2088] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.827701Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435675140578173:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.827761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435675140578174:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4633
2025-11-19T13:58:53.827774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435675140578080:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.827969Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435675051532554:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:53.828088Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435676168846207:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.828099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435676168846146:2074] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.828106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:53.828110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435676168846146:2074] [node 5] ICP22 created new session: [4:7574435676168846213:2074]
2025-11-19T13:58:53.828115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435676168846213:2074] [node 5] ICS09 handshake done sender: [4:7574435676168846161:2088] self: [4:7574435676168846160:0] peer: [5:7574435675051532564:0] socket: 167 qp: -1
2025-11-19T13:58:53.828119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435676168846213:2074] [node 5] ICS10 traffic start
2025-11-19T13:58:53.828124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:53.828222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435676168846146:2074] [node 5] ICP18 handshake (actor [4:7574435676168846207:2122]) is too old
2025-11-19T13:58:53.828384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435675051532517:2076] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.828397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:53.828401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435675051532517:2076] [node 4] ICP22 created new session: [5:7574435675051532567:2076]
2025-11-19T13:58:53.828405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435675051532567:2076] [node 4] ICS09 handshake done sender: [5:7574435675051532554:2117] self: [5:7574435675051532564:0] peer: [4:7574435676168846160:0] socket: 168 qp: -1
2025-11-19T13:58:53.828410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435675051532567:2076] [node 4] ICS10 traffic start
2025-11-19T13:58:53.828416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:53.829035Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435675081330060:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:53.829284Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435675081330058:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:53.829431Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435675140578159:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:53.829488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435675140578079:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.829499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:53.829503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435675140578079:2048] [node 1] ICP22 created new session: [3:7574435675140578176:2048]
2025-11-19T13:58:53.829508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435675140578176:2048] [node 1] ICS09 handshake done sender: [3:7574435675140578159:2116] self: [3:7574435675140578172:0] peer: [1:7574435675081330057:0] socket: 159 qp: -1
2025-11-19T13:58:53.829512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435675140578176:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:53.829518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:53.830279Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435675081330072:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.830301Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435675081330077:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.830307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435675081330051:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.830317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:53.830322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435675081330051:2108] [node 5] ICP22 created new session: [1:7574435675081330081:2108]
2025-11-19T13:58:53.830326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435675081330081:2108] [node 5] ICS09 handshake done sender: [1:7574435675081330060:2115] self: [1:7574435675081330059:0] peer: [5:7574435675051532562:0] socket: 151 qp: -1
2025-11-19T13:58:53.830330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435675081330081:2108] [node 5] ICS10 traffic start
2025-11-19T13:58:53.830336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:53.830397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435675081329986:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.830403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:53.830406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435675081329986:2049] [node 3] ICP22 created new session: [1:7574435675081330083:2049]
2025-11-19T13:58:53.830409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435675081330083:2049] [node 3] ICS09 handshake done sender: [1:7574435675081330058:2114] self: [1:7574435675081330057:0] peer: [3:7574435675140578172:0] socket: 156 qp: -1
2025-11-19T13:58:53.830412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435675081330083:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:53.830415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:53.830842Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435675916087901:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.830850Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435675916087901:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.831272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435675916087824:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.831281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:53.831286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435675916087824:2048] [node 1] ICP22 created new session: [2:7574435675916087922:2048]
2025-11-19T13:58:53.831290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435675916087922:2048] [node 1] ICS09 handshake done sender: [2:7574435675916087907:2118] self: [2:7574435675916087918:0] peer: [1:7574435675081330055:0] socket: 161 qp: -1
2025-11-19T13:58:53.831295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435675916087922:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:53.831300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.832176Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435676782126316:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.832250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435676782126227:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:53.832264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435676782126324:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36151
2025-11-19T13:58:53.832457Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435675051532556:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:53.832663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435675051532468:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.832675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:53.832680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435675051532468:2048] [node 1] ICP22 created new session: [5:7574435675051532569:2048]
2025-11-19T13:58:53.832685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435675051532569:2048] [node 1] ICS09 handshake done sender: [5:7574435675051532556:2119] self: [5:7574435675051532562:0] peer: [1:7574435675081330059:0] socket: 178 qp: -1
2025-11-19T13:58:53.832690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435675051532569:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:53.832695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.833763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435675081329985:2048] [node 2] ICP18 handshake (actor [1:7574435675081330072:2117]) is too old
2025-11-19T13:58:53.833814Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435675081330062:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:53.833817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435675081330062:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:53.834337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435675081329986:2049] [node 3] ICP18 handshake (actor [1:7574435675081330077:2118]) is too old
2025-11-19T13:58:53.834371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435675081330080:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24135
2025-11-19T13:58:53.834826Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435675916087897:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:53.834995Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435676782126300:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:53.835093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435676782126228:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.835106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:53.835111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435676782126228:2049] [node 2] ICP22 created new session: [6:7574435676782126326:2049]
2025-11-19T13:58:53.835116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435676782126326:2049] [node 2] ICS09 handshake done sender: [6:7574435676782126300:2115] self: [6:7574435676782126315:0] peer: [2:7574435675916087896:0] socket: 147 qp: -1
2025-11-19T13:58:53.835121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435676782126326:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:53.835126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:53.836242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435675916087891:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.836257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:53.836263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435675916087891:2111] [node 6] ICP22 created new session: [2:7574435675916087926:2111]
2025-11-19T13:58:53.836267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435675916087926:2111] [node 6] ICS09 handshake done sender: [2:7574435675916087897:2114] self: [2:7574435675916087896:0] peer: [6:7574435676782126315:0] socket: 141 qp: -1
2025-11-19T13:58:53.836272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435675916087926:2111] [node 6] ICS10 traffic start
2025-11-19T13:58:53.836276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.836568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435675140578178:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4633
2025-11-19T13:58:53.839877Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435675081330080:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:53.840101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435676782126328:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36151
2025-11-19T13:58:53.840186Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435675081330062:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:53.840329Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435676782126316:2122] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:53.840388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435676782126227:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.840401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:53.840406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435676782126227:2048] [node 1] ICP22 created new session: [6:7574435676782126329:2048]
2025-11-19T13:58:53.840411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435676782126329:2048] [node 1] ICS09 handshake done sender: [6:7574435676782126316:2122] self: [6:7574435676782126325:0] peer: [1:7574435675081330061:0] socket: 187 qp: -1
2025-11-19T13:58:53.840415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435676782126329:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:53.840421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:53.841333Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435675916087901:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:53.841498Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435675140578173:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:53.841557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435675140578080:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:53.841570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:53.841575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435675140578080:2049] [node 2] ICP22 created new session: [3:7574435675140578179:2049]
2025-11-19T13:58:53.841579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435675140578179:2049] [node 2] ICS09 handshake done sender: [3:7574435675140578173:2118] self: [3:7574435675140578175:0] peer: [2:7574435675916087900:0] socket: 158 qp: -1
2025-11-19T13:58:53.841584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435675140578179:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:53.841590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:53.842507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435675081330052:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.842521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:53.842526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435675081330052:2109] [node 6] ICP22 created new session: [1:7574435675081330087:2109]
2025-11-19T13:58:53.842530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435675081330087:2109] [node 6] ICS09 handshake done sender: [1:7574435675081330062:2116] self: [1:7574435675081330061:0] peer: [6:7574435676782126325:0] socket: 166 qp: -1
2025-11-19T13:58:53.842535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435675081330087:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:53.842540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:53.842850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435675916087825:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:53.842863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:53.842866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435675916087825:2049] [node 3] ICP22 created new session: [2:7574435675916087928:2049]
2025-11-19T13:58:53.842871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435675916087928:2049] [node 3] ICS09 handshake done sender: [2:7574435675916087901:2116] self: [2:7574435675916087900:0] peer: [3:7574435675140578175:0] socket: 184 qp: -1
2025-11-19T13:58:53.842874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435675916087928:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:53.842879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:53.845570Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:53.845639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435675916087922:2048] [node 1] ICS01 socket: 161 reason# EndOfStream
2025-11-19T13:58:53.845646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435675916087824:2048] [node 1] ICP30 unregister session Session# [2:7574435675916087922:2048] VirtualId# [2:7574435675916087918:0]
2025-11-19T13:58:53.845652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435675916087922:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.845682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:53.845701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:53.846022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:53.846077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435675140578176:2048] [node 1] ICS01 socket: 159 reason# EndOfStream
2025-11-19T13:58:53.846082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435675140578079:2048] [node 1] ICP30 unregister session Session# [3:7574435675140578176:2048] VirtualId# [3:7574435675140578172:0]
2025-11-19T13:58:53.846085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435675140578176:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.846108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:53.846122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:53.851072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:53.851154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435676168846209:2048] [node 1] ICS01 socket: 157 reason# EndOfStream
2025-11-19T13:58:53.851161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435676168846113:2048] [node 1] ICP30 unregister session Session# [4:7574435676168846209:2048] VirtualId# [4:7574435676168846197:0]
2025-11-19T13:58:53.851167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435676168846209:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.851210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:53.851233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:53.852477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:53.852556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435675051532569:2048] [node 1] ICS01 socket: 178 reason# EndOfStream
2025-11-19T13:58:53.852562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435675051532468:2048] [node 1] ICP30 unregister session Session# [5:7574435675051532569:2048] VirtualId# [5:7574435675051532562:0]
2025-11-19T13:58:53.852567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435675051532569:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.852613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:53.852636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:53.852667Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:53.852737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435676782126329:2048] [node 1] ICS01 socket: 187 reason# EndOfStream
2025-11-19T13:58:53.852742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435676782126227:2048] [node 1] ICP30 unregister session Session# [6:7574435676782126329:2048] VirtualId# [6:7574435676782126325:0]
2025-11-19T13:58:53.852746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435676782126329:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.852775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:53.852798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:53.853441Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:53.853509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435675140578179:2049] [node 2] ICS01 socket: 158 reason# EndOfStream
2025-11-19T13:58:53.853515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435675140578080:2049] [node 2] ICP30 unregister session Session# [3:7574435675140578179:2049] VirtualId# [3:7574435675140578175:0]
2025-11-19T13:58:53.853519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435675140578179:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.853551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:53.853568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:53.863911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:53.864023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435675051532560:2049] [node 2] ICS01 socket: 142 reason# EndOfStream
2025-11-19T13:58:53.864034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435675051532469:2049] [node 2] ICP30 unregister session Session# [5:7574435675051532560:2049] VirtualId# [5:7574435675051532548:0]
2025-11-19T13:58:53.864043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435675051532560:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.864105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:53.864132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:53.864166Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:53.864236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435676782126326:2049] [node 2] ICS01 socket: 147 reason# EndOfStream
2025-11-19T13:58:53.864241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435676782126228:2049] [node 2] ICP30 unregister session Session# [6:7574435676782126326:2049] VirtualId# [6:7574435676782126315:0]
2025-11-19T13:58:53.864245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435676782126326:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.864274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:53.864296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:53.864434Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:53.864504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435676168846211:2049] [node 2] ICS01 socket: 143 reason# EndOfStream
2025-11-19T13:58:53.864509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435676168846114:2049] [node 2] ICP30 unregister session Session# [4:7574435676168846211:2049] VirtualId# [4:7574435676168846196:0]
2025-11-19T13:58:53.864514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435676168846211:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.864556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:53.864580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:53.876511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:53.876622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435676782126321:2050] [node 3] ICS01 socket: 145 reason# EndOfStream
2025-11-19T13:58:53.876633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435676782126229:2050] [node 3] ICP30 unregister session Session# [6:7574435676782126321:2050] VirtualId# [6:7574435676782126311:0]
2025-11-19T13:58:53.876641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435676782126321:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.876694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:53.876722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:53.876760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:53.876825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435675051532558:2050] [node 3] ICS01 socket: 137 reason# EndOfStream
2025-11-19T13:58:53.876831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435675051532470:2050] [node 3] ICP30 unregister session Session# [5:7574435675051532558:2050] VirtualId# [5:7574435675051532549:0]
2025-11-19T13:58:53.876835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435675051532558:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.876869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:53.876893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:53.887915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:53.888024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435676782126319:2068] [node 4] ICS01 socket: 144 reason# EndOfStream
2025-11-19T13:58:53.888037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435676782126258:2068] [node 4] ICP30 unregister session Session# [6:7574435676782126319:2068] VirtualId# [6:7574435676782126309:0]
2025-11-19T13:58:53.888045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435676782126319:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:53.888102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:53.888127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:54.063900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435679512297704:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:54.063921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435679512297705:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:54.139913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435679402334475:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:54.139927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435679402334476:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:54.235853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435680542279602:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:54.235868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435680542279603:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:54.358383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435680615824643:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:54.358403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435680615824644:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:54.358409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435680615824645:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:54.423939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435677201541876:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:54.423957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435677201541877:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:54.423962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435677201541878:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:54.539876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435679454596254:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:54.539898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435679454596255:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:54.539903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435679454596256:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:54.552061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435679402334506:2075] [node 6] ICP01 ready to work
2025-11-19T13:58:54.553009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435679512297749:2084] [node 4] ICP01 ready to work
2025-11-19T13:58:54.553084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435679512297750:2086] [node 5] ICP01 ready to work
2025-11-19T13:58:54.553146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435679512297751:2088] [node 6] ICP01 ready to work
2025-11-19T13:58:54.553819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435677201541894:2063] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.554477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435677201541898:2065] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56915
2025-11-19T13:58:54.559531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435677201541923:2077] [node 4] ICP01 ready to work
2025-11-19T13:58:54.559332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435680542279628:2071] [node 4] ICP01 ready to work
2025-11-19T13:58:54.559470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435680542279638:2076] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.568268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435679454596300:2075] [node 5] ICP01 ready to work
2025-11-19T13:58:54.569054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435679454596299:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48110
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.588439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435679512297778:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63605
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.588412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435679402334541:2082] [node 5] ICP01 ready to work
2025-11-19T13:58:54.588519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435679402334542:2084] [node 4] ICP01 ready to work
2025-11-19T13:58:54.589595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435679402334521:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50029
2025-11-19T13:58:54.589608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435679402334522:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50029
2025-11-19T13:58:54.589638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435679402334545:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50029
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.592127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435680542279667:2079] [node 6] ICP01 ready to work
2025-11-19T13:58:54.593492Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435680542279602:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(41920849ns) {Configure:104(41918811ns) {ProcessConfigured:135(41514047ns) {StartInitialHandshake:157(41513695ns) {DropHandshakes:469(465ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(64ns)} GenerateSessionVirtualId:988(289ns) PrepareNewSessionHandshake:476(603ns) {SwitchToState:178(311ns)}}}}}
2025-11-19T13:58:54.593566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435680542279664:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33288
2025-11-19T13:58:54.593626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435680542279665:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33288
2025-11-19T13:58:54.593637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435680542279666:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33288
2025-11-19T13:58:54.609277Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435679402334506:2075] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(57119477ns) {Configure:104(57118148ns) {ProcessConfigured:135(56893235ns) {StartInitialHandshake:157(56892490ns) {DropHandshakes:469(265ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(68ns) PrepareNewSessionHandshake:476(1422ns) {SwitchToState:178(1087ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.619430Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435679512297778:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.616540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435677201541954:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56915
2025-11-19T13:58:54.616563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435677201541955:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56915
2025-11-19T13:58:54.616575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435677201541956:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56915
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.617872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435680615824712:2083] [node 6] ICP01 ready to work
2025-11-19T13:58:54.620041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435679512297785:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63605
2025-11-19T13:58:54.620287Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435677201541898:2065] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.620321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435677201541876:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:54.624593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435680542279603:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(31519954ns) {Configure:104(31519347ns) {ProcessConfigured:135(31351463ns) {StartInitialHandshake:157(31350312ns) {DropHandshakes:469(157ns) {DropIncomingHandshake:441(45ns) DropOutgoingHandshake:456(29ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(934ns) {SwitchToState:178(571ns)}}}}}
2025-11-19T13:58:54.624634Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435680542279664:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.624673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435680542279602:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.627825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435680615824709:2076] [node 5] ICP01 ready to work
2025-11-19T13:58:54.627996Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435679402334522:2091] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.635828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435680615824710:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
2025-11-19T13:58:54.635939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435680615824711:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.635879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435677201541878:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(76283648ns) {Configure:104(76282744ns) {ProcessConfigured:135(76045956ns) {StartInitialHandshake:157(76044800ns) {DropHandshakes:469(392ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(104ns) PrepareNewSessionHandshake:476(798ns) {SwitchToState:178(749ns)}}}}}
2025-11-19T13:58:54.637181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435679454596330:2060] [node 4] ICP01 ready to work
2025-11-19T13:58:54.637301Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435679454596299:2091] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.637649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435679454596300:2075] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:54.637670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435679454596331:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48110
2025-11-19T13:58:54.640367Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435677201541896:2064] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.640382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435677201541896:2064] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:54.641202Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435677201541896:2064] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.641297Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435680542279665:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.641503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435677201541894:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.641522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:54.641530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435677201541894:2063] [node 6] ICP22 created new session: [5:7574435677201541960:2063]
2025-11-19T13:58:54.641537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435677201541960:2063] [node 6] ICS09 handshake done sender: [5:7574435677201541896:2064] self: [5:7574435677201541895:0] peer: [6:7574435679454596334:0] socket: 135 qp: -1
2025-11-19T13:58:54.641545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435677201541960:2063] [node 6] ICS10 traffic start
2025-11-19T13:58:54.641555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:54.641318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435679454596335:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48110
2025-11-19T13:58:54.637085Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435679512297755:2092] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.637103Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435679512297755:2092] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:54.637148Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435679512297747:2089] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.637153Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435679512297747:2089] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:54.644463Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435677201541954:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.644494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435677201541894:2063] [node 6] ICP18 handshake (actor [5:7574435677201541954:2115]) is too old
2025-11-19T13:58:54.647009Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435677201541956:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.647057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435677201541878:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:54.647299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435679454596300:2075] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21470) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435679454596299:2091] held: no
2025-11-19T13:58:54.648394Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435679512297755:2092] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:54.648624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435679512297750:2086] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.648641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:54.648648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435679512297750:2086] [node 5] ICP22 created new session: [1:7574435679512297788:2086]
2025-11-19T13:58:54.648655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435679512297788:2086] [node 5] ICS09 handshake done sender: [1:7574435679512297755:2092] self: [1:7574435679512297754:0] peer: [5:7574435677201541957:0] socket: 134 qp: -1
2025-11-19T13:58:54.648661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435679512297788:2086] [node 5] ICS10 traffic start
2025-11-19T13:58:54.648672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.648928Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435677201541955:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.648951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435677201541877:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:54.648731Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435680615824710:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.648963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435680615824643:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:54.651550Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435680542279666:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.651580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435680542279603:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:54.652163Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435680542279669:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.652174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435680542279669:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:54.655944Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435679454596299:2091] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:54.658094Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435679402334547:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.658118Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435679402334547:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:54.658245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435680615824720:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
2025-11-19T13:58:54.658385Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435680615824711:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.658414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435680615824645:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.659819Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435677201541876:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30200) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435677201541898:2065] held: no
2025-11-19T13:58:54.659894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435677201541965:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56915
2025-11-19T13:58:54.660051Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435679512297753:2091] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.660061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435679512297753:2091] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:54.660922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435679512297790:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63605
2025-11-19T13:58:54.662169Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435679512297747:2089] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:54.663261Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435679512297753:2091] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:54.663491Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435677201541898:2065] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:54.661481Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435680542279671:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.661492Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435680542279671:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.662330Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435679402334544:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.662342Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435679402334544:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:54.662925Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435679402334545:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.662952Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435679402334521:2090] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.664115Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435680615824720:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.664172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435680615824726:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
2025-11-19T13:58:54.664207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435680615824727:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
2025-11-19T13:58:54.664382Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435679402334544:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.664332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435679454596300:2075] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.664349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:54.664355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435679454596300:2075] [node 5] ICP22 created new session: [6:7574435679454596339:2075]
2025-11-19T13:58:54.664360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435679454596339:2075] [node 5] ICS09 handshake done sender: [6:7574435679454596299:2091] self: [6:7574435679454596334:0] peer: [5:7574435677201541895:0] socket: 145 qp: -1
2025-11-19T13:58:54.664365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435679454596339:2075] [node 5] ICS10 traffic start
2025-11-19T13:58:54.664370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:54.664497Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435679454596331:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.664571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435679454596337:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48110
2025-11-19T13:58:54.664599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435679454596338:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48110
2025-11-19T13:58:54.664615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435679454596256:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:54.664041Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435680542279669:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:54.663603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435679512297705:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.663621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:54.663627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435679512297705:2049] [node 3] ICP22 created new session: [1:7574435679512297793:2049]
2025-11-19T13:58:54.663633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435679512297793:2049] [node 3] ICS09 handshake done sender: [1:7574435679512297747:2089] self: [1:7574435679512297746:0] peer: [3:7574435680542279684:0] socket: 132 qp: -1
2025-11-19T13:58:54.663640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435679512297793:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:54.663649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:54.663774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435679512297749:2084] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.663781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:54.663785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435679512297749:2084] [node 4] ICP22 created new session: [1:7574435679512297795:2084]
2025-11-19T13:58:54.663788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435679512297795:2084] [node 4] ICS09 handshake done sender: [1:7574435679512297753:2091] self: [1:7574435679512297752:0] peer: [4:7574435680615824724:0] socket: 133 qp: -1
2025-11-19T13:58:54.663792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435679512297795:2084] [node 4] ICS10 traffic start
2025-11-19T13:58:54.663796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:54.664472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435679402334476:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.664486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:54.664491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435679402334476:2049] [node 3] ICP22 created new session: [2:7574435679402334561:2049]
2025-11-19T13:58:54.664496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435679402334561:2049] [node 3] ICS09 handshake done sender: [2:7574435679402334544:2109] self: [2:7574435679402334543:0] peer: [3:7574435680542279686:0] socket: 151 qp: -1
2025-11-19T13:58:54.664501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435679402334561:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:54.664509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.665197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435677201541876:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.665210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:54.665214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435677201541876:2048] [node 1] ICP22 created new session: [5:7574435677201541967:2048]
2025-11-19T13:58:54.665219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435677201541967:2048] [node 1] ICS09 handshake done sender: [5:7574435677201541898:2065] self: [5:7574435677201541957:0] peer: [1:7574435679512297754:0] socket: 136 qp: -1
2025-11-19T13:58:54.665223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435677201541967:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:54.665228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:54.665385Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435680542279671:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:54.665486Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435679512297790:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.665873Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435679402334547:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:54.665877Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435679454596337:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.665921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435679454596330:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:54.665953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435679402334541:2082] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.665968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:54.665973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435679402334541:2082] [node 5] ICP22 created new session: [2:7574435679402334565:2082]
2025-11-19T13:58:54.665977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435679402334565:2082] [node 5] ICS09 handshake done sender: [2:7574435679402334547:2111] self: [2:7574435679402334546:0] peer: [5:7574435677201541964:0] socket: 152 qp: -1
2025-11-19T13:58:54.665982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435679402334565:2082] [node 5] ICS10 traffic start
2025-11-19T13:58:54.665997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.667563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435679454596343:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48110
2025-11-19T13:58:54.668105Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435679512297785:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.668243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435679512297749:2084] [node 4] ICP18 handshake (actor [1:7574435679512297790:2117]) is too old
2025-11-19T13:58:54.668672Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435680615824714:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.668682Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435680615824714:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:54.669030Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435680615824710:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.669678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435679402334568:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50029
2025-11-19T13:58:54.669822Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435679454596338:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.669863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435679454596254:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:54.670067Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435679454596343:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.670082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435679454596255:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.669229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435677201541966:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56915
2025-11-19T13:58:54.670395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435680542279638:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.670411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:54.670418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435680542279638:2076] [node 5] ICP22 created new session: [3:7574435680542279691:2076]
2025-11-19T13:58:54.670425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435680542279691:2076] [node 5] ICS09 handshake done sender: [3:7574435680542279669:2109] self: [3:7574435680542279668:0] peer: [5:7574435677201541963:0] socket: 155 qp: -1
2025-11-19T13:58:54.670430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435680542279691:2076] [node 5] ICS10 traffic start
2025-11-19T13:58:54.670436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:54.670760Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435680542279673:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.670778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435680542279673:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:54.670300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435679512297799:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63605
2025-11-19T13:58:54.673212Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435679402334556:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.673221Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435679402334556:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.673881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435679454596346:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48110
2025-11-19T13:58:54.674002Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435679402334556:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:54.674091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435679402334506:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.674109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:54.674115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435679402334506:2075] [node 6] ICP22 created new session: [2:7574435679402334570:2075]
2025-11-19T13:58:54.674121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435679402334570:2075] [node 6] ICS09 handshake done sender: [2:7574435679402334556:2117] self: [2:7574435679402334507:0] peer: [6:7574435679454596345:0] socket: 188 qp: -1
2025-11-19T13:58:54.674126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435679402334570:2075] [node 6] ICS10 traffic start
2025-11-19T13:58:54.674132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:54.675124Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435679454596343:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:54.675310Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435679512297757:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.675319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435679512297757:2093] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:54.675691Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435679512297799:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.676414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435680542279628:2071] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.676430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:54.676435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435680542279628:2071] [node 4] ICP22 created new session: [3:7574435680542279693:2071]
2025-11-19T13:58:54.676441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435680542279693:2071] [node 4] ICS09 handshake done sender: [3:7574435680542279671:2110] self: [3:7574435680542279670:0] peer: [4:7574435680615824725:0] socket: 156 qp: -1
2025-11-19T13:58:54.676447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435680542279693:2071] [node 4] ICS10 traffic start
2025-11-19T13:58:54.676453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:54.676616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435680542279689:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33288
2025-11-19T13:58:54.676870Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435679512297765:2097] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.676877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435679512297765:2097] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:54.677378Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435679512297757:2093] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:54.675861Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435679402334568:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.675888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435679402334475:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.679405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435679512297751:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.679420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:54.679426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435679512297751:2088] [node 6] ICP22 created new session: [1:7574435679512297803:2088]
2025-11-19T13:58:54.679431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435679512297803:2088] [node 6] ICS09 handshake done sender: [1:7574435679512297757:2093] self: [1:7574435679512297756:0] peer: [6:7574435679454596344:0] socket: 174 qp: -1
2025-11-19T13:58:54.679436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435679512297803:2088] [node 6] ICS10 traffic start
2025-11-19T13:58:54.679441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:54.676762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435677201541877:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28099) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435677201541955:2116] held: no
2025-11-19T13:58:54.676787Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435677201541956:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.680067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435679402334573:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50029
2025-11-19T13:58:54.677445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435680615824643:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.677463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:54.677468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435680615824643:2048] [node 1] ICP22 created new session: [4:7574435680615824731:2048]
2025-11-19T13:58:54.677474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435680615824731:2048] [node 1] ICS09 handshake done sender: [4:7574435680615824710:2111] self: [4:7574435680615824724:0] peer: [1:7574435679512297752:0] socket: 141 qp: -1
2025-11-19T13:58:54.677478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435680615824731:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:54.677484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:54.677738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435679454596255:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.677755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:54.677759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435679454596255:2049] [node 2] ICP22 created new session: [6:7574435679454596347:2049]
2025-11-19T13:58:54.677764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435679454596347:2049] [node 2] ICS09 handshake done sender: [6:7574435679454596343:2120] self: [6:7574435679454596345:0] peer: [2:7574435679402334507:0] socket: 189 qp: -1
2025-11-19T13:58:54.677769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435679454596347:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:54.677774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:54.678100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435679454596255:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:54.678134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435679454596349:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48110
2025-11-19T13:58:54.678261Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435679454596338:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:54.678309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435679454596254:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.678319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:54.678323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435679454596254:2048] [node 1] ICP22 created new session: [6:7574435679454596350:2048]
2025-11-19T13:58:54.678327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435679454596350:2048] [node 1] ICS09 handshake done sender: [6:7574435679454596338:2119] self: [6:7574435679454596344:0] peer: [1:7574435679512297756:0] socket: 182 qp: -1
2025-11-19T13:58:54.678331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435679454596350:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:54.678335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:54.680219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435677201541878:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.680233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:54.680238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435677201541878:2050] [node 3] ICP22 created new session: [5:7574435677201541970:2050]
2025-11-19T13:58:54.680243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435677201541970:2050] [node 3] ICS09 handshake done sender: [5:7574435677201541956:2117] self: [5:7574435677201541963:0] peer: [3:7574435680542279668:0] socket: 161 qp: -1
2025-11-19T13:58:54.680247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435677201541970:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:54.680253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.680897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435677201541969:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56915
2025-11-19T13:58:54.683903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435680542279690:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33288
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.686126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435679402334574:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50029
2025-11-19T13:58:54.686236Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435679512297765:2097] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:54.684573Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435680542279664:2106] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:54.684128Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435677201541955:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:54.686579Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435680542279673:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:54.686774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435679512297704:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.686794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:54.686798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435679512297704:2048] [node 2] ICP22 created new session: [1:7574435679512297806:2048]
2025-11-19T13:58:54.686803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435679512297806:2048] [node 2] ICS09 handshake done sender: [1:7574435679512297765:2097] self: [1:7574435679512297764:0] peer: [2:7574435679402334572:0] socket: 190 qp: -1
2025-11-19T13:58:54.686808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435679512297806:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:54.686814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:54.687108Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435679402334568:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:54.687269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435677201541877:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.687283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:54.687289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435677201541877:2049] [node 2] ICP22 created new session: [5:7574435677201541973:2049]
2025-11-19T13:58:54.687293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435677201541973:2049] [node 2] ICS09 handshake done sender: [5:7574435677201541955:2116] self: [5:7574435677201541964:0] peer: [2:7574435679402334546:0] socket: 160 qp: -1
2025-11-19T13:58:54.687297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435677201541973:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:54.687303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.687624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435680542279667:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.687639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:54.687645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435680542279667:2079] [node 6] ICP22 created new session: [3:7574435680542279698:2079]
2025-11-19T13:58:54.687649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435680542279698:2079] [node 6] ICS09 handshake done sender: [3:7574435680542279673:2111] self: [3:7574435680542279672:0] peer: [6:7574435679454596341:0] socket: 157 qp: -1
2025-11-19T13:58:54.687653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435680542279698:2079] [node 6] ICS10 traffic start
2025-11-19T13:58:54.687657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:54.687738Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435680542279666:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:54.686021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435679454596353:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48110
2025-11-19T13:58:54.687826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435679512297805:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63605
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.688096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435680615824729:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
2025-11-19T13:58:54.688141Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435680615824711:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:54.688132Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435679454596331:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.688291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435680615824643:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:54.688456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435680615824733:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
2025-11-19T13:58:54.688642Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435680615824714:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:54.688204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435679402334475:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.688217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:54.688221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435679402334475:2048] [node 1] ICP22 created new session: [2:7574435679402334575:2048]
2025-11-19T13:58:54.688225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435679402334575:2048] [node 1] ICS09 handshake done sender: [2:7574435679402334568:2118] self: [2:7574435679402334572:0] peer: [1:7574435679512297764:0] socket: 191 qp: -1
2025-11-19T13:58:54.688230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435679402334575:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:54.688236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:54.691795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435680615824645:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.691810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:54.691815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435680615824645:2050] [node 3] ICP22 created new session: [4:7574435680615824735:2050]
2025-11-19T13:58:54.691821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435680615824735:2050] [node 3] ICS09 handshake done sender: [4:7574435680615824711:2112] self: [4:7574435680615824725:0] peer: [3:7574435680542279670:0] socket: 162 qp: -1
2025-11-19T13:58:54.691826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435680615824735:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:54.691832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:54.691939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435680615824712:2083] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.691945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:54.691949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435680615824712:2083] [node 6] ICP22 created new session: [4:7574435680615824737:2083]
2025-11-19T13:58:54.691953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435680615824737:2083] [node 6] ICS09 handshake done sender: [4:7574435680615824714:2113] self: [4:7574435680615824713:0] peer: [6:7574435679454596342:0] socket: 168 qp: -1
2025-11-19T13:58:54.691957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435680615824737:2083] [node 6] ICS10 traffic start
2025-11-19T13:58:54.691961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:54.692059Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435680615824729:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.692283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435679454596256:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.692296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:54.692300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435679454596256:2050] [node 3] ICP22 created new session: [6:7574435679454596355:2050]
2025-11-19T13:58:54.692304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435679454596355:2050] [node 3] ICS09 handshake done sender: [6:7574435679454596331:2115] self: [6:7574435679454596341:0] peer: [3:7574435680542279672:0] socket: 163 qp: -1
2025-11-19T13:58:54.692308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435679454596355:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:54.692314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:54.693230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435680615824644:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:54.693456Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435679402334549:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.693465Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435679402334549:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:54.689430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435680542279602:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.689447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:54.689452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435680542279602:2048] [node 1] ICP22 created new session: [3:7574435680542279701:2048]
2025-11-19T13:58:54.689458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435680542279701:2048] [node 1] ICS09 handshake done sender: [3:7574435680542279664:2106] self: [3:7574435680542279684:0] peer: [1:7574435679512297746:0] socket: 143 qp: -1
2025-11-19T13:58:54.689462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435680542279701:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:54.689468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.694149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435680615824733:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.695282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435680615824712:2083] [node 6] ICP18 handshake (actor [4:7574435680615824733:2121]) is too old
2025-11-19T13:58:54.695306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435680615824741:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18549
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.695972Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435679402334549:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:54.695575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435677201541975:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56915
2025-11-19T13:58:54.695711Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435677201541975:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.695731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435677201541923:2077] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:54.696030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435680542279603:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.696043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:54.696051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435680542279603:2049] [node 2] ICP22 created new session: [3:7574435680542279703:2049]
2025-11-19T13:58:54.696056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435680542279703:2049] [node 2] ICS09 handshake done sender: [3:7574435680542279666:2108] self: [3:7574435680542279686:0] peer: [2:7574435679402334543:0] socket: 154 qp: -1
2025-11-19T13:58:54.696086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435680542279703:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:54.696095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:54.696282Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435680615824723:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:54.696294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435680615824723:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:54.696702Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435679454596256:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.698338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435679454596354:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48110
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.698478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435680542279700:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33288
2025-11-19T13:58:54.698826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435679454596330:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30024) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435679454596337:2118] held: no
2025-11-19T13:58:54.698867Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435679454596337:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:54.699097Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435680615824729:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:54.697502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435679402334542:2084] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.697522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:54.697526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435679402334542:2084] [node 4] ICP22 created new session: [2:7574435679402334578:2084]
2025-11-19T13:58:54.697532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435679402334578:2084] [node 4] ICS09 handshake done sender: [2:7574435679402334549:2112] self: [2:7574435679402334548:0] peer: [4:7574435680615824740:0] socket: 184 qp: -1
2025-11-19T13:58:54.697537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435679402334578:2084] [node 4] ICS10 traffic start
2025-11-19T13:58:54.697543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:54.699537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435679454596330:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.699550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:54.699556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435679454596330:2060] [node 4] ICP22 created new session: [6:7574435679454596357:2060]
2025-11-19T13:58:54.699561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435679454596357:2060] [node 4] ICS09 handshake done sender: [6:7574435679454596337:2118] self: [6:7574435679454596342:0] peer: [4:7574435680615824713:0] socket: 171 qp: -1
2025-11-19T13:58:54.699567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435679454596357:2060] [node 4] ICS10 traffic start
2025-11-19T13:58:54.699572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:54.699790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435677201541977:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56915
2025-11-19T13:58:54.700569Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435677201541975:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:54.700643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435677201541923:2077] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.700655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:54.700661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435677201541923:2077] [node 4] ICP22 created new session: [5:7574435677201541978:2077]
2025-11-19T13:58:54.700666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435677201541978:2077] [node 4] ICS09 handshake done sender: [5:7574435677201541975:2122] self: [5:7574435677201541976:0] peer: [4:7574435680615824719:0] socket: 199 qp: -1
2025-11-19T13:58:54.700670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435677201541978:2077] [node 4] ICS10 traffic start
2025-11-19T13:58:54.700676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:54.700425Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435680615824723:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:54.701603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435680615824644:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28099) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435680615824729:2120] held: no
2025-11-19T13:58:54.701617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435680615824644:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:54.701625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:54.701630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435680615824644:2049] [node 2] ICP22 created new session: [4:7574435680615824743:2049]
2025-11-19T13:58:54.701635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435680615824743:2049] [node 2] ICS09 handshake done sender: [4:7574435680615824729:2120] self: [4:7574435680615824740:0] peer: [2:7574435679402334548:0] socket: 185 qp: -1
2025-11-19T13:58:54.701640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435680615824743:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:54.701644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:54.701761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435680615824709:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:54.701769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:54.701774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435680615824709:2076] [node 5] ICP22 created new session: [4:7574435680615824745:2076]
2025-11-19T13:58:54.701778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435680615824745:2076] [node 5] ICS09 handshake done sender: [4:7574435680615824723:2117] self: [4:7574435680615824719:0] peer: [5:7574435677201541976:0] socket: 201 qp: -1
2025-11-19T13:58:54.701781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435680615824745:2076] [node 5] ICS10 traffic start
2025-11-19T13:58:54.701785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:54.706455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435679402334476:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:54.714449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:54.714554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435679402334575:2048] [node 1] ICS01 socket: 191 reason# EndOfStream
2025-11-19T13:58:54.714564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435679402334475:2048] [node 1] ICP30 unregister session Session# [2:7574435679402334575:2048] VirtualId# [2:7574435679402334572:0]
2025-11-19T13:58:54.714573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435679402334575:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:54.714622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:54.714647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:54.716286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:54.716360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435680542279701:2048] [node 1] ICS01 socket: 143 reason# EndOfStream
2025-11-19T13:58:54.716368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435680542279602:2048] [node 1] ICP30 unregister session Session# [3:7574435680542279701:2048] VirtualId# [3:7574435680542279684:0]
2025-11-19T13:58:54.716373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435680542279701:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:54.716407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:54.716425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:54.723920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:54.724006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435677201541967:2048] [node 1] ICS01 socket: 136 reason# EndOfStream
2025-11-19T13:58:54.724014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435677201541876:2048] [node 1] ICP30 unregister session Session# [5:7574435677201541967:2048] VirtualId# [5:7574435677201541957:0]
2025-11-19T13:58:54.724020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435677201541967:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:54.724060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:54.724083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:54.724835Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:54.724907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435679454596350:2048] [node 1] ICS01 socket: 182 reason# EndOfStream
2025-11-19T13:58:54.724914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435679454596254:2048] [node 1] ICP30 unregister session Session# [6:7574435679454596350:2048] VirtualId# [6:7574435679454596344:0]
2025-11-19T13:58:54.724919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435679454596350:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:54.724959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:54.724979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:54.725057Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:54.725114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435680615824731:2048] [node 1] ICS07 socket disconnect 141 reason# EndOfStream
2025-11-19T13:58:54.725117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435680615824731:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:54.725141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:54.725158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435680615824731:2048] [node 1] ICS15 start handshake
2025-11-19T13:58:54.725640Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435680615824643:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30200) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:54.725650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435680615824731:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:54.725654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435680615824643:2048] [node 1] ICP30 unregister session Session# [4:7574435680615824731:2048] VirtualId# [4:7574435680615824724:0]
2025-11-19T13:58:54.725667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:54.725673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435680615824643:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30200) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:54.725677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30200) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:54.735824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435680615824643:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:58:54.736522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:54.736615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435680542279703:2049] [node 2] ICS07 socket disconnect 154 reason# EndOfStream
2025-11-19T13:58:54.736621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435680542279703:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:54.736655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:54.736670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435680542279703:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:54.751651Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:54.751766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435679454596347:2049] [node 2] ICS01 socket: 189 reason# EndOfStream
2025-11-19T13:58:54.751778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435679454596255:2049] [node 2] ICP30 unregister session Session# [6:7574435679454596347:2049] VirtualId# [6:7574435679454596345:0]
2025-11-19T13:58:54.751786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435679454596347:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:54.751838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:54.751861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:54.751900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:54.751956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435680615824743:2049] [node 2] ICS01 socket: 185 reason# EndOfStream
2025-11-19T13:58:54.751961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435680615824644:2049] [node 2] ICP30 unregister session Session# [4:7574435680615824743:2049] VirtualId# [4:7574435680615824740:0]
2025-11-19T13:58:54.751965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435680615824743:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:54.751992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:54.752011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:54.755882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:54.755977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435677201541973:2049] [node 2] ICS01 socket: 160 reason# EndOfStream
2025-11-19T13:58:54.755989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435677201541877:2049] [node 2] ICP30 unregister session Session# [5:7574435677201541973:2049] VirtualId# [5:7574435677201541964:0]
2025-11-19T13:58:54.755995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435677201541973:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:54.756036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:54.756056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:54.757751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T13:58:54.757814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435680615824735:2050] [node 3] ICS01 socket: 162 reason# EndOfStream
2025-11-19T13:58:54.757820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435680615824645:2050] [node 3] ICP30 unregister session Session# [4:7574435680615824735:2050] VirtualId# [4:7574435680615824725:0]
2025-11-19T13:58:54.757825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435680615824735:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:54.757858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:54.757877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:54.757900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:54.757949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435677201541970:2050] [node 3] ICS01 socket: 161 reason# EndOfStream
2025-11-19T13:58:54.757953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435677201541878:2050] [node 3] ICP30 unregister session Session# [5:7574435677201541970:2050] VirtualId# [5:7574435677201541963:0]
2025-11-19T13:58:54.757957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435677201541970:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:54.757976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:54.757992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:54.758012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:54.758064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435679454596355:2050] [node 3] ICS01 socket: 163 reason# EndOfStream
2025-11-19T13:58:54.758068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435679454596256:2050] [node 3] ICP30 unregister session Session# [6:7574435679454596355:2050] VirtualId# [6:7574435679454596341:0]
2025-11-19T13:58:54.758072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435679454596355:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:54.758091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:54.758107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:54.791041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:54.791147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435679454596357:2060] [node 4] ICS01 socket: 171 reason# EndOfStream
2025-11-19T13:58:54.791157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435679454596330:2060] [node 4] ICP30 unregister session Session# [6:7574435679454596357:2060] VirtualId# [6:7574435679454596342:0]
2025-11-19T13:58:54.791165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435679454596357:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:54.791213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:54.791235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:54.872085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435677283141882:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:54.872104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435677283141883:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:54.923964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435680831501720:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:54.923983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435680831501721:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:54.991882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435679207840403:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:54.991898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435679207840404:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:55.084075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435684851696464:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:55.084093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435684851696465:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:55.084097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435684851696466:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:55.154025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435681348347917:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:55.154041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435681348347918:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:55.154046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435681348347919:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:55.226108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435683565496654:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:55.226126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435683565496655:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:55.226131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435683565496656:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:55.264370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435683502807760:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.267249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435685126469077:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34583
2025-11-19T13:58:55.267420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435685126469083:2111] [node 4] ICP01 ready to work
2025-11-19T13:58:55.267866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435685126469084:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.276055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435681578109244:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:55.276209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435684851696530:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.281970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435683565496720:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33638
2025-11-19T13:58:55.282227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435681578109239:2107] [node 6] ICP01 ready to work
2025-11-19T13:58:55.282303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435681578109245:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18161
2025-11-19T13:58:55.282478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435681578109250:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18161
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.282987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435683502807766:2108] [node 6] ICP01 ready to work
2025-11-19T13:58:55.281904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435685126469091:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34583
2025-11-19T13:58:55.287889Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435683565496720:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.289647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435685126469088:2110] [node 6] ICP01 ready to work
2025-11-19T13:58:55.289890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435683565496715:2086] [node 5] ICP01 ready to work
2025-11-19T13:58:55.290329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435683565496716:2088] [node 4] ICP01 ready to work
2025-11-19T13:58:55.292133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435681348347983:2109] [node 4] ICP01 ready to work
2025-11-19T13:58:55.292237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435681348347984:2110] [node 6] ICP01 ready to work
2025-11-19T13:58:55.289495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435683502807768:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:55.293639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435683502807767:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49712
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.298584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435681578109257:2114] [node 5] ICP01 ready to work
2025-11-19T13:58:55.299092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435681578109255:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18161
2025-11-19T13:58:55.299202Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435681578109239:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16816422ns) {Configure:104(16815891ns) {ProcessConfigured:135(16561494ns) {StartInitialHandshake:157(16561230ns) {DropHandshakes:469(403ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:476(646ns) {SwitchToState:178(340ns)}}}}}
2025-11-19T13:58:55.299983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435684851696535:2111] [node 5] ICP01 ready to work
2025-11-19T13:58:55.303026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435679207840403:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(22625786ns) {Configure:104(22624693ns) {ProcessConfigured:135(22337061ns) {StartInitialHandshake:157(22336641ns) {DropHandshakes:469(615ns) {DropIncomingHandshake:441(103ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(179ns) PrepareNewSessionHandshake:476(1311ns) {SwitchToState:178(364ns)}}}}}
2025-11-19T13:58:55.303944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435681578109256:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18161
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.308606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435681348347987:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19800
2025-11-19T13:58:55.308627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435681348347988:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19800
2025-11-19T13:58:55.309549Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681578109250:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.309039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435683502807776:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49712
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.318598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435683565496715:2086] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(28645165ns) {Configure:104(28644795ns) {ProcessConfigured:135(28255368ns) {StartInitialHandshake:157(28255110ns) {DropHandshakes:469(536ns) {DropIncomingHandshake:441(99ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(405ns) PrepareNewSessionHandshake:476(703ns) {SwitchToState:178(200ns)}}}}}
2025-11-19T13:58:55.318675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435683565496716:2088] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.318718Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435683565496716:2088] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(28337250ns) {Configure:104(28337017ns) {ProcessConfigured:135(28139968ns) {StartInitialHandshake:157(28120598ns) {DropHandshakes:469(201ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(23ns)} GenerateSessionVirtualId:988(274ns) PrepareNewSessionHandshake:476(697ns) {SwitchToState:178(343ns)}} IncomingHandshake:261(11718ns) {DropIncomingHandshake:441(232ns) PrepareNewSessionHandshake:476(139ns) {SwitchToState:178(100ns)} GenerateSessionVirtualId:988(282ns) IssueIncomingHandshakeReply:193(8029ns) {SwitchToState:178(61ns)}}}}}
2025-11-19T13:58:55.319278Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681578109256:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.319534Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681578109243:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.319538Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435681578109243:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:55.316444Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435685126469088:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26597606ns) {Configure:104(26595631ns) {ProcessConfigured:135(26285444ns) {StartInitialHandshake:157(26284570ns) {DropHandshakes:469(1573ns) {DropIncomingHandshake:441(260ns) DropOutgoingHandshake:456(28ns)} GenerateSessionVirtualId:988(585ns) PrepareNewSessionHandshake:476(1444ns) {SwitchToState:178(183ns)}}}}}
2025-11-19T13:58:55.318108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435685126469096:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34583
2025-11-19T13:58:55.318148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435685126469097:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34583
2025-11-19T13:58:55.318296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435685126469083:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(50003626ns) {Configure:104(50002591ns) {ProcessConfigured:135(49797883ns) {StartInitialHandshake:157(49797313ns) {DropHandshakes:469(227ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(23ns)} GenerateSessionVirtualId:988(139ns) PrepareNewSessionHandshake:476(1080ns) {SwitchToState:178(434ns)}}}}}
2025-11-19T13:58:55.318347Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435685126469077:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.319378Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435685126469091:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.319403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435680831501720:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.320255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435681348347983:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(27974819ns) {Configure:104(27973443ns) {ProcessConfigured:135(27682628ns) {StartInitialHandshake:157(27682051ns) {DropHandshakes:469(471ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(64ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(1613ns) {SwitchToState:178(858ns)}}}}}
2025-11-19T13:58:55.320366Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435681348347984:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27517435ns) {Configure:104(27516533ns) {ProcessConfigured:135(27298294ns) {StartInitialHandshake:157(27298169ns) {DropHandshakes:469(299ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(1120ns) {SwitchToState:178(140ns)}}}}}
2025-11-19T13:58:55.320230Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435681578109243:2109] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:55.320331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435677283141882:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.320350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:55.320359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435677283141882:2048] [node 2] ICP22 created new session: [1:7574435681578109264:2048]
2025-11-19T13:58:55.320366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435681578109264:2048] [node 2] ICS09 handshake done sender: [1:7574435681578109243:2109] self: [1:7574435681578109241:0] peer: [2:7574435685126469100:0] socket: 147 qp: -1
2025-11-19T13:58:55.320374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435681578109264:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:55.320394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:55.321628Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681578109255:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.323777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435683502807777:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49712
2025-11-19T13:58:55.321091Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435685126469096:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.327883Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435681348347987:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.325656Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681578109245:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.326248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435683502807766:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23154479ns) {Configure:104(23153493ns) {ProcessConfigured:135(22897902ns) {StartInitialHandshake:157(22897396ns) {DropHandshakes:469(420ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(25ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(2692ns) {SwitchToState:178(2042ns)}}}}}
2025-11-19T13:58:55.326358Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435683502807776:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.326379Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435683502807767:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.326426Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435683502807777:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.326628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435679207840404:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.326637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435679207840403:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.327969Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435684851696465:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(59331423ns) {Configure:104(59330100ns) {ProcessConfigured:135(59108620ns) {StartInitialHandshake:157(59107791ns) {DropHandshakes:469(332ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(154ns) PrepareNewSessionHandshake:476(974ns) {SwitchToState:178(759ns)}}}}}
2025-11-19T13:58:55.331116Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435684851696532:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.331137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435684851696532:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:55.331791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435683502807778:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49712
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.337781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435681348347998:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19800
2025-11-19T13:58:55.337976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435685126469102:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34583
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.340287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435677283141882:2048] [node 2] ICP18 handshake (actor [1:7574435681578109255:2115]) is too old
2025-11-19T13:58:55.340318Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [1:7574435677283141882:2048] [node 2] ICP35 event processing took too much time StateWork:293(17474412ns) {IncomingHandshake:261(17473309ns)}
2025-11-19T13:58:55.339944Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435685126469074:2104] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.339978Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435685126469074:2104] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:55.340045Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435685126469090:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.340052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435685126469090:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:55.341786Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435681348347998:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.341849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435681348347917:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.339649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435684851696540:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23901
2025-11-19T13:58:55.339718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435684851696541:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23901
2025-11-19T13:58:55.342079Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435684851696540:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.348039Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435681348347988:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.348094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435681348347983:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.348225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435684851696537:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.348238Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435684851696537:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.349439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435681348348001:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19800
2025-11-19T13:58:55.349540Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435684851696537:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:55.349732Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435681348347988:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.350285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435683565496734:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33638
2025-11-19T13:58:55.349799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435681348347983:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.349815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:55.349821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435681348347983:2109] [node 4] ICP22 created new session: [5:7574435681348348002:2109]
2025-11-19T13:58:55.349827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435681348348002:2109] [node 4] ICS09 handshake done sender: [5:7574435681348347988:2112] self: [5:7574435681348348000:0] peer: [4:7574435684851696536:0] socket: 162 qp: -1
2025-11-19T13:58:55.349835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435681348348002:2109] [node 4] ICS10 traffic start
2025-11-19T13:58:55.349855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:55.351967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435684851696535:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.351984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:55.351989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435684851696535:2111] [node 5] ICP22 created new session: [4:7574435684851696546:2111]
2025-11-19T13:58:55.351995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435684851696546:2111] [node 5] ICS09 handshake done sender: [4:7574435684851696537:2112] self: [4:7574435684851696536:0] peer: [5:7574435681348348000:0] socket: 161 qp: -1
2025-11-19T13:58:55.352001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435684851696546:2111] [node 5] ICS10 traffic start
2025-11-19T13:58:55.352009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:55.353925Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435683565496734:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.353962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435683565496715:2086] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.354320Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435681348347996:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.354328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435681348347996:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.354999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435683565496736:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33638
2025-11-19T13:58:55.355136Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435681348347996:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:55.355304Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435683565496734:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:55.355415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435683565496715:2086] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.355429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:55.355434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435683565496715:2086] [node 5] ICP22 created new session: [6:7574435683565496737:2086]
2025-11-19T13:58:55.355438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435683565496737:2086] [node 5] ICS09 handshake done sender: [6:7574435683565496734:2116] self: [6:7574435683565496735:0] peer: [5:7574435681348347986:0] socket: 181 qp: -1
2025-11-19T13:58:55.355443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435683565496737:2086] [node 5] ICS10 traffic start
2025-11-19T13:58:55.355449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:55.356908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435681348347984:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.356920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:55.356928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435681348347984:2110] [node 6] ICP22 created new session: [5:7574435681348348006:2110]
2025-11-19T13:58:55.356933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435681348348006:2110] [node 6] ICS09 handshake done sender: [5:7574435681348347996:2116] self: [5:7574435681348347986:0] peer: [6:7574435683565496735:0] socket: 180 qp: -1
2025-11-19T13:58:55.356937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435681348348006:2110] [node 6] ICS10 traffic start
2025-11-19T13:58:55.356943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:55.355924Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681578109259:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.355941Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435681578109259:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:55.356454Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681578109247:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.356457Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435681578109247:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.352358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435683502807768:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(58563093ns) {Configure:104(58561205ns) {ProcessConfigured:135(58301853ns) {StartInitialHandshake:157(58300691ns) {DropHandshakes:469(411ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:476(1176ns) {SwitchToState:178(260ns)}}}}}
2025-11-19T13:58:55.352421Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435683502807778:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.357376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435681578109268:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18161
2025-11-19T13:58:55.359987Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435681578109259:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:55.360838Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435684851696532:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:55.363966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435681348347918:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.364021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435681348347918:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(36082630ns) {IncomingHandshake:261(36081961ns) {DropIncomingHandshake:441(75ns) PrepareNewSessionHandshake:476(781ns) {SwitchToState:178(487ns)} GenerateSessionVirtualId:988(153ns) IssueIncomingHandshakeReply:193(36076067ns) {SwitchToState:178(1268ns)}}}
2025-11-19T13:58:55.364178Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681578109248:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.364197Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435681578109248:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.372378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435684851696530:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.372403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:55.372413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435684851696530:2109] [node 6] ICP22 created new session: [4:7574435684851696552:2109]
2025-11-19T13:58:55.372421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435684851696552:2109] [node 6] ICS09 handshake done sender: [4:7574435684851696532:2110] self: [4:7574435684851696531:0] peer: [6:7574435683565496733:0] socket: 145 qp: -1
2025-11-19T13:58:55.372430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435684851696552:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:55.372441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:55.372668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435684851696550:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23901
2025-11-19T13:58:55.372695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435684851696551:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23901
2025-11-19T13:58:55.373270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435684851696464:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.373299Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435684851696464:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(31175684ns) {IncomingHandshake:261(31175454ns) {DropIncomingHandshake:441(75ns) PrepareNewSessionHandshake:476(406ns) {SwitchToState:178(111ns)} GenerateSessionVirtualId:988(214ns) IssueIncomingHandshakeReply:193(31168977ns) {SwitchToState:178(745ns)}}}
2025-11-19T13:58:55.373452Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435684851696541:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.374939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435684851696466:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.379949Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435684851696550:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.379990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435684851696530:2109] [node 6] ICP18 handshake (actor [4:7574435684851696550:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.380246Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435685126469091:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:55.380454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435683565496740:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33638
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.380601Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435681578109247:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:55.380806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435681348348008:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19800
2025-11-19T13:58:55.384380Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435684851696540:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:55.384495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435684851696464:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.384513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:55.384519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435684851696464:2048] [node 1] ICP22 created new session: [4:7574435684851696556:2048]
2025-11-19T13:58:55.384526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435684851696556:2048] [node 1] ICS09 handshake done sender: [4:7574435684851696540:2114] self: [4:7574435684851696544:0] peer: [1:7574435681578109246:0] socket: 158 qp: -1
2025-11-19T13:58:55.384533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435684851696556:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:55.384541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.387955Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435683502807765:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.387973Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435683502807765:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:55.388672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435683502807787:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49712
2025-11-19T13:58:55.388684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435683502807788:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49712
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.387728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435681348347984:2110] [node 6] ICP35 event processing took too much time PendingConnection:278(30783302ns) {HandleHandshakeStatus:329(30782412ns) {DropIncomingHandshake:441(78ns) DropOutgoingHandshake:456(191ns) SwitchToState:178(67ns) ProcessPendingSessionEvents:495(30710282ns) {ForwardSessionEventToSession:612(30698394ns) ForwardSessionEventToSession:612(2662ns) ForwardSessionEventToSession:612(570ns) ForwardSessionEventToSession:612(1226ns) ForwardSessionEventToSession:612(673ns) ForwardSessionEventToSession:612(573ns) ForwardSessionEventToSession:612(602ns) ForwardSessionEventToSession:612(631ns) ForwardSessionEventToSession:612(616ns) ForwardSessionEventToSession:612(1050ns)}}}
2025-11-19T13:58:55.381602Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681578109268:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.381619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435681578109257:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.381637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:55.381643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435681578109257:2114] [node 5] ICP22 created new session: [1:7574435681578109271:2114]
2025-11-19T13:58:55.381649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435681578109271:2114] [node 5] ICS09 handshake done sender: [1:7574435681578109259:2117] self: [1:7574435681578109258:0] peer: [5:7574435681348347999:0] socket: 172 qp: -1
2025-11-19T13:58:55.381656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435681578109271:2114] [node 5] ICS10 traffic start
2025-11-19T13:58:55.381679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:55.389207Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435680831501720:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61766) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435685126469091:2114] held: no
2025-11-19T13:58:55.389623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435685126469106:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34583
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.389958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435683565496741:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33638
2025-11-19T13:58:55.389976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435683565496742:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33638
2025-11-19T13:58:55.390084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435681348348009:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19800
2025-11-19T13:58:55.390312Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435685126469074:2104] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:55.390487Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435681348347998:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:55.390625Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435685126469090:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:55.390742Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435681578109248:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:55.390880Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435683502807765:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:55.391058Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435683565496720:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:55.391226Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435681348347987:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.391353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435684851696558:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23901
2025-11-19T13:58:55.391670Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435684851696541:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:55.391767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435684851696466:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.391815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435683502807760:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.391855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:55.391870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435683502807760:2107] [node 4] ICP22 created new session: [3:7574435683502807790:2107]
2025-11-19T13:58:55.391879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435683502807790:2107] [node 4] ICS09 handshake done sender: [3:7574435683502807765:2110] self: [3:7574435683502807764:0] peer: [4:7574435684851696555:0] socket: 167 qp: -1
2025-11-19T13:58:55.391890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435683502807790:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:55.391936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:55.391786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:55.391794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435684851696466:2050] [node 3] ICP22 created new session: [4:7574435684851696559:2050]
2025-11-19T13:58:55.391811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435684851696559:2050] [node 3] ICS09 handshake done sender: [4:7574435684851696541:2115] self: [4:7574435684851696555:0] peer: [3:7574435683502807764:0] socket: 177 qp: -1
2025-11-19T13:58:55.391817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435684851696559:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:55.391825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:55.392825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435680831501720:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.392848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:55.392857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435680831501720:2048] [node 1] ICP22 created new session: [2:7574435685126469107:2048]
2025-11-19T13:58:55.392863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435685126469107:2048] [node 1] ICS09 handshake done sender: [2:7574435685126469091:2114] self: [2:7574435685126469100:0] peer: [1:7574435681578109241:0] socket: 149 qp: -1
2025-11-19T13:58:55.392870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435685126469107:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:55.392877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:55.393013Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435685126469106:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.393040Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435685126469097:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.393046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435685126469084:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.393057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:55.393061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435685126469084:2107] [node 5] ICP22 created new session: [2:7574435685126469109:2107]
2025-11-19T13:58:55.393064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435685126469109:2107] [node 5] ICS09 handshake done sender: [2:7574435685126469090:2113] self: [2:7574435685126469089:0] peer: [5:7574435681348347997:0] socket: 151 qp: -1
2025-11-19T13:58:55.393068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435685126469109:2107] [node 5] ICS10 traffic start
2025-11-19T13:58:55.393073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:55.394693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435680831501721:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.394711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:55.394718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435680831501721:2049] [node 3] ICP22 created new session: [2:7574435685126469111:2049]
2025-11-19T13:58:55.394724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435685126469111:2049] [node 3] ICS09 handshake done sender: [2:7574435685126469074:2104] self: [2:7574435685126469073:0] peer: [3:7574435683502807782:0] socket: 138 qp: -1
2025-11-19T13:58:55.394731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435685126469111:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:55.394737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:55.394804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435680831501721:2049] [node 3] ICP18 handshake (actor [2:7574435685126469097:2117]) is too old
2025-11-19T13:58:55.395075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435681348347917:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.395095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:55.395100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435681348347917:2048] [node 1] ICP22 created new session: [5:7574435681348348010:2048]
2025-11-19T13:58:55.395104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435681348348010:2048] [node 1] ICS09 handshake done sender: [5:7574435681348347998:2117] self: [5:7574435681348347999:0] peer: [1:7574435681578109258:0] socket: 173 qp: -1
2025-11-19T13:58:55.395109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435681348348010:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:55.395115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.395918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435681578109244:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.395942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:55.395948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435681578109244:2108] [node 4] ICP22 created new session: [1:7574435681578109276:2108]
2025-11-19T13:58:55.395954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435681578109276:2108] [node 4] ICS09 handshake done sender: [1:7574435681578109247:2111] self: [1:7574435681578109246:0] peer: [4:7574435684851696544:0] socket: 154 qp: -1
2025-11-19T13:58:55.395959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435681578109276:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:55.395967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:55.396217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435677283141883:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.396227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:55.396231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435677283141883:2049] [node 3] ICP22 created new session: [1:7574435681578109278:2049]
2025-11-19T13:58:55.396236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435681578109278:2049] [node 3] ICS09 handshake done sender: [1:7574435681578109248:2112] self: [1:7574435681578109242:0] peer: [3:7574435683502807783:0] socket: 157 qp: -1
2025-11-19T13:58:55.396240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435681578109278:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:55.396246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:55.396366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435683565496716:2088] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23901) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435683565496720:2109] held: no
2025-11-19T13:58:55.396566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435681348347918:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.396579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:55.396583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435681348347918:2049] [node 2] ICP22 created new session: [5:7574435681348348012:2049]
2025-11-19T13:58:55.396588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435681348348012:2049] [node 2] ICS09 handshake done sender: [5:7574435681348347987:2111] self: [5:7574435681348347997:0] peer: [2:7574435685126469089:0] socket: 155 qp: -1
2025-11-19T13:58:55.396593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435681348348012:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:55.396598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:55.396929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435683565496716:2088] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.396941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:55.396946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435683565496716:2088] [node 4] ICP22 created new session: [6:7574435683565496743:2088]
2025-11-19T13:58:55.396952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435683565496743:2088] [node 4] ICS09 handshake done sender: [6:7574435683565496720:2109] self: [6:7574435683565496733:0] peer: [4:7574435684851696531:0] socket: 148 qp: -1
2025-11-19T13:58:55.396956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435683565496743:2088] [node 4] ICS10 traffic start
2025-11-19T13:58:55.396961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.397546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435684851696561:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23901
2025-11-19T13:58:55.396058Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435683502807787:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.396161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435683502807789:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49712
2025-11-19T13:58:55.397108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435683502807760:2107] [node 4] ICP18 handshake (actor [3:7574435683502807787:2119]) is too old
2025-11-19T13:58:55.397128Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435683502807777:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:55.397981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435679207840403:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.397997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:55.398003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435679207840403:2048] [node 1] ICP22 created new session: [3:7574435683502807793:2048]
2025-11-19T13:58:55.398009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435683502807793:2048] [node 1] ICS09 handshake done sender: [3:7574435683502807777:2115] self: [3:7574435683502807783:0] peer: [1:7574435681578109242:0] socket: 165 qp: -1
2025-11-19T13:58:55.398013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435683502807793:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:55.398030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:55.398179Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435683502807767:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:55.400250Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435683565496742:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.400658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435683565496655:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.400976Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435685126469095:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.400986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435685126469095:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:55.401830Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435685126469095:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:55.401929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435685126469088:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.401948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:55.401954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435685126469088:2110] [node 6] ICP22 created new session: [2:7574435685126469116:2110]
2025-11-19T13:58:55.401959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435685126469116:2110] [node 6] ICS09 handshake done sender: [2:7574435685126469095:2115] self: [2:7574435685126469094:0] peer: [6:7574435683565496745:0] socket: 199 qp: -1
2025-11-19T13:58:55.401966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435685126469116:2110] [node 6] ICS10 traffic start
2025-11-19T13:58:55.401976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:55.406634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435679207840404:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.406683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:55.406693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435679207840404:2049] [node 2] ICP22 created new session: [3:7574435683502807795:2049]
2025-11-19T13:58:55.406701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435683502807795:2049] [node 2] ICS09 handshake done sender: [3:7574435683502807767:2111] self: [3:7574435683502807782:0] peer: [2:7574435685126469073:0] socket: 141 qp: -1
2025-11-19T13:58:55.406708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435683502807795:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:55.406716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:55.408485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435685126469098:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.408494Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435685126469098:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.409432Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435685126469098:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:55.410733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435685126469083:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.410753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:55.410761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435685126469083:2111] [node 4] ICP22 created new session: [2:7574435685126469119:2111]
2025-11-19T13:58:55.410766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435685126469119:2111] [node 4] ICS09 handshake done sender: [2:7574435685126469098:2118] self: [2:7574435685126469087:0] peer: [4:7574435684851696562:0] socket: 186 qp: -1
2025-11-19T13:58:55.410771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435685126469119:2111] [node 4] ICS10 traffic start
2025-11-19T13:58:55.410778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:55.408293Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435684851696561:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.408345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435684851696465:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.409294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435684851696563:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23901
2025-11-19T13:58:55.409602Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435684851696561:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:55.409692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435684851696465:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.409709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:55.409714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435684851696465:2049] [node 2] ICP22 created new session: [4:7574435684851696564:2049]
2025-11-19T13:58:55.409720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435684851696564:2049] [node 2] ICS09 handshake done sender: [4:7574435684851696561:2120] self: [4:7574435684851696562:0] peer: [2:7574435685126469087:0] socket: 146 qp: -1
2025-11-19T13:58:55.409725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435684851696564:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:55.409734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:55.411854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435683565496655:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6023) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435683565496742:2120] held: no
2025-11-19T13:58:55.412084Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435683565496741:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.412116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435683565496654:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.412048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435679207840404:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:55.412236Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681578109260:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.412249Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435681578109260:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.413376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435681348348014:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19800
2025-11-19T13:58:55.413550Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435681578109260:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:55.413780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435681578109239:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.413804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:55.413809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435681578109239:2107] [node 6] ICP22 created new session: [1:7574435681578109281:2107]
2025-11-19T13:58:55.413814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435681578109281:2107] [node 6] ICS09 handshake done sender: [1:7574435681578109260:2118] self: [1:7574435681578109253:0] peer: [6:7574435683565496746:0] socket: 150 qp: -1
2025-11-19T13:58:55.413819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435681578109281:2107] [node 6] ICS10 traffic start
2025-11-19T13:58:55.413826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.414929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435683565496747:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33638
2025-11-19T13:58:55.414977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435683565496748:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33638
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.415057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435683565496749:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33638
2025-11-19T13:58:55.415318Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435683565496747:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.415340Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435683565496741:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:55.415464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435683565496656:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.415479Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435683565496742:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:55.415564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435683565496654:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.415576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:55.415581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435683565496654:2048] [node 1] ICP22 created new session: [6:7574435683565496751:2048]
2025-11-19T13:58:55.415585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435683565496751:2048] [node 1] ICS09 handshake done sender: [6:7574435683565496741:2119] self: [6:7574435683565496746:0] peer: [1:7574435681578109253:0] socket: 200 qp: -1
2025-11-19T13:58:55.415590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435683565496751:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:55.415596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:55.415987Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435683502807779:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.416001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435683502807779:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:55.416989Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435681348348014:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.417034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435681348347919:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.417328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435683565496655:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.417345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:55.417353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435683565496655:2049] [node 2] ICP22 created new session: [6:7574435683565496754:2049]
2025-11-19T13:58:55.417359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435683565496754:2049] [node 2] ICS09 handshake done sender: [6:7574435683565496742:2120] self: [6:7574435683565496745:0] peer: [2:7574435685126469094:0] socket: 201 qp: -1
2025-11-19T13:58:55.417364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435683565496754:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:55.417370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:55.417533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435683565496753:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33638
2025-11-19T13:58:55.418290Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435683502807779:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:55.418480Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435683502807784:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.418489Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435683502807784:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:55.419819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435683502807766:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.419836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:55.419844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435683502807766:2108] [node 6] ICP22 created new session: [3:7574435683502807799:2108]
2025-11-19T13:58:55.419850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435683502807799:2108] [node 6] ICS09 handshake done sender: [3:7574435683502807779:2117] self: [3:7574435683502807775:0] peer: [6:7574435683565496750:0] socket: 205 qp: -1
2025-11-19T13:58:55.419857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435683502807799:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:55.419865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:55.420422Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435683565496747:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:55.420626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435683565496656:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.420641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:55.420647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435683565496656:2050] [node 3] ICP22 created new session: [6:7574435683565496756:2050]
2025-11-19T13:58:55.420653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435683565496756:2050] [node 3] ICS09 handshake done sender: [6:7574435683565496747:2121] self: [6:7574435683565496750:0] peer: [3:7574435683502807775:0] socket: 169 qp: -1
2025-11-19T13:58:55.420659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435683565496756:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:55.420668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.422393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435681348348016:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19800
2025-11-19T13:58:55.422821Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435683502807784:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:55.423004Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435681348348014:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:55.423109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435681348347919:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:55.423126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:55.423132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435681348347919:2050] [node 3] ICP22 created new session: [5:7574435681348348017:2050]
2025-11-19T13:58:55.423138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435681348348017:2050] [node 3] ICS09 handshake done sender: [5:7574435681348348014:2121] self: [5:7574435681348348015:0] peer: [3:7574435683502807773:0] socket: 170 qp: -1
2025-11-19T13:58:55.423143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435681348348017:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:55.423150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:55.424243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435683502807768:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.424263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:55.424269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435683502807768:2109] [node 5] ICP22 created new session: [3:7574435683502807802:2109]
2025-11-19T13:58:55.424275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435683502807802:2109] [node 5] ICS09 handshake done sender: [3:7574435683502807784:2118] self: [3:7574435683502807773:0] peer: [5:7574435681348348015:0] socket: 140 qp: -1
2025-11-19T13:58:55.424281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435683502807802:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:55.424288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:55.427508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:55.427584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435685126469107:2048] [node 1] ICS01 socket: 149 reason# EndOfStream
2025-11-19T13:58:55.427605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435680831501720:2048] [node 1] ICP30 unregister session Session# [2:7574435685126469107:2048] VirtualId# [2:7574435685126469100:0]
2025-11-19T13:58:55.427611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435685126469107:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:55.427641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:55.427662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:55.428063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:55.428125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435683502807793:2048] [node 1] ICS01 socket: 165 reason# EndOfStream
2025-11-19T13:58:55.428130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435679207840403:2048] [node 1] ICP30 unregister session Session# [3:7574435683502807793:2048] VirtualId# [3:7574435683502807783:0]
2025-11-19T13:58:55.428134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435683502807793:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:55.428159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:55.428175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:55.442958Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:55.443031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435683565496751:2048] [node 1] ICS01 socket: 200 reason# EndOfStream
2025-11-19T13:58:55.443038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435683565496654:2048] [node 1] ICP30 unregister session Session# [6:7574435683565496751:2048] VirtualId# [6:7574435683565496746:0]
2025-11-19T13:58:55.443043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435683565496751:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:55.443105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:55.443133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:55.443172Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:55.443248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435681348348010:2048] [node 1] ICS01 socket: 173 reason# EndOfStream
2025-11-19T13:58:55.443255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435681348347917:2048] [node 1] ICP30 unregister session Session# [5:7574435681348348010:2048] VirtualId# [5:7574435681348347999:0]
2025-11-19T13:58:55.443261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435681348348010:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:55.443321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:55.443348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:55.442528Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:55.442665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435684851696556:2048] [node 1] ICS01 socket: 158 reason# EndOfStream
2025-11-19T13:58:55.442679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435684851696464:2048] [node 1] ICP30 unregister session Session# [4:7574435684851696556:2048] VirtualId# [4:7574435684851696544:0]
2025-11-19T13:58:55.442689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435684851696556:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:55.442775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:55.442813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:55.448371Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:55.448474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435683502807795:2049] [node 2] ICS01 socket: 141 reason# EndOfStream
2025-11-19T13:58:55.448483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435679207840404:2049] [node 2] ICP30 unregister session Session# [3:7574435683502807795:2049] VirtualId# [3:7574435683502807782:0]
2025-11-19T13:58:55.448490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435683502807795:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:55.448547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:55.448573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:55.489161Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:55.489275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435681348348012:2049] [node 2] ICS07 socket disconnect 155 reason# EndOfStream
2025-11-19T13:58:55.489283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435681348348012:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:55.489343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:55.489367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435681348348012:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:55.489597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:55.489672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435683565496754:2049] [node 2] ICS01 socket: 201 reason# EndOfStream
2025-11-19T13:58:55.489680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435683565496655:2049] [node 2] ICP30 unregister session Session# [6:7574435683565496754:2049] VirtualId# [6:7574435683565496745:0]
2025-11-19T13:58:55.489687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435683565496754:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:55.489714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:55.489737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:55.490858Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435681348347918:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6023) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:55.490873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435681348348012:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:55.490877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435681348347918:2049] [node 2] ICP30 unregister session Session# [5:7574435681348348012:2049] VirtualId# [5:7574435681348347997:0]
2025-11-19T13:58:55.490891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:55.490896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435681348347918:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6023) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:55.490899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6023) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:55.489778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:55.489846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435684851696564:2049] [node 2] ICS01 socket: 146 reason# EndOfStream
2025-11-19T13:58:55.489851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435684851696465:2049] [node 2] ICP30 unregister session Session# [4:7574435684851696564:2049] VirtualId# [4:7574435684851696562:0]
2025-11-19T13:58:55.489855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435684851696564:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:55.489881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:55.489900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:55.507477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435681348347918:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:58:55.507522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:55.507603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435681348348017:2050] [node 3] ICS01 socket: 170 reason# EndOfStream
2025-11-19T13:58:55.507611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435681348347919:2050] [node 3] ICP30 unregister session Session# [5:7574435681348348017:2050] VirtualId# [5:7574435681348348015:0]
2025-11-19T13:58:55.507618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435681348348017:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:55.507694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:55.507719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:55.508272Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:55.508362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435683565496756:2050] [node 3] ICS01 socket: 169 reason# EndOfStream
2025-11-19T13:58:55.508370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435683565496656:2050] [node 3] ICP30 unregister session Session# [6:7574435683565496756:2050] VirtualId# [6:7574435683565496750:0]
2025-11-19T13:58:55.508376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435683565496756:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:55.508442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:55.508468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:55.520329Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:55.520418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435683565496743:2088] [node 4] ICS01 socket: 148 reason# EndOfStream
2025-11-19T13:58:55.520426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435683565496716:2088] [node 4] ICP30 unregister session Session# [6:7574435683565496743:2088] VirtualId# [6:7574435683565496733:0]
2025-11-19T13:58:55.520434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435683565496743:2088] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:55.520490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:55.520514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:55.667869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435681622775243:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:55.667887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435681622775244:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:55.719260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435681394412620:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:55.719276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435681394412621:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:55.784041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435681767574729:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:55.784063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435681767574730:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:55.816553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435682872449043:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:55.816567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435682872449044:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:55.816572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435682872449045:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:55.844007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435682960750872:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:55.844022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435682960750873:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:55.844028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435682960750874:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:55.876445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435685164509032:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:55.876458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435685164509033:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:55.876463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435685164509034:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.948311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435681394412680:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3348
2025-11-19T13:58:55.948641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435685164509082:2096] [node 4] ICP01 ready to work
2025-11-19T13:58:55.948941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435685164509088:2098] [node 5] ICP01 ready to work
2025-11-19T13:58:55.949191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435682872449105:2108] [node 6] ICP01 ready to work
2025-11-19T13:58:55.950314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435682960750927:2098] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.951831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435681767574789:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3927
2025-11-19T13:58:55.953516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435682960750943:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.960874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435681767574790:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3927
2025-11-19T13:58:55.959413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435681622775308:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:55.959511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435681622775309:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:55.960517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435681394412682:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.964054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685164509094:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25713
2025-11-19T13:58:55.964082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685164509095:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25713
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.965576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435682872449109:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53335
2025-11-19T13:58:55.965600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435682872449110:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53335
2025-11-19T13:58:55.964849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435681394412691:2112] [node 6] ICP01 ready to work
2025-11-19T13:58:55.966853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435681394412695:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.967809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435682960750950:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8782
2025-11-19T13:58:55.966381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435681767574795:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:55.969204Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435682872449110:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.970379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435681622775320:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10596
2025-11-19T13:58:55.970426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435681622775321:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10596
2025-11-19T13:58:55.971074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435681622775319:2113] [node 5] ICP01 ready to work
2025-11-19T13:58:55.971352Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681622775321:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.972681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435681394412683:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3348
2025-11-19T13:58:55.975226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435682872449044:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.973008Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681622775320:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.976008Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435682872449109:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.977431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435682872449115:2112] [node 5] ICP01 ready to work
2025-11-19T13:58:55.977966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435681767574801:2112] [node 4] ICP01 ready to work
2025-11-19T13:58:55.979844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435681394412701:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3348
2025-11-19T13:58:55.979956Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435681394412680:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.979985Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435681394412683:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.980151Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435681394412685:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.980164Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435681394412685:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:55.981200Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681622775304:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.981211Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435681622775304:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:55.983265Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435685164509095:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.983392Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435685164509094:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.983448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435685164509088:2098] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.983606Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435682960750950:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.983644Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435682960750929:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.983651Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435682960750929:2102] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:55.985086Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681622775311:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.985101Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435681622775311:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:55.981187Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435681767574790:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.981230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435681767574729:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.984126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435681767574730:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(18576547ns) {Configure:104(18574503ns) {ProcessConfigured:135(18254629ns) {StartInitialHandshake:157(18254037ns) {DropHandshakes:469(613ns) {DropIncomingHandshake:441(99ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:476(852ns) {SwitchToState:178(502ns)}}}}}
2025-11-19T13:58:55.985207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435681767574804:2113] [node 5] ICP01 ready to work
2025-11-19T13:58:55.985996Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435681622775304:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:55.986150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435681622775243:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.986170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:55.986177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435681622775243:2048] [node 2] ICP22 created new session: [1:7574435681622775327:2048]
2025-11-19T13:58:55.986196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435681622775327:2048] [node 2] ICS09 handshake done sender: [1:7574435681622775304:2107] self: [1:7574435681622775303:0] peer: [2:7574435681394412702:0] socket: 144 qp: -1
2025-11-19T13:58:55.986203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435681622775327:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:55.986221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:55.987669Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435681394412685:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.985908Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435681767574789:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.987981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435681394412620:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.988040Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435681394412701:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.988326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685164509108:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25713
2025-11-19T13:58:55.988394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685164509109:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25713
2025-11-19T13:58:55.988501Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435685164509032:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(40027990ns) {Configure:104(40026750ns) {ProcessConfigured:135(26129141ns) {StartInitialHandshake:157(26128815ns) {DropHandshakes:469(548ns) {DropIncomingHandshake:441(139ns) DropOutgoingHandshake:456(145ns)} GenerateSessionVirtualId:988(596ns) PrepareNewSessionHandshake:476(750ns) {SwitchToState:178(27ns)}}}}}
2025-11-19T13:58:55.990539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435681767574809:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3927
2025-11-19T13:58:55.990722Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435681767574809:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.993366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435681394412682:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:55.993386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:55.993394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435681394412682:2108] [node 4] ICP22 created new session: [2:7574435681394412706:2108]
2025-11-19T13:58:55.993400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435681394412706:2108] [node 4] ICS09 handshake done sender: [2:7574435681394412685:2110] self: [2:7574435681394412684:0] peer: [4:7574435682872449111:0] socket: 157 qp: -1
2025-11-19T13:58:55.993410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435681394412706:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:55.993432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:55.991967Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435685164509108:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.991998Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435685164509109:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.993312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435685164509033:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:55.993697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435685164509034:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:55.996022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435682872449121:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53335
2025-11-19T13:58:55.996159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435682960750952:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8782
2025-11-19T13:58:55.996146Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435681394412694:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:55.996155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435681394412694:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.000019Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435682960750929:2102] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:56.001652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435681767574804:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16393450ns) {Configure:104(16392073ns) {ProcessConfigured:135(16035638ns) {StartInitialHandshake:157(16035186ns) {DropHandshakes:469(544ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(188ns)} GenerateSessionVirtualId:988(273ns) PrepareNewSessionHandshake:476(1063ns) {SwitchToState:178(606ns)}}}}}
2025-11-19T13:58:56.002648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435685164509082:2096] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(53954155ns) {Configure:104(53953324ns) {ProcessConfigured:135(40067183ns) {StartInitialHandshake:157(40066778ns) {DropHandshakes:469(465ns) {DropIncomingHandshake:441(147ns) DropOutgoingHandshake:456(35ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:476(1092ns) {SwitchToState:178(393ns)}}}}}
2025-11-19T13:58:56.002689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435685164509082:2096] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.002488Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435681622775243:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(16314436ns) {HandleHandshakeStatus:329(16313607ns) {DropIncomingHandshake:441(4361ns) DropOutgoingHandshake:456(222ns) SwitchToState:178(257ns) ProcessPendingSessionEvents:495(16193258ns) {ForwardSessionEventToSession:612(16181602ns) ForwardSessionEventToSession:612(1700ns) ForwardSessionEventToSession:612(1021ns) ForwardSessionEventToSession:612(544ns) ForwardSessionEventToSession:612(511ns) ForwardSessionEventToSession:612(1245ns) ForwardSessionEventToSession:612(646ns) ForwardSessionEventToSession:612(581ns) ForwardSessionEventToSession:612(659ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.002932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435681394412709:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3348
2025-11-19T13:58:56.003114Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435682960750952:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.003152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435682960750943:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.003204Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435681622775311:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.005617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435687255718252:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8782
2025-11-19T13:58:56.001198Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435682872449110:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:56.001402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435682872449044:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.001414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:56.001421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435682872449044:2049] [node 2] ICP22 created new session: [4:7574435682872449122:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.006731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435681394412710:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3348
2025-11-19T13:58:56.006793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435681394412620:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25641) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435681394412680:2107] held: no
2025-11-19T13:58:56.006816Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435681394412680:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:56.001428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435682872449122:2049] [node 2] ICS09 handshake done sender: [4:7574435682872449110:2111] self: [4:7574435682872449111:0] peer: [2:7574435681394412684:0] socket: 159 qp: -1
2025-11-19T13:58:56.001433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435682872449122:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:56.001465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:56.006944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435681394412620:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.006959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:56.006965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435681394412620:2048] [node 1] ICP22 created new session: [2:7574435685689380007:2048]
2025-11-19T13:58:56.006971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435685689380007:2048] [node 1] ICS09 handshake done sender: [2:7574435681394412680:2107] self: [2:7574435681394412702:0] peer: [1:7574435681622775303:0] socket: 145 qp: -1
2025-11-19T13:58:56.006976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435685689380007:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:56.006984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:56.007669Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435681394412710:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.007591Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435681767574798:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.007612Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435681767574798:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.004760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435681622775244:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.004785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:56.004793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435681622775244:2049] [node 3] ICP22 created new session: [1:7574435685917742627:2049]
2025-11-19T13:58:56.004800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435685917742627:2049] [node 3] ICS09 handshake done sender: [1:7574435681622775311:2110] self: [1:7574435681622775310:0] peer: [3:7574435681767574805:0] socket: 154 qp: -1
2025-11-19T13:58:56.004807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435685917742627:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:56.004816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:56.007976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435681767574729:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25641) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435681767574790:2108] held: no
2025-11-19T13:58:56.007972Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435687255718252:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.007999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435682960750872:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.009722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435682960750927:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.009760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:56.009769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435682960750927:2098] [node 6] ICP22 created new session: [5:7574435687255718254:2098]
2025-11-19T13:58:56.009776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435687255718254:2098] [node 6] ICS09 handshake done sender: [5:7574435682960750929:2102] self: [5:7574435682960750928:0] peer: [6:7574435685164509107:0] socket: 147 qp: -1
2025-11-19T13:58:56.009783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435687255718254:2098] [node 6] ICS10 traffic start
2025-11-19T13:58:56.009805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.011474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435686062542110:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3927
2025-11-19T13:58:56.011491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435686062542111:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3927
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.011562Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435682872449043:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(36827980ns) {Configure:104(36826662ns) {ProcessConfigured:135(36486386ns) {StartInitialHandshake:157(36485749ns) {DropHandshakes:469(797ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(136ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:476(645ns) {SwitchToState:178(270ns)}}}}}
2025-11-19T13:58:56.011623Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435682872449107:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.011626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435682872449107:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.012137Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435682872449119:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.012142Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435682872449119:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.012782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435682872449043:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.016247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435689459476412:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25713
2025-11-19T13:58:56.016401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435689459476413:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25713
2025-11-19T13:58:56.016457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435689459476414:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25713
2025-11-19T13:58:56.017427Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435685164509088:2098] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20002) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435685164509095:2108] held: no
2025-11-19T13:58:56.017683Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435685164509095:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:56.018107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435685164509088:2098] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.018122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:56.018128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435685164509088:2098] [node 5] ICP22 created new session: [6:7574435689459476415:2098]
2025-11-19T13:58:56.018134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435689459476415:2098] [node 5] ICS09 handshake done sender: [6:7574435685164509095:2108] self: [6:7574435685164509107:0] peer: [5:7574435682960750928:0] socket: 151 qp: -1
2025-11-19T13:58:56.018143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435689459476415:2098] [node 5] ICS10 traffic start
2025-11-19T13:58:56.018167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:56.019694Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435681767574790:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:56.020173Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681622775326:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.020191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435681622775326:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.018795Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435682872449107:2109] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:56.020349Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435682872449119:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:56.020918Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435685164509094:2107] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:56.020998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435685164509082:2096] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.021013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:56.021019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435685164509082:2096] [node 4] ICP22 created new session: [6:7574435689459476418:2096]
2025-11-19T13:58:56.021025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435689459476418:2096] [node 4] ICS09 handshake done sender: [6:7574435685164509094:2107] self: [6:7574435685164509115:0] peer: [4:7574435682872449106:0] socket: 150 qp: -1
2025-11-19T13:58:56.021031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435689459476418:2096] [node 4] ICS10 traffic start
2025-11-19T13:58:56.021052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.022816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435685917742630:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10596
2025-11-19T13:58:56.023912Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435681394412694:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.023420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435681767574729:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.023438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:56.023444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435681767574729:2048] [node 1] ICP22 created new session: [3:7574435686062542112:2048]
2025-11-19T13:58:56.023450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435686062542112:2048] [node 1] ICS09 handshake done sender: [3:7574435681767574790:2108] self: [3:7574435681767574805:0] peer: [1:7574435681622775310:0] socket: 156 qp: -1
2025-11-19T13:58:56.023457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435686062542112:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:56.023464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:56.024407Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435681767574798:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:56.024285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435681394412691:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.024305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:56.024313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435681394412691:2112] [node 6] ICP22 created new session: [2:7574435685689380012:2112]
2025-11-19T13:58:56.024329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435685689380012:2112] [node 6] ICS09 handshake done sender: [2:7574435681394412694:2114] self: [2:7574435681394412693:0] peer: [6:7574435685164509112:0] socket: 164 qp: -1
2025-11-19T13:58:56.024335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435685689380012:2112] [node 6] ICS10 traffic start
2025-11-19T13:58:56.023961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435682872449105:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.024342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:56.023978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:56.023985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435682872449105:2108] [node 6] ICP22 created new session: [4:7574435687167416425:2108]
2025-11-19T13:58:56.023990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435687167416425:2108] [node 6] ICS09 handshake done sender: [4:7574435682872449107:2109] self: [4:7574435682872449106:0] peer: [6:7574435685164509115:0] socket: 146 qp: -1
2025-11-19T13:58:56.023996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435687167416425:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:56.024002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:56.024100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435682872449115:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.024107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:56.024112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435682872449115:2112] [node 5] ICP22 created new session: [4:7574435687167416427:2112]
2025-11-19T13:58:56.024116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435687167416427:2112] [node 5] ICS09 handshake done sender: [4:7574435682872449119:2114] self: [4:7574435682872449116:0] peer: [5:7574435687255718251:0] socket: 180 qp: -1
2025-11-19T13:58:56.024122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435687167416427:2112] [node 5] ICS10 traffic start
2025-11-19T13:58:56.024126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:56.024282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435687167416424:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53335
2025-11-19T13:58:56.024479Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435682872449044:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(23057037ns) {HandleHandshakeStatus:329(23056672ns) {DropIncomingHandshake:441(400ns) DropOutgoingHandshake:456(146ns) SwitchToState:178(185ns) ProcessPendingSessionEvents:495(16393799ns) {ForwardSessionEventToSession:612(16386463ns) ForwardSessionEventToSession:612(1155ns) ForwardSessionEventToSession:612(631ns) ForwardSessionEventToSession:612(448ns) ForwardSessionEventToSession:612(569ns) ForwardSessionEventToSession:612(1004ns) ForwardSessionEventToSession:612(583ns)}}}
2025-11-19T13:58:56.024503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435681767574795:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.024520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:56.024528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435681767574795:2109] [node 6] ICP22 created new session: [3:7574435686062542115:2109]
2025-11-19T13:58:56.024534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435686062542115:2109] [node 6] ICS09 handshake done sender: [3:7574435681767574798:2111] self: [3:7574435681767574797:0] peer: [6:7574435685164509113:0] socket: 167 qp: -1
2025-11-19T13:58:56.024540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435686062542115:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:56.024546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.025386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435687255718256:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8782
2025-11-19T13:58:56.026097Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681622775313:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.026110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435681622775313:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.027097Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435682960750952:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:56.027181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435682960750943:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.027194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:56.027200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435682960750943:2108] [node 4] ICP22 created new session: [5:7574435687255718258:2108]
2025-11-19T13:58:56.027205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435687255718258:2108] [node 4] ICS09 handshake done sender: [5:7574435682960750952:2116] self: [5:7574435687255718251:0] peer: [4:7574435682872449116:0] socket: 181 qp: -1
2025-11-19T13:58:56.027212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435687255718258:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:56.027218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.030023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435685917742631:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10596
2025-11-19T13:58:56.030108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435685917742632:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10596
2025-11-19T13:58:56.030762Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435689459476413:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.033052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435685917742630:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.033319Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435685917742632:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.034345Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435686062542110:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.034380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435681767574730:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.033793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435685164509033:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5133) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435685164509108:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.035300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435689459476420:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25713
2025-11-19T13:58:56.035615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435687255718257:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8782
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.041269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435689459476421:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25713
2025-11-19T13:58:56.041356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435685164509082:2096] [node 4] ICP35 event processing took too much time PendingConnection:278(20337212ns) {HandleHandshakeStatus:329(20336616ns) {DropIncomingHandshake:441(365ns) DropOutgoingHandshake:456(1593ns) SwitchToState:178(170ns) ProcessPendingSessionEvents:495(15250029ns) {ForwardSessionEventToSession:612(15247600ns)}}}
2025-11-19T13:58:56.041380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435685164509032:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.041565Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435685164509108:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:56.042094Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435681622775326:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.048001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435687167416430:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53335
2025-11-19T13:58:56.048044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435687167416431:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53335
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.049763Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435687167416430:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.049873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435682872449045:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.050066Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435681622775313:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:56.050385Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435682872449109:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:56.048439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435687255718260:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8782
2025-11-19T13:58:56.048560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435682960750943:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(21356218ns) {HandleHandshakeStatus:329(21355249ns) {DropIncomingHandshake:441(210ns) DropOutgoingHandshake:456(1599ns) SwitchToState:178(209ns) ProcessPendingSessionEvents:495(21271941ns) {ForwardSessionEventToSession:612(21266151ns) ForwardSessionEventToSession:612(1672ns) ForwardSessionEventToSession:612(577ns) ForwardSessionEventToSession:612(511ns) ForwardSessionEventToSession:612(482ns)}}}
2025-11-19T13:58:56.048576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435687255718261:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8782
2025-11-19T13:58:56.050441Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435687255718252:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:56.051918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435681394412697:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.051938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435681394412697:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.052129Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435685917742631:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.052149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435681622775308:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.052166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:56.052175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435681622775308:2108] [node 4] ICP22 created new session: [1:7574435685917742637:2108]
2025-11-19T13:58:56.052181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435685917742637:2108] [node 4] ICS09 handshake done sender: [1:7574435681622775313:2111] self: [1:7574435681622775312:0] peer: [4:7574435687167416421:0] socket: 155 qp: -1
2025-11-19T13:58:56.052188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435685917742637:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:56.052209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:56.053417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435682960750872:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.053441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:56.053451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435682960750872:2048] [node 1] ICP22 created new session: [5:7574435687255718262:2048]
2025-11-19T13:58:56.053457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435687255718262:2048] [node 1] ICS09 handshake done sender: [5:7574435687255718252:2117] self: [5:7574435687255718253:0] peer: [1:7574435681622775323:0] socket: 191 qp: -1
2025-11-19T13:58:56.053464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435687255718262:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:56.053496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:56.053605Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435681767574810:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.053621Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435681767574810:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.055989Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435681622775315:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.056012Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435681622775315:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.056833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435685164509033:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.056849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:56.056856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435685164509033:2049] [node 2] ICP22 created new session: [6:7574435689459476423:2049]
2025-11-19T13:58:56.056861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435689459476423:2049] [node 2] ICS09 handshake done sender: [6:7574435685164509108:2114] self: [6:7574435685164509112:0] peer: [2:7574435681394412693:0] socket: 170 qp: -1
2025-11-19T13:58:56.056866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435689459476423:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:56.056871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:56.056952Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435685164509109:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:56.056467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435681622775319:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.056482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:56.056491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435681622775319:2113] [node 5] ICP22 created new session: [1:7574435685917742639:2113]
2025-11-19T13:58:56.056498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435685917742639:2113] [node 5] ICS09 handshake done sender: [1:7574435681622775326:2116] self: [1:7574435681622775323:0] peer: [5:7574435687255718253:0] socket: 168 qp: -1
2025-11-19T13:58:56.056505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435685917742639:2113] [node 5] ICS10 traffic start
2025-11-19T13:58:56.056515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:56.058286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435681622775319:2113] [node 5] ICP18 handshake (actor [1:7574435685917742631:2118]) is too old
2025-11-19T13:58:56.058679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435682872449105:2108] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:56.058975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435685164509034:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.058992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:56.058998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435685164509034:2050] [node 3] ICP22 created new session: [6:7574435689459476425:2050]
2025-11-19T13:58:56.059004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435689459476425:2050] [node 3] ICS09 handshake done sender: [6:7574435685164509109:2115] self: [6:7574435685164509113:0] peer: [3:7574435681767574797:0] socket: 176 qp: -1
2025-11-19T13:58:56.059010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435689459476425:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:56.059031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:56.057192Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435687255718257:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.061115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435682872449043:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.061137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:56.061148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435682872449043:2048] [node 1] ICP22 created new session: [4:7574435687167416433:2048]
2025-11-19T13:58:56.061154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435687167416433:2048] [node 1] ICS09 handshake done sender: [4:7574435682872449109:2110] self: [4:7574435687167416421:0] peer: [1:7574435681622775312:0] socket: 158 qp: -1
2025-11-19T13:58:56.061161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435687167416433:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:56.061172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:56.060207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435682960750873:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.061554Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435681394412697:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:56.061729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435681394412621:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.061742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:56.061748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435681394412621:2049] [node 3] ICP22 created new session: [2:7574435685689380015:2049]
2025-11-19T13:58:56.061753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435685689380015:2049] [node 3] ICS09 handshake done sender: [2:7574435681394412697:2115] self: [2:7574435681394412696:0] peer: [3:7574435686062542118:0] socket: 182 qp: -1
2025-11-19T13:58:56.061758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435685689380015:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:56.061763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:56.061852Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435687255718261:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.061877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435682960750874:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.065939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435687167416435:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53335
2025-11-19T13:58:56.066883Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435687167416430:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:56.066680Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435681767574810:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:56.066973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435682872449045:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.066991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:56.066997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435682872449045:2050] [node 3] ICP22 created new session: [4:7574435687167416436:2050]
2025-11-19T13:58:56.067003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435687167416436:2050] [node 3] ICS09 handshake done sender: [4:7574435687167416430:2118] self: [4:7574435687167416432:0] peer: [3:7574435681767574802:0] socket: 202 qp: -1
2025-11-19T13:58:56.067011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435687167416436:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:56.067020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:56.069803Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435681394412699:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.069820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435681394412699:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.070884Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435681622775315:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.071050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435689459476427:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25713
2025-11-19T13:58:56.071431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435681622775309:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.071456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:56.071462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435681622775309:2109] [node 6] ICP22 created new session: [1:7574435685917742642:2109]
2025-11-19T13:58:56.071468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435685917742642:2109] [node 6] ICS09 handshake done sender: [1:7574435681622775315:2112] self: [1:7574435681622775314:0] peer: [6:7574435689459476422:0] socket: 189 qp: -1
2025-11-19T13:58:56.071473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435685917742642:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:56.071481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:56.069653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435681767574801:2112] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.069677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:56.069684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435681767574801:2112] [node 4] ICP22 created new session: [3:7574435686062542121:2112]
2025-11-19T13:58:56.069692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435686062542121:2112] [node 4] ICS09 handshake done sender: [3:7574435681767574810:2116] self: [3:7574435681767574802:0] peer: [4:7574435687167416432:0] socket: 166 qp: -1
2025-11-19T13:58:56.069699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435686062542121:2112] [node 4] ICS10 traffic start
2025-11-19T13:58:56.069708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:56.069872Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435681767574812:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.069879Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435681767574812:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.073056Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435689459476413:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:56.073225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435685164509032:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.073243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:56.073248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435685164509032:2048] [node 1] ICP22 created new session: [6:7574435689459476428:2048]
No RDMA context found for GID: 2025-11-19T13:58:56.073254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435689459476428:2048] [node 1] ICS09 handshake done sender: [6:7574435689459476413:2119] self: [6:7574435689459476422:0] peer: [1:7574435681622775314:0] socket: 165 qp: -1
::ffff:127.1.0.0
2025-11-19T13:58:56.073259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435689459476428:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:56.073270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:56.074424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435682872449043:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22322625ns) {HandleHandshakeStatus:329(22321905ns) {DropIncomingHandshake:441(881ns) DropOutgoingHandshake:456(1478ns) SwitchToState:178(62ns) ProcessPendingSessionEvents:495(4603347ns) {ForwardSessionEventToSession:612(4592430ns) ForwardSessionEventToSession:612(1383ns) ForwardSessionEventToSession:612(1526ns) ForwardSessionEventToSession:612(604ns) ForwardSessionEventToSession:612(667ns) ForwardSessionEventToSession:612(701ns) ForwardSessionEventToSession:612(861ns) ForwardSessionEventToSession:612(629ns) ForwardSessionEventToSession:612(744ns) ForwardSessionEventToSession:612(697ns)}}}
2025-11-19T13:58:56.074488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435682872449043:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:56.074197Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435681767574730:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5133) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435686062542110:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.077192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435686062542123:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3927
2025-11-19T13:58:56.077390Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435686062542110:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:56.078394Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435681767574812:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:56.078420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435681767574730:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.078433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:56.078447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435681767574730:2049] [node 2] ICP22 created new session: [3:7574435686062542125:2049]
2025-11-19T13:58:56.078454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435686062542125:2049] [node 2] ICS09 handshake done sender: [3:7574435686062542110:2118] self: [3:7574435686062542118:0] peer: [2:7574435681394412696:0] socket: 183 qp: -1
2025-11-19T13:58:56.078460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435686062542125:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:56.078467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:56.078095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435687255718266:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8782
2025-11-19T13:58:56.078116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435687255718267:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8782
2025-11-19T13:58:56.078793Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435687255718261:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:56.078892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435682960750874:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.078916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:56.078920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435682960750874:2050] [node 3] ICP22 created new session: [5:7574435687255718268:2050]
2025-11-19T13:58:56.078925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435687255718268:2050] [node 3] ICS09 handshake done sender: [5:7574435687255718261:2121] self: [5:7574435687255718265:0] peer: [3:7574435681767574808:0] socket: 209 qp: -1
2025-11-19T13:58:56.078930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435687255718268:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:56.078938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:56.080661Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435681394412699:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:56.080423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435681767574804:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.080456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:56.080462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435681767574804:2113] [node 5] ICP22 created new session: [3:7574435686062542127:2113]
2025-11-19T13:58:56.080467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435686062542127:2113] [node 5] ICS09 handshake done sender: [3:7574435681767574812:2117] self: [3:7574435681767574808:0] peer: [5:7574435687255718265:0] socket: 162 qp: -1
2025-11-19T13:58:56.080473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435686062542127:2113] [node 5] ICS10 traffic start
2025-11-19T13:58:56.080482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:56.080918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435681394412695:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.080944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:56.080950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435681394412695:2113] [node 5] ICP22 created new session: [2:7574435685689380018:2113]
2025-11-19T13:58:56.080955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435685689380018:2113] [node 5] ICS09 handshake done sender: [2:7574435681394412699:2116] self: [2:7574435681394412698:0] peer: [5:7574435687255718264:0] socket: 197 qp: -1
2025-11-19T13:58:56.080961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435685689380018:2113] [node 5] ICS10 traffic start
2025-11-19T13:58:56.080969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:56.081317Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435687255718257:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:56.081503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435682960750873:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.081517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:56.081523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435682960750873:2049] [node 2] ICP22 created new session: [5:7574435687255718270:2049]
2025-11-19T13:58:56.081527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435687255718270:2049] [node 2] ICS09 handshake done sender: [5:7574435687255718257:2119] self: [5:7574435687255718264:0] peer: [2:7574435681394412698:0] socket: 201 qp: -1
2025-11-19T13:58:56.081532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435687255718270:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:56.081549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:56.104394Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:56.104493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435685689380007:2048] [node 1] ICS01 socket: 145 reason# EndOfStream
2025-11-19T13:58:56.104503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435681394412620:2048] [node 1] ICP30 unregister session Session# [2:7574435685689380007:2048] VirtualId# [2:7574435681394412702:0]
2025-11-19T13:58:56.104511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435685689380007:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.104559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:56.104581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:56.105078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:56.105145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435686062542112:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-11-19T13:58:56.105151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435681767574729:2048] [node 1] ICP30 unregister session Session# [3:7574435686062542112:2048] VirtualId# [3:7574435681767574805:0]
2025-11-19T13:58:56.105156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435686062542112:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.105181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:56.105198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:56.111621Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:56.111713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435687255718262:2048] [node 1] ICS01 socket: 191 reason# EndOfStream
2025-11-19T13:58:56.111721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435682960750872:2048] [node 1] ICP30 unregister session Session# [5:7574435687255718262:2048] VirtualId# [5:7574435687255718253:0]
2025-11-19T13:58:56.111729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435687255718262:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.111804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:56.111838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:56.115935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:56.116028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435687167416433:2048] [node 1] ICS07 socket disconnect 158 reason# EndOfStream
2025-11-19T13:58:56.116034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435687167416433:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.116100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:56.116123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435687167416433:2048] [node 1] ICS15 start handshake
2025-11-19T13:58:56.116356Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:56.116417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435689459476428:2048] [node 1] ICS01 socket: 165 reason# EndOfStream
2025-11-19T13:58:56.116423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435685164509032:2048] [node 1] ICP30 unregister session Session# [6:7574435689459476428:2048] VirtualId# [6:7574435689459476422:0]
2025-11-19T13:58:56.116428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435689459476428:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.116454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:56.116476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:56.119110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:56.119183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435686062542125:2049] [node 2] ICS01 socket: 183 reason# EndOfStream
2025-11-19T13:58:56.119190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435681767574730:2049] [node 2] ICP30 unregister session Session# [3:7574435686062542125:2049] VirtualId# [3:7574435686062542118:0]
2025-11-19T13:58:56.119195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435686062542125:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.119223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:56.119240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:56.120426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435682872449043:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25641) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:56.120444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435687167416433:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:56.120449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435682872449043:2048] [node 1] ICP30 unregister session Session# [4:7574435687167416433:2048] VirtualId# [4:7574435687167416421:0]
2025-11-19T13:58:56.120464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:56.120469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435682872449043:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.120472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.123861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:56.123945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435687255718270:2049] [node 2] ICS01 socket: 201 reason# EndOfStream
2025-11-19T13:58:56.123953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435682960750873:2049] [node 2] ICP30 unregister session Session# [5:7574435687255718270:2049] VirtualId# [5:7574435687255718264:0]
2025-11-19T13:58:56.123959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435687255718270:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.124003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:56.124032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:56.128935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:56.129024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435689459476423:2049] [node 2] ICS01 socket: 170 reason# EndOfStream
2025-11-19T13:58:56.129032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435685164509033:2049] [node 2] ICP30 unregister session Session# [6:7574435689459476423:2049] VirtualId# [6:7574435685164509112:0]
2025-11-19T13:58:56.129038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435689459476423:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.129099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:56.129124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:56.129428Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:56.129496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435682872449122:2049] [node 2] ICS07 socket disconnect 159 reason# EndOfStream
2025-11-19T13:58:56.129500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435682872449122:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.129534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:56.129558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435682872449122:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:56.129605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435682872449043:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:58:56.130033Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435682872449044:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5133) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:56.130045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435682872449122:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:56.130049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435682872449044:2049] [node 2] ICP30 unregister session Session# [4:7574435682872449122:2049] VirtualId# [4:7574435682872449111:0]
2025-11-19T13:58:56.130061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:56.130066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435682872449044:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5133) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.130068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5133) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.140208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435682872449044:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:58:56.152545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:56.152632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435687255718268:2050] [node 3] ICS01 socket: 209 reason# EndOfStream
2025-11-19T13:58:56.152643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435682960750874:2050] [node 3] ICP30 unregister session Session# [5:7574435687255718268:2050] VirtualId# [5:7574435687255718265:0]
2025-11-19T13:58:56.152651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435687255718268:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.152713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:56.152736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:56.157950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:56.158045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435689459476425:2050] [node 3] ICS01 socket: 176 reason# EndOfStream
2025-11-19T13:58:56.158054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435685164509034:2050] [node 3] ICP30 unregister session Session# [6:7574435689459476425:2050] VirtualId# [6:7574435685164509113:0]
2025-11-19T13:58:56.158060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435689459476425:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.158117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:56.158143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:56.172809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T13:58:56.172909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435687255718258:2108] [node 4] ICS07 socket disconnect 181 reason# EndOfStream
2025-11-19T13:58:56.172916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435687255718258:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.172973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:56.172995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435687255718258:2108] [node 4] ICS15 start handshake
2025-11-19T13:58:56.175267Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:56.175347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435689459476418:2096] [node 4] ICS07 socket disconnect 150 reason# EndOfStream
2025-11-19T13:58:56.175354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435689459476418:2096] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.175407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:56.175432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435689459476418:2096] [node 4] ICS15 start handshake
2025-11-19T13:58:56.178434Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435685164509082:2096] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22480) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:56.178452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435689459476418:2096] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:56.178457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435685164509082:2096] [node 4] ICP30 unregister session Session# [6:7574435689459476418:2096] VirtualId# [6:7574435685164509115:0]
2025-11-19T13:58:56.178473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:58:56.178478Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435685164509082:2096] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22480) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.178481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22480) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.188211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435685164509082:2096] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:56.248034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435688684956476:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:56.248051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435688684956477:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:56.271957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435689737750254:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:56.271975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435689737750255:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:56.287911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435687225465388:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:56.288150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435687225465387:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:56.307877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435686900731795:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:56.307899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435686900731796:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:56.308017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435686900731794:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:56.319994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435688347570838:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:56.320009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435688347570839:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:56.320013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435688347570840:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:56.336628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435685816736131:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:56.336641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435685816736132:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:56.336645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435685816736133:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:56.370874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435688684956537:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:56.372320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435686900731832:2078] [node 5] ICP01 ready to work
2025-11-19T13:58:56.372270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435687225465435:2094] [node 6] ICP01 ready to work
2025-11-19T13:58:56.372204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435685816736148:2062] [node 4] ICP01 ready to work
2025-11-19T13:58:56.372298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435685816736152:2065] [node 5] ICP01 ready to work
2025-11-19T13:58:56.376300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435689737750314:2105] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.380324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435686900731844:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1902
2025-11-19T13:58:56.380346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435686900731845:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1902
2025-11-19T13:58:56.380356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435686900731846:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1902
2025-11-19T13:58:56.380367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435686900731847:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1902
2025-11-19T13:58:56.380144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435687225465445:2099] [node 5] ICP01 ready to work
2025-11-19T13:58:56.376418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435688347570869:2072] [node 4] ICP01 ready to work
2025-11-19T13:58:56.379248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435688347570886:2079] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.382699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435687225465448:2102] [node 4] ICP01 ready to work
2025-11-19T13:58:56.383564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435687225465449:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20976
2025-11-19T13:58:56.377599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435689737750319:2107] [node 5] ICP01 ready to work
2025-11-19T13:58:56.381362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435689737750324:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.391948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435687225465435:2094] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19598036ns) {Configure:104(19596559ns) {ProcessConfigured:135(19215040ns) {StartInitialHandshake:157(19214374ns) {DropHandshakes:469(446ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(129ns)} GenerateSessionVirtualId:988(127ns) PrepareNewSessionHandshake:476(2170ns) {SwitchToState:178(796ns)}}}}}
2025-11-19T13:58:56.392115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435687225465464:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20976
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.392158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435685816736152:2065] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19666166ns) {Configure:104(19663841ns) {ProcessConfigured:135(19357165ns) {StartInitialHandshake:157(19356453ns) {DropHandshakes:469(505ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(83ns)} GenerateSessionVirtualId:988(272ns) PrepareNewSessionHandshake:476(859ns) {SwitchToState:178(176ns)}}}}}
2025-11-19T13:58:56.392507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435689737750331:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:346
2025-11-19T13:58:56.392532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435689737750332:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:346
2025-11-19T13:58:56.392664Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435687225465387:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(19896591ns) {Configure:104(19895789ns) {ProcessConfigured:135(19532010ns) {StartInitialHandshake:157(19530915ns) {DropHandshakes:469(408ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:476(688ns) {SwitchToState:178(496ns)}}}}}
2025-11-19T13:58:56.395253Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435689737750255:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(22482932ns) {Configure:104(22482178ns) {ProcessConfigured:135(22030391ns) {StartInitialHandshake:157(22029472ns) {DropHandshakes:469(700ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(303ns) PrepareNewSessionHandshake:476(1308ns) {SwitchToState:178(596ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.398972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435686900731795:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(26617154ns) {Configure:104(26616028ns) {ProcessConfigured:135(25304328ns) {StartInitialHandshake:157(25302477ns) {DropHandshakes:469(330ns) {DropIncomingHandshake:441(125ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:476(1499ns) {SwitchToState:178(940ns)}}}}}
2025-11-19T13:58:56.399078Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435686900731846:2095] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.399100Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435686900731847:2096] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.399171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435686900731869:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1902
2025-11-19T13:58:56.399189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435686900731795:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.399463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435686900731794:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(27541170ns) {Configure:104(27539593ns) {ProcessConfigured:135(25722628ns) {StartInitialHandshake:157(25721922ns) {DropHandshakes:469(526ns) {DropIncomingHandshake:441(136ns) DropOutgoingHandshake:456(14ns)} GenerateSessionVirtualId:988(500ns) PrepareNewSessionHandshake:476(775ns) {SwitchToState:178(45ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.400206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435689737750334:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:346
2025-11-19T13:58:56.404167Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435689737750316:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.404190Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435689737750316:2106] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.407377Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435686900731844:2093] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.407497Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435689737750331:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.407525Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435689737750334:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.407426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435686900731794:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.403839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435687225465436:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20976
2025-11-19T13:58:56.404550Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435687225465436:2095] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.404593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435687225465387:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.411424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435688347570840:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(39015341ns) {Configure:104(39013923ns) {ProcessConfigured:135(38716176ns) {StartInitialHandshake:157(38715429ns) {DropHandshakes:469(538ns) {DropIncomingHandshake:441(97ns) DropOutgoingHandshake:456(76ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:476(1534ns) {SwitchToState:178(1228ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.411773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435688684956551:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34812
2025-11-19T13:58:56.411844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435688684956552:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34812
2025-11-19T13:58:56.411855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435688684956553:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34812
2025-11-19T13:58:56.412077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435688347570914:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
2025-11-19T13:58:56.412593Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435688347570914:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.412635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435688347570839:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.412740Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435687225465449:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.412885Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435689737750321:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.412894Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435689737750321:2108] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.413583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435688347570916:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
2025-11-19T13:58:56.413727Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435688684956551:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.413825Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435689737750321:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:56.414007Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435688347570914:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:56.414103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435688347570839:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.414122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:56.414128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435688347570839:2049] [node 2] ICP22 created new session: [5:7574435688347570917:2049]
2025-11-19T13:58:56.414134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435688347570917:2049] [node 2] ICS09 handshake done sender: [5:7574435688347570914:2113] self: [5:7574435688347570915:0] peer: [2:7574435689737750320:0] socket: 172 qp: -1
2025-11-19T13:58:56.414140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435688347570917:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:56.414160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.414578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435687225465470:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20976
2025-11-19T13:58:56.416141Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435687225465470:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.416260Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435688684956539:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.416269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435688684956539:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.415928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435689737750254:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(34514721ns) {Configure:104(34514107ns) {ProcessConfigured:135(30881077ns) {StartInitialHandshake:157(30880382ns) {DropHandshakes:469(847ns) {DropIncomingHandshake:441(418ns) DropOutgoingHandshake:456(151ns)} GenerateSessionVirtualId:988(303ns) PrepareNewSessionHandshake:476(1326ns) {SwitchToState:178(791ns)}}}}}
2025-11-19T13:58:56.415974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435689737750254:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.416945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435689737750339:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:346
2025-11-19T13:58:56.417120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435689737750319:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.417130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:56.417137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435689737750319:2107] [node 5] ICP22 created new session: [2:7574435689737750340:2107]
2025-11-19T13:58:56.417144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435689737750340:2107] [node 5] ICS09 handshake done sender: [2:7574435689737750321:2108] self: [2:7574435689737750320:0] peer: [5:7574435688347570915:0] socket: 166 qp: -1
2025-11-19T13:58:56.417152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435689737750340:2107] [node 5] ICS10 traffic start
2025-11-19T13:58:56.417170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.416952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435687225465445:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(36736958ns) {Configure:104(36736073ns) {ProcessConfigured:135(36417479ns) {StartInitialHandshake:157(36416386ns) {DropHandshakes:469(531ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:476(1174ns) {SwitchToState:178(902ns)}}}}}
2025-11-19T13:58:56.419043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435688684956550:2111] [node 5] ICP01 ready to work
2025-11-19T13:58:56.419130Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435688684956539:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:56.419838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435686900731863:2089] [node 6] ICP01 ready to work
2025-11-19T13:58:56.419838Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435688684956541:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.419858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435688684956541:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.419958Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435688684956543:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.419972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435688684956543:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.420259Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435689737750331:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:56.423227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435687225465473:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20976
2025-11-19T13:58:56.426080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435689737750254:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.426104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:56.426111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435689737750254:2048] [node 1] ICP22 created new session: [2:7574435689737750343:2048]
2025-11-19T13:58:56.426119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435689737750343:2048] [node 1] ICS09 handshake done sender: [2:7574435689737750331:2113] self: [2:7574435689737750338:0] peer: [1:7574435688684956538:0] socket: 154 qp: -1
2025-11-19T13:58:56.426127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435689737750343:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:56.426150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.427078Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435687225465473:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.427112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435687225465388:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.424942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435685816736132:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(52955766ns) {Configure:104(52955163ns) {ProcessConfigured:135(52224095ns) {StartInitialHandshake:157(52223062ns) {DropHandshakes:469(213ns) {DropIncomingHandshake:441(31ns) DropOutgoingHandshake:456(31ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(1174ns) {SwitchToState:178(732ns)}}}}}
2025-11-19T13:58:56.427357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685816736209:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34844
2025-11-19T13:58:56.427418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685816736210:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34844
2025-11-19T13:58:56.427429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685816736211:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34844
2025-11-19T13:58:56.427532Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435686900731832:2078] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(55157134ns) {Configure:104(55156584ns) {ProcessConfigured:135(52975938ns) {StartInitialHandshake:157(52974583ns) {DropHandshakes:469(394ns) {DropIncomingHandshake:441(46ns) DropOutgoingHandshake:456(44ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:476(1202ns) {SwitchToState:178(178ns)}}}}}
2025-11-19T13:58:56.428865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435688684956549:2112] [node 6] ICP01 ready to work
2025-11-19T13:58:56.428900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435688684956476:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.428912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:56.428920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435688684956476:2048] [node 2] ICP22 created new session: [1:7574435688684956556:2048]
2025-11-19T13:58:56.428926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435688684956556:2048] [node 2] ICS09 handshake done sender: [1:7574435688684956539:2108] self: [1:7574435688684956538:0] peer: [2:7574435689737750338:0] socket: 150 qp: -1
2025-11-19T13:58:56.428932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435688684956556:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:56.428952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:56.427871Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435689737750316:2106] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.435669Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435686900731845:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.432877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435689737750314:2105] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.432901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:56.435349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435685816736209:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.432907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435689737750314:2105] [node 4] ICP22 created new session: [2:7574435689737750346:2105]
2025-11-19T13:58:56.435466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435685816736152:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.432913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435689737750346:2105] [node 4] ICS09 handshake done sender: [2:7574435689737750316:2106] self: [2:7574435689737750315:0] peer: [4:7574435686900731870:0] socket: 156 qp: -1
2025-11-19T13:58:56.435516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685816736213:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34844
2025-11-19T13:58:56.432918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435689737750346:2105] [node 4] ICS10 traffic start
2025-11-19T13:58:56.432925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:56.435269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435687225465475:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20976
2025-11-19T13:58:56.435868Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435687225465464:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.435894Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435688684956541:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:56.436043Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435687225465436:2095] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:56.436190Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435686900731795:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435686900731846:2095] held: no
2025-11-19T13:58:56.436487Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435688684956552:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.436587Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435688684956553:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.436603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435688684956477:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.436615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:56.436621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435688684956477:2049] [node 3] ICP22 created new session: [1:7574435688684956562:2049]
2025-11-19T13:58:56.436627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435688684956562:2049] [node 3] ICS09 handshake done sender: [1:7574435688684956541:2109] self: [1:7574435688684956540:0] peer: [3:7574435687225465468:0] socket: 151 qp: -1
2025-11-19T13:58:56.436634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435688684956562:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:56.436641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:56.437190Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435688684956543:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:56.437315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435688684956537:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.437329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:56.437338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435688684956537:2107] [node 4] ICP22 created new session: [1:7574435688684956567:2107]
2025-11-19T13:58:56.437344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435688684956567:2107] [node 4] ICS09 handshake done sender: [1:7574435688684956543:2110] self: [1:7574435688684956542:0] peer: [4:7574435686900731875:0] socket: 152 qp: -1
2025-11-19T13:58:56.437348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435688684956567:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:56.437354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:56.437655Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435685816736213:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.437691Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435685816736210:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.437721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435685816736131:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.437731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435685816736132:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.438003Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435688684956559:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.438011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435688684956559:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.439763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685816736216:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34844
2025-11-19T13:58:56.439938Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435686900731794:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64647) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435686900731844:2093] held: no
2025-11-19T13:58:56.440320Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435688347570889:2089] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.440340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435688347570889:2089] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.441171Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435688684956559:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:56.441337Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435685816736213:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:56.441417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435685816736131:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.441436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:56.441443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435685816736131:2048] [node 1] ICP22 created new session: [6:7574435685816736217:2048]
2025-11-19T13:58:56.441448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435685816736217:2048] [node 1] ICS09 handshake done sender: [6:7574435685816736213:2118] self: [6:7574435685816736214:0] peer: [1:7574435688684956558:0] socket: 199 qp: -1
2025-11-19T13:58:56.441454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435685816736217:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:56.441472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:56.443705Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435689737750333:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.443724Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435689737750333:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.443785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435686900731878:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1902
2025-11-19T13:58:56.444237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435687225465387:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.444259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:56.444265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435687225465387:2048] [node 1] ICP22 created new session: [3:7574435687225465477:2048]
2025-11-19T13:58:56.444272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435687225465477:2048] [node 1] ICS09 handshake done sender: [3:7574435687225465436:2095] self: [3:7574435687225465468:0] peer: [1:7574435688684956540:0] socket: 153 qp: -1
2025-11-19T13:58:56.444278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435687225465477:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:56.444295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:56.444711Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435686900731863:2089] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(24789243ns) {Configure:104(24788464ns) {ProcessConfigured:135(15938183ns) {StartInitialHandshake:157(15937608ns) {DropHandshakes:469(623ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(149ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:476(629ns) {SwitchToState:178(78ns)}}}}}
2025-11-19T13:58:56.444756Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435686900731846:2095] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:56.444877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435686900731795:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.444887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:56.444892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435686900731795:2049] [node 2] ICP22 created new session: [4:7574435686900731883:2049]
2025-11-19T13:58:56.444898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435686900731883:2049] [node 2] ICS09 handshake done sender: [4:7574435686900731846:2095] self: [4:7574435686900731870:0] peer: [2:7574435689737750315:0] socket: 162 qp: -1
2025-11-19T13:58:56.444902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435686900731883:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:56.444915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:56.445216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435688684956549:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.445234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:56.445239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435688684956549:2112] [node 6] ICP22 created new session: [1:7574435688684956570:2112]
2025-11-19T13:58:56.445244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435688684956570:2112] [node 6] ICS09 handshake done sender: [1:7574435688684956559:2116] self: [1:7574435688684956558:0] peer: [6:7574435685816736214:0] socket: 196 qp: -1
2025-11-19T13:58:56.445249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435688684956570:2112] [node 6] ICS10 traffic start
2025-11-19T13:58:56.445262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.448089Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435688684956550:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(28362956ns) {Configure:104(28362475ns) {ProcessConfigured:135(28004945ns) {StartInitialHandshake:157(27990146ns) {DropHandshakes:469(733ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(188ns)} GenerateSessionVirtualId:988(1209ns) PrepareNewSessionHandshake:476(1133ns) {SwitchToState:178(96ns)}} IncomingHandshake:261(6578ns) {DropIncomingHandshake:441(14ns) PrepareNewSessionHandshake:476(203ns) {SwitchToState:178(153ns)} GenerateSessionVirtualId:988(429ns) IssueIncomingHandshakeReply:193(308ns)}}}}
2025-11-19T13:58:56.448372Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435686900731869:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.448421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435686900731885:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1902
2025-11-19T13:58:56.450808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435686900731796:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.455894Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435686900731844:2093] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:56.468828Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435687225465451:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.468858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435687225465451:2104] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.469356Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435689737750326:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.469380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435689737750326:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.469447Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435689737750332:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.473945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685816736219:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34844
2025-11-19T13:58:56.475128Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435685816736211:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.475174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435685816736133:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.473370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435686900731794:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.473400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:56.473413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435686900731794:2048] [node 1] ICP22 created new session: [4:7574435686900731887:2048]
2025-11-19T13:58:56.473422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435686900731887:2048] [node 1] ICS09 handshake done sender: [4:7574435686900731844:2093] self: [4:7574435686900731875:0] peer: [1:7574435688684956542:0] socket: 160 qp: -1
2025-11-19T13:58:56.473430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435686900731887:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:56.473440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:56.476138Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435688684956476:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(47202675ns) {HandleHandshakeStatus:329(47201835ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(172ns) SwitchToState:178(290ns) ProcessPendingSessionEvents:495(32125315ns) {ForwardSessionEventToSession:612(32108591ns) ForwardSessionEventToSession:612(3124ns) ForwardSessionEventToSession:612(810ns) ForwardSessionEventToSession:612(852ns) ForwardSessionEventToSession:612(557ns) ForwardSessionEventToSession:612(1434ns) ForwardSessionEventToSession:612(498ns) ForwardSessionEventToSession:612(636ns) ForwardSessionEventToSession:612(1007ns) ForwardSessionEventToSession:612(1215ns)}}}
2025-11-19T13:58:56.475931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435688347570923:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
2025-11-19T13:58:56.476008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435688347570924:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
2025-11-19T13:58:56.476873Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435688347570889:2089] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:56.477572Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435688347570923:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.477624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435688347570869:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.477777Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435686900731876:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.477788Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435686900731876:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.478988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435687225465480:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20976
2025-11-19T13:58:56.479198Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435687225465451:2104] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:56.476662Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435689737750333:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:56.480039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435687225465448:2102] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.480064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:56.480073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435687225465448:2102] [node 4] ICP22 created new session: [3:7574435687225465482:2102]
2025-11-19T13:58:56.480079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435687225465482:2102] [node 4] ICS09 handshake done sender: [3:7574435687225465451:2104] self: [3:7574435687225465450:0] peer: [4:7574435686900731886:0] socket: 169 qp: -1
2025-11-19T13:58:56.480086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435687225465482:2102] [node 4] ICS10 traffic start
2025-11-19T13:58:56.480106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: 2025-11-19T13:58:56.481796Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435686900731876:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:56.482130Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435688347570924:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.482176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435688347570838:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.482461Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435687225465463:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.482474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435687225465463:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.482372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435688347570886:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.482386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:56.482393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435688347570886:2079] [node 6] ICP22 created new session: [5:7574435688347570927:2079]
2025-11-19T13:58:56.482399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435688347570927:2079] [node 6] ICS09 handshake done sender: [5:7574435688347570889:2089] self: [5:7574435688347570888:0] peer: [6:7574435685816736212:0] socket: 167 qp: -1
2025-11-19T13:58:56.482406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435688347570927:2079] [node 6] ICS10 traffic start
2025-11-19T13:58:56.482413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:56.480253Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435685816736131:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(38789133ns) {HandleHandshakeStatus:329(38788645ns) {DropIncomingHandshake:441(253ns) DropOutgoingHandshake:456(5554ns) SwitchToState:178(80ns) ProcessPendingSessionEvents:495(11077557ns) {ForwardSessionEventToSession:612(11064316ns) ForwardSessionEventToSession:612(1940ns) ForwardSessionEventToSession:612(1462ns) ForwardSessionEventToSession:612(622ns) ForwardSessionEventToSession:612(902ns) ForwardSessionEventToSession:612(983ns) ForwardSessionEventToSession:612(612ns) ForwardSessionEventToSession:612(564ns) ForwardSessionEventToSession:612(638ns) ForwardSessionEventToSession:612(863ns)}}}
2025-11-19T13:58:56.480408Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435685816736209:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:56.482754Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435689737750326:2110] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-19T13:58:56.483010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435689737750324:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.483020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:56.483026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435689737750324:2109] [node 6] ICP22 created new session: [2:7574435689737750351:2109]
2025-11-19T13:58:56.483031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435689737750351:2109] [node 6] ICS09 handshake done sender: [2:7574435689737750326:2110] self: [2:7574435689737750325:0] peer: [6:7574435685816736215:0] socket: 173 qp: -1
2025-11-19T13:58:56.483036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435689737750351:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:56.483043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:56.483849Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435687225465473:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:56.483901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435686900731832:2078] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.483919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:56.483924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435686900731832:2078] [node 5] ICP22 created new session: [4:7574435686900731891:2078]
2025-11-19T13:58:56.483928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435686900731891:2078] [node 5] ICS09 handshake done sender: [4:7574435686900731876:2118] self: [4:7574435686900731838:0] peer: [5:7574435688347570925:0] socket: 181 qp: -1
2025-11-19T13:58:56.483935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435686900731891:2078] [node 5] ICS10 traffic start
2025-11-19T13:58:56.483943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:56.484035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435685816736152:2065] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.484048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:56.484054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435685816736152:2065] [node 5] ICP22 created new session: [6:7574435685816736222:2065]
2025-11-19T13:58:56.484058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435685816736222:2065] [node 5] ICS09 handshake done sender: [6:7574435685816736209:2115] self: [6:7574435685816736212:0] peer: [5:7574435688347570888:0] socket: 171 qp: -1
2025-11-19T13:58:56.484062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435685816736222:2065] [node 5] ICS10 traffic start
2025-11-19T13:58:56.484068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:56.484324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435687225465388:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.484356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:56.484366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435687225465388:2049] [node 2] ICP22 created new session: [3:7574435687225465484:2049]
2025-11-19T13:58:56.484371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435687225465484:2049] [node 2] ICS09 handshake done sender: [3:7574435687225465473:2119] self: [3:7574435687225465474:0] peer: [2:7574435689737750312:0] socket: 191 qp: -1
2025-11-19T13:58:56.484378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435687225465484:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:56.484385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:56.488027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435689737750255:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.488075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:56.488085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435689737750255:2049] [node 3] ICP22 created new session: [2:7574435689737750353:2049]
2025-11-19T13:58:56.488092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435689737750353:2049] [node 3] ICS09 handshake done sender: [2:7574435689737750333:2115] self: [2:7574435689737750312:0] peer: [3:7574435687225465474:0] socket: 190 qp: -1
2025-11-19T13:58:56.488098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435689737750353:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:56.488110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:56.488181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435686900731890:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1902
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.484929Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435688684956572:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.484939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435688684956572:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.490067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435688347570930:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.490432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435688347570929:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
2025-11-19T13:58:56.490499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435688347570931:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
2025-11-19T13:58:56.490513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435688347570932:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
2025-11-19T13:58:56.490713Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435688684956572:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:56.491214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435688684956550:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.491239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:56.491247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435688684956550:2111] [node 5] ICP22 created new session: [1:7574435688684956576:2111]
2025-11-19T13:58:56.491255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435688684956576:2111] [node 5] ICS09 handshake done sender: [1:7574435688684956572:2117] self: [1:7574435688684956555:0] peer: [5:7574435688347570926:0] socket: 203 qp: -1
2025-11-19T13:58:56.491261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435688684956576:2111] [node 5] ICS10 traffic start
2025-11-19T13:58:56.491270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.491808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435689737750355:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:346
2025-11-19T13:58:56.492119Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435688347570924:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:56.492122Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435686900731796:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61521) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435686900731869:2115] held: no
2025-11-19T13:58:56.492161Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435686900731869:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:56.491903Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435687225465463:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:56.492467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435686900731796:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.492481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:56.492487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435686900731796:2050] [node 3] ICP22 created new session: [4:7574435686900731894:2050]
2025-11-19T13:58:56.492493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435686900731894:2050] [node 3] ICS09 handshake done sender: [4:7574435686900731869:2115] self: [4:7574435686900731886:0] peer: [3:7574435687225465450:0] socket: 178 qp: -1
2025-11-19T13:58:56.492498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435686900731894:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:56.492515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:56.492704Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435688347570929:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.492751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435688347570840:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.493100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435687225465435:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.493119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:56.493126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435687225465435:2094] [node 6] ICP22 created new session: [3:7574435687225465487:2094]
2025-11-19T13:58:56.493131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435687225465487:2094] [node 6] ICS09 handshake done sender: [3:7574435687225465463:2113] self: [3:7574435687225465437:0] peer: [6:7574435685816736220:0] socket: 176 qp: -1
2025-11-19T13:58:56.493137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435687225465487:2094] [node 6] ICS10 traffic start
2025-11-19T13:58:56.493147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:56.493251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435688347570838:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.493268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:56.493274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435688347570838:2048] [node 1] ICP22 created new session: [5:7574435688347570934:2048]
2025-11-19T13:58:56.493278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435688347570934:2048] [node 1] ICS09 handshake done sender: [5:7574435688347570924:2118] self: [5:7574435688347570926:0] peer: [1:7574435688684956555:0] socket: 207 qp: -1
2025-11-19T13:58:56.493282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435688347570934:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:56.493290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:56.494515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435689737750255:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(17187524ns) {HandleHandshakeStatus:329(17186810ns) {DropIncomingHandshake:441(10655ns) DropOutgoingHandshake:456(99ns) SwitchToState:178(82ns) ProcessPendingSessionEvents:495(6322913ns) {ForwardSessionEventToSession:612(6311737ns) ForwardSessionEventToSession:612(1211ns) ForwardSessionEventToSession:612(923ns) ForwardSessionEventToSession:612(640ns) ForwardSessionEventToSession:612(1027ns) ForwardSessionEventToSession:612(961ns) ForwardSessionEventToSession:612(759ns) ForwardSessionEventToSession:612(616ns) ForwardSessionEventToSession:612(561ns) ForwardSessionEventToSession:612(893ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.494128Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435688347570923:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:56.495196Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435687225465471:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.495210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435687225465471:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.500213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435688347570869:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.500231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:56.500237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435688347570869:2072] [node 4] ICP22 created new session: [5:7574435688347570936:2072]
2025-11-19T13:58:56.500244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435688347570936:2072] [node 4] ICS09 handshake done sender: [5:7574435688347570923:2117] self: [5:7574435688347570925:0] peer: [4:7574435686900731838:0] socket: 202 qp: -1
2025-11-19T13:58:56.500251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435688347570936:2072] [node 4] ICS10 traffic start
2025-11-19T13:58:56.500259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:56.500317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435688347570869:2072] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.508151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685816736226:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34844
2025-11-19T13:58:56.508580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685816736225:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34844
2025-11-19T13:58:56.509248Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435685816736210:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:56.509401Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435689737750355:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.509453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435689737750324:2109] [node 6] ICP18 handshake (actor [2:7574435689737750355:2119]) is too old
2025-11-19T13:58:56.509762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435685816736132:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.509785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:56.509793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435685816736132:2049] [node 2] ICP22 created new session: [6:7574435685816736228:2049]
2025-11-19T13:58:56.509799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435685816736228:2049] [node 2] ICS09 handshake done sender: [6:7574435685816736210:2116] self: [6:7574435685816736215:0] peer: [2:7574435689737750325:0] socket: 193 qp: -1
2025-11-19T13:58:56.509808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435685816736228:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:56.509818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:56.510805Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435685816736211:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:56.510854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435685816736133:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.510869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:56.510877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435685816736133:2050] [node 3] ICP22 created new session: [6:7574435685816736230:2050]
2025-11-19T13:58:56.510884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435685816736230:2050] [node 3] ICS09 handshake done sender: [6:7574435685816736211:2117] self: [6:7574435685816736220:0] peer: [3:7574435687225465437:0] socket: 194 qp: -1
2025-11-19T13:58:56.510891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435685816736230:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:56.510898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:56.512182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685816736227:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34844
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.512470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435688347570938:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
2025-11-19T13:58:56.515940Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435687225465471:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:56.516197Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435685816736227:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.516258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435685816736148:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:56.516606Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435688347570929:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:56.516768Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435686900731881:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:56.516784Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435686900731881:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:56.516337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435687225465445:2099] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.516366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:56.516373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435687225465445:2099] [node 5] ICP22 created new session: [3:7574435687225465490:2099]
2025-11-19T13:58:56.516381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435687225465490:2099] [node 5] ICS09 handshake done sender: [3:7574435687225465471:2118] self: [3:7574435687225465446:0] peer: [5:7574435688347570933:0] socket: 208 qp: -1
2025-11-19T13:58:56.516388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435687225465490:2099] [node 5] ICS10 traffic start
2025-11-19T13:58:56.516401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:56.519595Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435686900731881:2120] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:56.519717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435686900731863:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:56.519740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:56.519773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435686900731863:2089] [node 6] ICP22 created new session: [4:7574435686900731897:2089]
2025-11-19T13:58:56.519781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435686900731897:2089] [node 6] ICS09 handshake done sender: [4:7574435686900731881:2120] self: [4:7574435686900731879:0] peer: [6:7574435685816736232:0] socket: 177 qp: -1
2025-11-19T13:58:56.519789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435686900731897:2089] [node 6] ICS10 traffic start
2025-11-19T13:58:56.519800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:56.519236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435688347570840:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61521) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435688347570929:2119] held: no
2025-11-19T13:58:56.520089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435688347570840:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.520101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:56.520107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435688347570840:2050] [node 3] ICP22 created new session: [5:7574435688347570939:2050]
2025-11-19T13:58:56.520112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435688347570939:2050] [node 3] ICS09 handshake done sender: [5:7574435688347570929:2119] self: [5:7574435688347570933:0] peer: [3:7574435687225465446:0] socket: 209 qp: -1
2025-11-19T13:58:56.520120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435688347570939:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:56.520136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.520618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435685816736233:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34844
2025-11-19T13:58:56.521678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435685816736148:2062] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28167) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435685816736227:2123] held: no
2025-11-19T13:58:56.521903Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435685816736227:2123] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:56.521993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435685816736148:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:56.522006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:56.522013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435685816736148:2062] [node 4] ICP22 created new session: [6:7574435685816736234:2062]
2025-11-19T13:58:56.522020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435685816736234:2062] [node 4] ICS09 handshake done sender: [6:7574435685816736227:2123] self: [6:7574435685816736232:0] peer: [4:7574435686900731879:0] socket: 157 qp: -1
2025-11-19T13:58:56.522026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435685816736234:2062] [node 4] ICS10 traffic start
2025-11-19T13:58:56.522038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:56.530600Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:56.530700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435689737750343:2048] [node 1] ICS01 socket: 154 reason# EndOfStream
2025-11-19T13:58:56.530711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435689737750254:2048] [node 1] ICP30 unregister session Session# [2:7574435689737750343:2048] VirtualId# [2:7574435689737750338:0]
2025-11-19T13:58:56.530720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435689737750343:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.530770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:56.530793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:56.532410Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:56.532499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435687225465477:2048] [node 1] ICS01 socket: 153 reason# EndOfStream
2025-11-19T13:58:56.532508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435687225465387:2048] [node 1] ICP30 unregister session Session# [3:7574435687225465477:2048] VirtualId# [3:7574435687225465468:0]
2025-11-19T13:58:56.532515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435687225465477:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.532551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:56.532572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:56.537562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:56.537656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435686900731887:2048] [node 1] ICS01 socket: 160 reason# EndOfStream
2025-11-19T13:58:56.537665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435686900731794:2048] [node 1] ICP30 unregister session Session# [4:7574435686900731887:2048] VirtualId# [4:7574435686900731875:0]
2025-11-19T13:58:56.537672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435686900731887:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.537738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:56.537764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:56.537805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:56.537866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435688347570934:2048] [node 1] ICS07 socket disconnect 207 reason# EndOfStream
2025-11-19T13:58:56.537872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435688347570934:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.537899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:56.537921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435688347570934:2048] [node 1] ICS15 start handshake
2025-11-19T13:58:56.538176Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:56.538253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435685816736217:2048] [node 1] ICS07 socket disconnect 199 reason# EndOfStream
2025-11-19T13:58:56.538257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435685816736217:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.538286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:56.538305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435685816736217:2048] [node 1] ICS15 start handshake
2025-11-19T13:58:56.542927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435685816736131:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64647) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:56.542947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435685816736217:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:56.542952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435685816736131:2048] [node 1] ICP30 unregister session Session# [6:7574435685816736217:2048] VirtualId# [6:7574435685816736214:0]
2025-11-19T13:58:56.542969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:56.542975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435685816736131:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64647) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.542978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64647) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.543367Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435688347570838:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64647) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:56.543379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435688347570934:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:56.543384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435688347570838:2048] [node 1] ICP30 unregister session Session# [5:7574435688347570934:2048] VirtualId# [5:7574435688347570926:0]
2025-11-19T13:58:56.543395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:56.543399Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435688347570838:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64647) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.543400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64647) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.548071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:56.548176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435686900731883:2049] [node 2] ICS01 socket: 162 reason# EndOfStream
2025-11-19T13:58:56.548185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435686900731795:2049] [node 2] ICP30 unregister session Session# [4:7574435686900731883:2049] VirtualId# [4:7574435686900731870:0]
2025-11-19T13:58:56.548193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435686900731883:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.548244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:56.548268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:56.548456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:56.548519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435688347570917:2049] [node 2] ICS07 socket disconnect 172 reason# EndOfStream
2025-11-19T13:58:56.548524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435688347570917:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.548553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:56.548574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435688347570917:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:56.548819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:56.548884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435685816736228:2049] [node 2] ICS01 socket: 193 reason# EndOfStream
2025-11-19T13:58:56.548888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435685816736132:2049] [node 2] ICP30 unregister session Session# [6:7574435685816736228:2049] VirtualId# [6:7574435685816736215:0]
2025-11-19T13:58:56.548893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435685816736228:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.548916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:56.548941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:56.549855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435688347570839:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23041) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:56.549867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435688347570917:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:56.549871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435688347570839:2049] [node 2] ICP30 unregister session Session# [5:7574435688347570917:2049] VirtualId# [5:7574435688347570915:0]
2025-11-19T13:58:56.549881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:56.549887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435688347570839:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23041) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.549890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23041) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.556066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435685816736131:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:58:56.556120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435688347570838:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:58:56.559859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435688347570839:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:58:56.563351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:56.563445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435685816736230:2050] [node 3] ICS07 socket disconnect 194 reason# EndOfStream
2025-11-19T13:58:56.563451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435685816736230:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.563514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:56.563535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435685816736230:2050] [node 3] ICS15 start handshake
2025-11-19T13:58:56.564283Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:56.564388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435688347570939:2050] [node 3] ICS01 socket: 209 reason# EndOfStream
2025-11-19T13:58:56.564394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435688347570840:2050] [node 3] ICP30 unregister session Session# [5:7574435688347570939:2050] VirtualId# [5:7574435688347570933:0]
2025-11-19T13:58:56.564400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435688347570939:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.564451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:56.564482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:56.567667Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435685816736133:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61521) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:56.567688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435685816736230:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:56.567693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435685816736133:2050] [node 3] ICP30 unregister session Session# [6:7574435685816736230:2050] VirtualId# [6:7574435685816736220:0]
2025-11-19T13:58:56.567713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:56.567718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435685816736133:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61521) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.567721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61521) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:56.575879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435685816736133:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:58:56.587973Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:56.588076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435685816736234:2062] [node 4] ICS01 socket: 157 reason# EndOfStream
2025-11-19T13:58:56.588085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435685816736148:2062] [node 4] ICP30 unregister session Session# [6:7574435685816736234:2062] VirtualId# [6:7574435685816736232:0]
2025-11-19T13:58:56.588094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435685816736234:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:56.588167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:56.588200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:56.764006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435686473578720:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:56.764030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435686473578721:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:56.784453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435688781520060:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:56.784467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435688781520061:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:56.811960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435686382948008:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:56.811973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435686382948009:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:56.851956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435686943860247:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:56.851972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435686943860248:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:56.851978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435686943860249:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:56.895931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435686653813192:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:56.895946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435686653813193:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:56.895951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435686653813194:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:56.936155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435686792590306:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:56.936168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435686792590307:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:56.936172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435686792590308:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.968438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435686473578782:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:56.968523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435686473578783:2108] [node 5] ICP01 ready to work
2025-11-19T13:58:56.968558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435688781520119:2105] [node 6] ICP01 ready to work
2025-11-19T13:58:56.971099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435688781520125:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:56.971175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435688781520126:2108] [node 5] ICP01 ready to work
2025-11-19T13:58:56.969630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435686792590347:2088] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7740
2025-11-19T13:58:56.972470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435686473578796:2113] [node 6] ICP01 ready to work
2025-11-19T13:58:56.976548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435686792590350:2090] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:56.986077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435686943860310:2108] [node 5] ICP01 ready to work
2025-11-19T13:58:56.987051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435686943860309:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34872
2025-11-19T13:58:56.987117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435686653813257:2109] [node 4] ICP01 ready to work
2025-11-19T13:58:56.987453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435686382948067:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8749
2025-11-19T13:58:56.987590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435686382948074:2108] [node 5] ICP01 ready to work
2025-11-19T13:58:56.987670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435686382948075:2109] [node 6] ICP01 ready to work
2025-11-19T13:58:56.987738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435686382948076:2110] [node 4] ICP01 ready to work
2025-11-19T13:58:56.989120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435686943860321:2113] [node 6] ICP01 ready to work
2025-11-19T13:58:56.995946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435686653813261:2111] [node 6] ICP01 ready to work
2025-11-19T13:58:56.996485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435686792590350:2090] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19870114ns) {Configure:104(19869043ns) {ProcessConfigured:135(19480235ns) {StartInitialHandshake:157(19479730ns) {DropHandshakes:469(859ns) {DropIncomingHandshake:441(180ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:476(1212ns) {SwitchToState:178(637ns)}}}}}
2025-11-19T13:58:56.996618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435686792590373:2092] [node 5] ICP01 ready to work
2025-11-19T13:58:57.008476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435688781520125:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(37253224ns) {Configure:104(37251481ns) {ProcessConfigured:135(36954294ns) {StartInitialHandshake:157(36953630ns) {DropHandshakes:469(924ns) {DropIncomingHandshake:441(162ns) DropOutgoingHandshake:456(167ns)} GenerateSessionVirtualId:988(74ns) PrepareNewSessionHandshake:476(1321ns) {SwitchToState:178(1072ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.013697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435686473578799:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14409
2025-11-19T13:58:57.013720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435686473578800:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14409
2025-11-19T13:58:57.013764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435690768546097:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14409
2025-11-19T13:58:57.019308Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435686473578800:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.019440Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435686943860309:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.019472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435686943860247:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.022837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435686792590373:2092] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(26122063ns) {Configure:104(26121081ns) {ProcessConfigured:135(25790876ns) {StartInitialHandshake:157(25790200ns) {DropHandshakes:469(908ns) {DropIncomingHandshake:441(131ns) DropOutgoingHandshake:456(140ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:476(1876ns) {SwitchToState:178(502ns)}}}}}
2025-11-19T13:58:57.027796Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435686473578789:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.027816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435686473578789:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.034346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435690948780558:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
2025-11-19T13:58:57.034404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435690948780559:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
2025-11-19T13:58:57.034434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435690948780560:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.035728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435693076487429:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17760
2025-11-19T13:58:57.035809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435693076487430:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17760
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.037045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435691238827623:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34872
2025-11-19T13:58:57.035992Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435690948780559:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.036049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435690948780558:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.037295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435693076487431:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17760
2025-11-19T13:58:57.040507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435691238827624:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34872
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.045227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435690768546099:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14409
2025-11-19T13:58:57.045488Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435686473578789:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:57.045605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435686473578782:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.045630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:57.045636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435686473578782:2107] [node 4] ICP22 created new session: [1:7574435690768546101:2107]
2025-11-19T13:58:57.045643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435690768546101:2107] [node 4] ICS09 handshake done sender: [1:7574435686473578789:2111] self: [1:7574435686473578788:0] peer: [4:7574435691238827622:0] socket: 160 qp: -1
2025-11-19T13:58:57.045650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435690768546101:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:57.045659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:57.046349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435686653813194:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.046374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435686653813257:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.046144Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435686382948067:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.046229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435686382948008:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.051508Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435693076487430:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.051694Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435693076487429:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.051913Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435686382948078:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.051928Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435686382948078:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.051732Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435693076487431:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.051868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435688781520060:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.062320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435690948780571:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
2025-11-19T13:58:57.062359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435690948780572:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
2025-11-19T13:58:57.060529Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435690768546097:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.060574Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435686473578787:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.060578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435686473578787:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.061203Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435686473578799:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.061225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435686473578785:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.061228Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435686473578785:2109] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.063015Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435686943860314:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.063027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435686943860314:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.063763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435686943860247:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18744) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435686943860309:2109] held: no
2025-11-19T13:58:57.063878Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435691238827624:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.064007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435691238827625:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34872
2025-11-19T13:58:57.064020Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435686382948078:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.064358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435690948780573:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
2025-11-19T13:58:57.064149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435686382948074:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.064165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:57.064171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435686382948074:2108] [node 5] ICP22 created new session: [3:7574435690677915387:2108]
2025-11-19T13:58:57.064177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435690677915387:2108] [node 5] ICS09 handshake done sender: [3:7574435686382948078:2112] self: [3:7574435686382948077:0] peer: [5:7574435690948780569:0] socket: 171 qp: -1
2025-11-19T13:58:57.064182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435690677915387:2108] [node 5] ICS10 traffic start
2025-11-19T13:58:57.064190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:57.064349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435690677915386:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8749
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.064552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435691238827626:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34872
2025-11-19T13:58:57.064633Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435686943860309:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:57.064768Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435686943860314:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:57.064852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435686943860247:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.064866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:57.064871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435686943860247:2048] [node 1] ICP22 created new session: [4:7574435691238827630:2048]
2025-11-19T13:58:57.064876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435691238827630:2048] [node 1] ICS09 handshake done sender: [4:7574435686943860309:2109] self: [4:7574435691238827622:0] peer: [1:7574435686473578788:0] socket: 162 qp: -1
2025-11-19T13:58:57.064881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435691238827630:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:57.064888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:57.064994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435686943860310:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.065002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:57.065006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435686943860310:2108] [node 5] ICP22 created new session: [4:7574435691238827632:2108]
2025-11-19T13:58:57.065020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435691238827632:2108] [node 5] ICS09 handshake done sender: [4:7574435686943860314:2111] self: [4:7574435686943860313:0] peer: [5:7574435690948780570:0] socket: 165 qp: -1
2025-11-19T13:58:57.065024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435691238827632:2108] [node 5] ICS10 traffic start
2025-11-19T13:58:57.065029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:57.065456Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435686473578787:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:57.065618Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435686473578785:2109] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.072062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435690948780574:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
2025-11-19T13:58:57.076061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435686473578721:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.076092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:57.076101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435686473578721:2049] [node 3] ICP22 created new session: [1:7574435690768546108:2049]
2025-11-19T13:58:57.076108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435690768546108:2049] [node 3] ICS09 handshake done sender: [1:7574435686473578787:2110] self: [1:7574435686473578786:0] peer: [3:7574435690677915384:0] socket: 159 qp: -1
2025-11-19T13:58:57.076130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435690768546108:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:57.076139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.076568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435693076487441:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17760
2025-11-19T13:58:57.079618Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435686382948067:2106] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:57.076446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435686473578720:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.076463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:57.076468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435686473578720:2048] [node 2] ICP22 created new session: [1:7574435690768546110:2048]
2025-11-19T13:58:57.076474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435690768546110:2048] [node 2] ICS09 handshake done sender: [1:7574435686473578785:2109] self: [1:7574435686473578784:0] peer: [2:7574435693076487439:0] socket: 158 qp: -1
2025-11-19T13:58:57.076481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435690768546110:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:57.076490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:57.079969Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435693076487429:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:57.080470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435688781520060:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.080485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:57.080491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435688781520060:2048] [node 1] ICP22 created new session: [2:7574435693076487442:2048]
2025-11-19T13:58:57.080497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435693076487442:2048] [node 1] ICS09 handshake done sender: [2:7574435693076487429:2113] self: [2:7574435693076487439:0] peer: [1:7574435686473578784:0] socket: 173 qp: -1
2025-11-19T13:58:57.080504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435693076487442:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:57.080512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:57.080552Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435691238827626:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.080590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435686943860248:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.081110Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435693076487426:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.081120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435693076487426:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.084264Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435690948780571:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.084356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435686653813257:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14472) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435690948780558:2112] held: no
2025-11-19T13:58:57.084397Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435690948780559:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.086240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435691238827636:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34872
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.086894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435686382948008:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.086920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:57.086929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435686382948008:2048] [node 1] ICP22 created new session: [3:7574435690677915392:2048]
2025-11-19T13:58:57.086935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435690677915392:2048] [node 1] ICS09 handshake done sender: [3:7574435686382948067:2106] self: [3:7574435690677915384:0] peer: [1:7574435686473578786:0] socket: 161 qp: -1
2025-11-19T13:58:57.086942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435690677915392:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:57.086951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:57.087026Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435686382948008:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:57.087153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435690677915390:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8749
2025-11-19T13:58:57.087260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435690677915391:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8749
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.087645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435691238827637:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34872
2025-11-19T13:58:57.090640Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435686792590347:2088] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.092085Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435690948780558:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:57.092344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435686653813194:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.092365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:57.092372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435686653813194:2050] [node 3] ICP22 created new session: [5:7574435690948780575:2050]
2025-11-19T13:58:57.092378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435690948780575:2050] [node 3] ICS09 handshake done sender: [5:7574435690948780559:2113] self: [5:7574435690948780569:0] peer: [3:7574435686382948077:0] socket: 180 qp: -1
2025-11-19T13:58:57.092386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435690948780575:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:57.092392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:57.092700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435686653813257:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.092709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:57.092713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435686653813257:2109] [node 4] ICP22 created new session: [5:7574435690948780577:2109]
2025-11-19T13:58:57.092718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435690948780577:2109] [node 4] ICS09 handshake done sender: [5:7574435690948780558:2112] self: [5:7574435690948780570:0] peer: [4:7574435686943860313:0] socket: 169 qp: -1
2025-11-19T13:58:57.092722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435690948780577:2109] [node 4] ICS10 traffic start
2025-11-19T13:58:57.092727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:57.092848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435686653813192:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.093200Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435686473578791:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.093210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435686473578791:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.093984Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435686473578791:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.095290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435690948780582:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
2025-11-19T13:58:57.095451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435686473578783:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.095474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:57.095479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435686473578783:2108] [node 5] ICP22 created new session: [1:7574435690768546114:2108]
2025-11-19T13:58:57.095485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435690768546114:2108] [node 5] ICS09 handshake done sender: [1:7574435686473578791:2112] self: [1:7574435686473578790:0] peer: [5:7574435690948780580:0] socket: 168 qp: -1
2025-11-19T13:58:57.095491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435690768546114:2108] [node 5] ICS10 traffic start
2025-11-19T13:58:57.095499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:57.100318Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435690677915390:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.100145Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435693076487426:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:57.102142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435688781520125:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.102158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:57.102165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435688781520125:2107] [node 4] ICP22 created new session: [2:7574435693076487447:2107]
2025-11-19T13:58:57.102171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435693076487447:2107] [node 4] ICS09 handshake done sender: [2:7574435693076487426:2111] self: [2:7574435688781520127:0] peer: [4:7574435691238827635:0] socket: 190 qp: -1
2025-11-19T13:58:57.102177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435693076487447:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:57.102184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:57.102819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435693076487446:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17760
2025-11-19T13:58:57.100839Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435691238827626:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:57.100905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435686943860248:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.100920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:57.100926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435686943860248:2049] [node 2] ICP22 created new session: [4:7574435691238827638:2049]
2025-11-19T13:58:57.100932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435691238827638:2049] [node 2] ICS09 handshake done sender: [4:7574435691238827626:2119] self: [4:7574435691238827635:0] peer: [2:7574435688781520127:0] socket: 195 qp: -1
2025-11-19T13:58:57.100940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435691238827638:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:57.100949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:57.103290Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435690677915391:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.103339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435686382948009:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.103592Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435686653813192:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18744) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435690948780571:2118] held: no
2025-11-19T13:58:57.103649Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435690948780572:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.103720Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435690948780571:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.103849Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435693076487433:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.103861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435693076487433:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.104717Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435693076487446:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.106648Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435693076487433:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:57.106752Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435693076487428:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.106755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435693076487428:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.105414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435690677915397:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8749
2025-11-19T13:58:57.107339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435688781520061:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.107353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:57.105615Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435690677915380:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.107358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435688781520061:2049] [node 3] ICP22 created new session: [2:7574435693076487451:2049]
2025-11-19T13:58:57.105618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435690677915380:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.107363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435693076487451:2049] [node 3] ICS09 handshake done sender: [2:7574435693076487433:2116] self: [2:7574435693076487432:0] peer: [3:7574435690677915396:0] socket: 205 qp: -1
2025-11-19T13:58:57.107713Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435690677915380:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:57.107369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435693076487451:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:57.107378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:57.105483Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435691238827637:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.104988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435686653813192:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.105006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:57.105012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435686653813192:2048] [node 1] ICP22 created new session: [5:7574435690948780583:2048]
2025-11-19T13:58:57.105017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435690948780583:2048] [node 1] ICS09 handshake done sender: [5:7574435690948780571:2118] self: [5:7574435690948780580:0] peer: [1:7574435686473578790:0] socket: 189 qp: -1
2025-11-19T13:58:57.105023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435690948780583:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:57.105030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:57.105138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435686653813193:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.105505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435686943860249:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.106501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435691238827641:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34872
2025-11-19T13:58:57.107880Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435691238827637:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:57.107936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435686943860249:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.107946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:57.107951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435686943860249:2050] [node 3] ICP22 created new session: [4:7574435691238827642:2050]
2025-11-19T13:58:57.107959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435691238827642:2050] [node 3] ICS09 handshake done sender: [4:7574435691238827637:2121] self: [4:7574435691238827640:0] peer: [3:7574435690677915379:0] socket: 211 qp: -1
2025-11-19T13:58:57.107965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435691238827642:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:57.107971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:57.109620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435686382948076:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.109654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:57.109660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435686382948076:2110] [node 4] ICP22 created new session: [3:7574435690677915399:2110]
2025-11-19T13:58:57.109666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435690677915399:2110] [node 4] ICS09 handshake done sender: [3:7574435690677915380:2115] self: [3:7574435690677915379:0] peer: [4:7574435691238827640:0] socket: 210 qp: -1
2025-11-19T13:58:57.109671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435690677915399:2110] [node 4] ICS10 traffic start
2025-11-19T13:58:57.109680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:57.110101Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435690677915391:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:57.110260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435686382948009:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.110269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:57.110273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435686382948009:2049] [node 2] ICP22 created new session: [3:7574435690677915401:2049]
2025-11-19T13:58:57.110278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435690677915401:2049] [node 2] ICS09 handshake done sender: [3:7574435690677915391:2118] self: [3:7574435690677915396:0] peer: [2:7574435693076487432:0] socket: 172 qp: -1
2025-11-19T13:58:57.110282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435690677915401:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:57.110286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.111535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435690948780586:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26172
2025-11-19T13:58:57.111616Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435693076487428:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.112751Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435690768546099:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.112546Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435691238827623:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.111797Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435690948780572:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:57.111876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435686653813193:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.111886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:57.111891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435686653813193:2049] [node 2] ICP22 created new session: [5:7574435690948780587:2049]
2025-11-19T13:58:57.111896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435690948780587:2049] [node 2] ICS09 handshake done sender: [5:7574435690948780572:2119] self: [5:7574435690948780585:0] peer: [2:7574435693076487427:0] socket: 194 qp: -1
2025-11-19T13:58:57.111901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435690948780587:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:57.111907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:57.113128Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435690948780560:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.111942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435691087557681:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7740
2025-11-19T13:58:57.112028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435691087557682:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7740
2025-11-19T13:58:57.112228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435691087557683:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7740
2025-11-19T13:58:57.112303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435691087557684:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7740
2025-11-19T13:58:57.112345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435686792590307:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.113288Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435688781520121:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.113301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435688781520121:2106] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.113902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435688781520126:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.113915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:57.113927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435688781520126:2108] [node 5] ICP22 created new session: [2:7574435693076487454:2108]
2025-11-19T13:58:57.113934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435693076487454:2108] [node 5] ICS09 handshake done sender: [2:7574435693076487428:2112] self: [2:7574435693076487427:0] peer: [5:7574435690948780585:0] socket: 191 qp: -1
2025-11-19T13:58:57.113939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435693076487454:2108] [node 5] ICS10 traffic start
2025-11-19T13:58:57.113946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:57.116814Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435686943860323:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.116824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435686943860323:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.117627Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435686943860323:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:57.117711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435686943860321:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.117724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:57.117730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435686943860321:2113] [node 6] ICP22 created new session: [4:7574435691238827646:2113]
2025-11-19T13:58:57.117736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435691238827646:2113] [node 6] ICS09 handshake done sender: [4:7574435686943860323:2114] self: [4:7574435686943860322:0] peer: [6:7574435691087557686:0] socket: 201 qp: -1
2025-11-19T13:58:57.117741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435691238827646:2113] [node 6] ICS10 traffic start
2025-11-19T13:58:57.117749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:57.116527Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435691087557683:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.116576Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435691087557684:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.116592Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435691087557681:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.116627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435686792590350:2090] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.116637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435686792590306:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.116646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435686792590373:2092] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.119875Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435686473578798:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.119889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435686473578798:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.123234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435686792590350:2090] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14472) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435691087557683:2118] held: no
2025-11-19T13:58:57.123296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435691087557682:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.123360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435691087557689:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7740
2025-11-19T13:58:57.120996Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435690948780562:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.121008Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435690948780562:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.121947Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435690948780562:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:57.123483Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435686473578798:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:57.123592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435686473578796:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.123613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:57.123619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435686473578796:2113] [node 6] ICP22 created new session: [1:7574435690768546118:2113]
2025-11-19T13:58:57.123625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435690768546118:2113] [node 6] ICS09 handshake done sender: [1:7574435686473578798:2114] self: [1:7574435686473578797:0] peer: [6:7574435691087557687:0] socket: 176 qp: -1
2025-11-19T13:58:57.123631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435690768546118:2113] [node 6] ICS10 traffic start
2025-11-19T13:58:57.123648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.124241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435691087557690:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7740
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.128000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435691087557691:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7740
2025-11-19T13:58:57.127886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435686653813261:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.127920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:57.127929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435686653813261:2111] [node 6] ICP22 created new session: [5:7574435690948780591:2111]
2025-11-19T13:58:57.127935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435690948780591:2111] [node 6] ICS09 handshake done sender: [5:7574435690948780562:2115] self: [5:7574435690948780561:0] peer: [6:7574435691087557688:0] socket: 184 qp: -1
2025-11-19T13:58:57.127944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435690948780591:2111] [node 6] ICS10 traffic start
2025-11-19T13:58:57.127965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:57.129811Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435688781520121:2106] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:57.129900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435688781520119:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.129916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:57.130182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435686792590308:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.130280Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435691087557683:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:57.129923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435688781520119:2105] [node 6] ICP22 created new session: [2:7574435693076487457:2105]
2025-11-19T13:58:57.129933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435693076487457:2105] [node 6] ICS09 handshake done sender: [2:7574435688781520121:2106] self: [2:7574435688781520120:0] peer: [6:7574435691087557685:0] socket: 156 qp: -1
2025-11-19T13:58:57.129939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435693076487457:2105] [node 6] ICS10 traffic start
2025-11-19T13:58:57.129946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.136106Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435691087557684:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:57.136211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435686792590306:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18744) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435691087557684:2119] held: no
2025-11-19T13:58:57.136234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435686792590306:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.136246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:57.136252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435686792590306:2048] [node 1] ICP22 created new session: [6:7574435691087557694:2048]
2025-11-19T13:58:57.136268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435691087557694:2048] [node 1] ICS09 handshake done sender: [6:7574435691087557684:2119] self: [6:7574435691087557687:0] peer: [1:7574435686473578797:0] socket: 208 qp: -1
2025-11-19T13:58:57.136275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435691087557694:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:57.136283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:57.137494Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435691087557681:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:57.136000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435693076487459:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17760
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.137623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435686792590350:2090] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.137635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:57.137639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435686792590350:2090] [node 4] ICP22 created new session: [6:7574435691087557697:2090]
2025-11-19T13:58:57.137644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435691087557697:2090] [node 4] ICS09 handshake done sender: [6:7574435691087557683:2118] self: [6:7574435691087557686:0] peer: [4:7574435686943860322:0] socket: 207 qp: -1
2025-11-19T13:58:57.137649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435691087557697:2090] [node 4] ICS10 traffic start
2025-11-19T13:58:57.137655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:57.137773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435686792590373:2092] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435691087557681:2116] held: no
2025-11-19T13:58:57.137786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435686792590373:2092] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.137791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:57.137795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435686792590373:2092] [node 5] ICP22 created new session: [6:7574435691087557699:2092]
2025-11-19T13:58:57.137798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435691087557699:2092] [node 5] ICS09 handshake done sender: [6:7574435691087557681:2116] self: [6:7574435691087557688:0] peer: [5:7574435690948780561:0] socket: 203 qp: -1
2025-11-19T13:58:57.137801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435691087557699:2092] [node 5] ICS10 traffic start
2025-11-19T13:58:57.137806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:57.137940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435691087557696:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7740
2025-11-19T13:58:57.138103Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435690677915378:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.138114Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435690677915378:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.139722Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435686792590347:2088] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.141681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435686792590307:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.141699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:57.141706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435686792590307:2049] [node 2] ICP22 created new session: [6:7574435691087557703:2049]
2025-11-19T13:58:57.141712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435691087557703:2049] [node 2] ICS09 handshake done sender: [6:7574435686792590347:2088] self: [6:7574435691087557685:0] peer: [2:7574435688781520120:0] socket: 157 qp: -1
2025-11-19T13:58:57.141718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435691087557703:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:57.141724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:57.141866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435691087557702:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7740
2025-11-19T13:58:57.139198Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435690677915378:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:57.139274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435686382948075:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.139284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:57.139290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435686382948075:2109] [node 6] ICP22 created new session: [3:7574435690677915404:2109]
2025-11-19T13:58:57.139296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435690677915404:2109] [node 6] ICS09 handshake done sender: [3:7574435690677915378:2114] self: [3:7574435690677915377:0] peer: [6:7574435691087557693:0] socket: 200 qp: -1
2025-11-19T13:58:57.139301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435690677915404:2109] [node 6] ICS10 traffic start
2025-11-19T13:58:57.139308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:57.140276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435690677915406:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8749
2025-11-19T13:58:57.148918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435688781520119:2105] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:57.165231Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435691087557682:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:57.165581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435686792590308:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.165597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:57.165604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435686792590308:2050] [node 3] ICP22 created new session: [6:7574435691087557705:2050]
2025-11-19T13:58:57.165610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435691087557705:2050] [node 3] ICS09 handshake done sender: [6:7574435691087557682:2117] self: [6:7574435691087557693:0] peer: [3:7574435690677915377:0] socket: 206 qp: -1
2025-11-19T13:58:57.165616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435691087557705:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:57.165625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:57.166413Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435690677915406:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.166519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435686382948075:2109] [node 6] ICP18 handshake (actor [3:7574435690677915406:2120]) is too old
2025-11-19T13:58:57.187907Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:57.188012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435693076487442:2048] [node 1] ICS01 socket: 173 reason# EndOfStream
2025-11-19T13:58:57.188023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435688781520060:2048] [node 1] ICP30 unregister session Session# [2:7574435693076487442:2048] VirtualId# [2:7574435693076487439:0]
2025-11-19T13:58:57.188032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435693076487442:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.188096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:57.188120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:57.188471Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:57.188554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435690677915392:2048] [node 1] ICS01 socket: 161 reason# EndOfStream
2025-11-19T13:58:57.188561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435686382948008:2048] [node 1] ICP30 unregister session Session# [3:7574435690677915392:2048] VirtualId# [3:7574435690677915384:0]
2025-11-19T13:58:57.188566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435690677915392:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.188591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:57.188609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:57.199914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:57.200006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435690948780583:2048] [node 1] ICS01 socket: 189 reason# EndOfStream
2025-11-19T13:58:57.200016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435686653813192:2048] [node 1] ICP30 unregister session Session# [5:7574435690948780583:2048] VirtualId# [5:7574435690948780580:0]
2025-11-19T13:58:57.200025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435690948780583:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.200093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:57.200117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:57.200159Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:57.200216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435691087557694:2048] [node 1] ICS01 socket: 208 reason# EndOfStream
2025-11-19T13:58:57.200220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435686792590306:2048] [node 1] ICP30 unregister session Session# [6:7574435691087557694:2048] VirtualId# [6:7574435691087557687:0]
2025-11-19T13:58:57.200224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435691087557694:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.200255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:57.200278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:57.200423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:57.200494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435691238827630:2048] [node 1] ICS01 socket: 162 reason# EndOfStream
2025-11-19T13:58:57.200504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435686943860247:2048] [node 1] ICP30 unregister session Session# [4:7574435691238827630:2048] VirtualId# [4:7574435691238827622:0]
2025-11-19T13:58:57.200509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435691238827630:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.200541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:57.200561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:57.207550Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:57.207639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435690677915401:2049] [node 2] ICS01 socket: 172 reason# EndOfStream
2025-11-19T13:58:57.207648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435686382948009:2049] [node 2] ICP30 unregister session Session# [3:7574435690677915401:2049] VirtualId# [3:7574435690677915396:0]
2025-11-19T13:58:57.207654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435690677915401:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.207685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:57.207703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:57.216453Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:57.216517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435690948780587:2049] [node 2] ICS01 socket: 194 reason# EndOfStream
2025-11-19T13:58:57.216522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435686653813193:2049] [node 2] ICP30 unregister session Session# [5:7574435690948780587:2049] VirtualId# [5:7574435690948780585:0]
2025-11-19T13:58:57.216526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435690948780587:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.216557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:57.216579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:57.216682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:57.216763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435691087557703:2049] [node 2] ICS01 socket: 157 reason# EndOfStream
2025-11-19T13:58:57.216767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435686792590307:2049] [node 2] ICP30 unregister session Session# [6:7574435691087557703:2049] VirtualId# [6:7574435691087557685:0]
2025-11-19T13:58:57.216785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435691087557703:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.216820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:57.216844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:57.216207Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:57.216309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435691238827638:2049] [node 2] ICS01 socket: 195 reason# EndOfStream
2025-11-19T13:58:57.216319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435686943860248:2049] [node 2] ICP30 unregister session Session# [4:7574435691238827638:2049] VirtualId# [4:7574435691238827635:0]
2025-11-19T13:58:57.216325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435691238827638:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.216386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:57.216411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:57.228742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:57.228841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435691087557705:2050] [node 3] ICS01 socket: 206 reason# EndOfStream
2025-11-19T13:58:57.228850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435686792590308:2050] [node 3] ICP30 unregister session Session# [6:7574435691087557705:2050] VirtualId# [6:7574435691087557693:0]
2025-11-19T13:58:57.228857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435691087557705:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.228919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:57.228945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:57.229036Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:57.229098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435690948780575:2050] [node 3] ICS01 socket: 180 reason# EndOfStream
2025-11-19T13:58:57.229104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435686653813194:2050] [node 3] ICP30 unregister session Session# [5:7574435690948780575:2050] VirtualId# [5:7574435690948780569:0]
2025-11-19T13:58:57.229109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435690948780575:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.229134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:57.229155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:57.242681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:57.242774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435691087557697:2090] [node 4] ICS01 socket: 207 reason# EndOfStream
2025-11-19T13:58:57.242785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435686792590350:2090] [node 4] ICP30 unregister session Session# [6:7574435691087557697:2090] VirtualId# [6:7574435691087557686:0]
2025-11-19T13:58:57.242793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435691087557697:2090] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.242856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:57.242881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:57.504024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435693637446478:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:57.504041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435693637446479:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:57.563849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435690383063092:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:57.563862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435690383063093:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:57.619941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435690543672239:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:57.619953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435690543672240:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:57.635904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435690471681532:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:57.635917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435690471681533:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:57.635921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435690471681534:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:57.659170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435693346556565:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:57.659186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435693346556566:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:57.659192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435693346556567:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:57.691917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435692348003221:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:57.691952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435692348003222:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:57.691957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435692348003223:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:57.706929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435693637446539:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.707507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435693346556599:2080] [node 4] ICP01 ready to work
2025-11-19T13:58:57.708324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435693346556603:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50269
2025-11-19T13:58:57.709103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435690471681561:2073] [node 5] ICP01 ready to work
2025-11-19T13:58:57.708690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435690543672273:2076] [node 6] ICP01 ready to work
2025-11-19T13:58:57.708775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435690543672274:2078] [node 4] ICP01 ready to work
2025-11-19T13:58:57.709384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435690543672282:2080] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.711023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435693637446545:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49713
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.710702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435690383063153:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:57.711241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435690383063154:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37999
2025-11-19T13:58:57.711548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435692348003239:2062] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.714989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435690543672295:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57206
2025-11-19T13:58:57.712323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435693637446548:2111] [node 6] ICP01 ready to work
2025-11-19T13:58:57.715953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435693346556616:2088] [node 6] ICP01 ready to work
2025-11-19T13:58:57.717209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435690383063162:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.719742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435690543672301:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57206
2025-11-19T13:58:57.720200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435693637446553:2113] [node 4] ICP01 ready to work
2025-11-19T13:58:57.723919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435692348003234:2060] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45112
2025-11-19T13:58:57.724432Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435690543672301:2098] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.724469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435690543672240:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.732016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435693637446554:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49713
2025-11-19T13:58:57.732487Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435690543672295:2092] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.732521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435690543672239:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.732251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435690383063169:2114] [node 6] ICP01 ready to work
2025-11-19T13:58:57.734533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435692348003235:2059] [node 5] ICP01 ready to work
2025-11-19T13:58:57.733617Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435693346556603:2083] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.734040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435693346556565:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.734573Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435690383063164:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.734588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435690383063164:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.742801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435690543672322:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57206
2025-11-19T13:58:57.740301Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435693637446544:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.740323Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435693637446544:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.740882Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435693637446541:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.740885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435693637446541:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.741618Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435693637446554:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.741871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435693637446558:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49713
2025-11-19T13:58:57.743543Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435692348003239:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19561470ns) {Configure:104(19559273ns) {ProcessConfigured:135(19138634ns) {StartInitialHandshake:157(19137444ns) {DropHandshakes:469(875ns) {DropIncomingHandshake:441(134ns) DropOutgoingHandshake:456(181ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:476(964ns) {SwitchToState:178(206ns)}}}}}
2025-11-19T13:58:57.741958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435693346556566:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(25594219ns) {Configure:104(25593603ns) {ProcessConfigured:135(25369962ns) {StartInitialHandshake:157(25369433ns) {DropHandshakes:469(498ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(39ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(867ns) {SwitchToState:178(741ns)}}}}}
No RDMA context found for GID: 2025-11-19T13:58:57.743839Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435690383063154:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.743559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435690471681607:2079] [node 6] ICP01 ready to work
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.744174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435693346556644:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50269
2025-11-19T13:58:57.744159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435690471681604:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14670
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.744261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435690471681605:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14670
2025-11-19T13:58:57.744281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435690471681606:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14670
2025-11-19T13:58:57.748053Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435693637446544:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:57.744593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435692348003288:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45112
2025-11-19T13:58:57.744642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435692348003289:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45112
2025-11-19T13:58:57.746931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435692348003289:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.746981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435692348003235:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.753416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435692348003221:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18812465ns) {Configure:104(18811427ns) {ProcessConfigured:135(18457509ns) {StartInitialHandshake:157(18456837ns) {DropHandshakes:469(518ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:476(836ns) {SwitchToState:178(243ns)}}}}}
2025-11-19T13:58:57.753531Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435692348003234:2060] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.753572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435692348003223:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.756144Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435690383063164:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:57.756243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435690383063093:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.756262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:57.756272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435690383063093:2049] [node 3] ICP22 created new session: [2:7574435690383063174:2049]
2025-11-19T13:58:57.756280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435690383063174:2049] [node 3] ICS09 handshake done sender: [2:7574435690383063164:2112] self: [2:7574435690383063163:0] peer: [3:7574435690543672311:0] socket: 177 qp: -1
2025-11-19T13:58:57.756303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435690383063174:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:57.756315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.760597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435693346556646:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50269
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.761277Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435693637446541:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:57.761412Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435690471681561:2073] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(49599238ns) {Configure:104(49596801ns) {ProcessConfigured:135(49232551ns) {StartInitialHandshake:157(49231796ns) {DropHandshakes:469(501ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:476(1589ns) {SwitchToState:178(781ns)}}}}}
2025-11-19T13:58:57.761643Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435690471681604:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.761721Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435690471681606:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.761811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435690471681533:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.761829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435690471681613:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14670
2025-11-19T13:58:57.762351Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435690543672278:2081] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.762364Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435690543672278:2081] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.762906Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435690543672295:2092] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:57.763362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435693637446479:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.763382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:57.763392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435693637446479:2049] [node 3] ICP22 created new session: [1:7574435693637446562:2049]
2025-11-19T13:58:57.763398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435693637446562:2049] [node 3] ICS09 handshake done sender: [1:7574435693637446544:2109] self: [1:7574435693637446543:0] peer: [3:7574435690543672320:0] socket: 172 qp: -1
2025-11-19T13:58:57.763406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435693637446562:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:57.763415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:57.763557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435693637446539:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.763563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:57.763571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435693637446539:2107] [node 5] ICP22 created new session: [1:7574435693637446564:2107]
2025-11-19T13:58:57.763575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435693637446564:2107] [node 5] ICS09 handshake done sender: [1:7574435693637446541:2108] self: [1:7574435693637446540:0] peer: [5:7574435693346556640:0] socket: 162 qp: -1
2025-11-19T13:58:57.763580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435693637446564:2107] [node 5] ICS10 traffic start
2025-11-19T13:58:57.763584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:57.764255Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435693346556620:2096] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.764272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435693346556620:2096] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.763918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435692348003288:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.763967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435692348003221:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.768617Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435693637446553:2113] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(43690939ns) {Configure:104(43688914ns) {ProcessConfigured:135(43368403ns) {StartInitialHandshake:157(43367794ns) {DropHandshakes:469(494ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(316ns) PrepareNewSessionHandshake:476(1972ns) {SwitchToState:178(1199ns)}}}}}
2025-11-19T13:58:57.768477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435690543672323:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57206
2025-11-19T13:58:57.768508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435690543672324:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57206
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.772107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435690471681616:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14670
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.772770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435690543672239:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.772794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:57.772803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435690543672239:2048] [node 1] ICP22 created new session: [3:7574435690543672327:2048]
2025-11-19T13:58:57.772811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435690543672327:2048] [node 1] ICS09 handshake done sender: [3:7574435690543672295:2092] self: [3:7574435690543672320:0] peer: [1:7574435693637446543:0] socket: 175 qp: -1
2025-11-19T13:58:57.772821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435690543672327:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:57.772845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:57.773153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435690543672325:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57206
2025-11-19T13:58:57.773281Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435690543672301:2098] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:57.773801Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435690543672278:2081] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:57.772186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435692348003305:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45112
2025-11-19T13:58:57.774021Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435692348003234:2060] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:57.774090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435692348003223:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.774104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:57.774111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435692348003223:2050] [node 3] ICP22 created new session: [6:7574435692348003306:2050]
2025-11-19T13:58:57.774118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435692348003306:2050] [node 3] ICS09 handshake done sender: [6:7574435692348003234:2060] self: [6:7574435692348003303:0] peer: [3:7574435690543672277:0] socket: 169 qp: -1
2025-11-19T13:58:57.774124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435692348003306:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:57.774142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:57.777200Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435690471681605:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.777238Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435690471681616:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.777273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435690471681534:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.777285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435690471681532:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.778420Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435693346556603:2083] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:57.778718Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435693637446550:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.778734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435693637446550:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.778835Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435693637446567:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.778843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435693637446567:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.784037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435690543672240:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.784070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:57.784081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435690543672240:2049] [node 2] ICP22 created new session: [3:7574435690543672329:2049]
2025-11-19T13:58:57.784090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435690543672329:2049] [node 2] ICS09 handshake done sender: [3:7574435690543672301:2098] self: [3:7574435690543672311:0] peer: [2:7574435690383063163:0] socket: 178 qp: -1
2025-11-19T13:58:57.784103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435690543672329:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:57.784114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:57.784224Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435690543672240:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:57.785700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435690543672273:2076] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.785714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:57.785721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435690543672273:2076] [node 6] ICP22 created new session: [3:7574435690543672331:2076]
2025-11-19T13:58:57.785727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435690543672331:2076] [node 6] ICS09 handshake done sender: [3:7574435690543672278:2081] self: [3:7574435690543672277:0] peer: [6:7574435692348003303:0] socket: 167 qp: -1
2025-11-19T13:58:57.785735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435690543672331:2076] [node 6] ICS10 traffic start
2025-11-19T13:58:57.785742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:57.785928Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435690543672280:2082] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.785932Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435690543672280:2082] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.788993Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435690543672280:2082] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.784653Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435690383063156:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.784669Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435690383063156:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.788374Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435690383063093:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(32094765ns) {HandleHandshakeStatus:329(32093450ns) {DropIncomingHandshake:441(4897ns) DropOutgoingHandshake:456(276ns) SwitchToState:178(100ns) ProcessPendingSessionEvents:495(23969455ns) {ForwardSessionEventToSession:612(23954427ns) ForwardSessionEventToSession:612(3029ns) ForwardSessionEventToSession:612(1080ns) ForwardSessionEventToSession:612(755ns) ForwardSessionEventToSession:612(519ns) ForwardSessionEventToSession:612(1248ns) ForwardSessionEventToSession:612(560ns) ForwardSessionEventToSession:612(564ns) ForwardSessionEventToSession:612(708ns) ForwardSessionEventToSession:612(1051ns)}}}
2025-11-19T13:58:57.788753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435690383063176:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37999
2025-11-19T13:58:57.788769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435690383063177:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37999
2025-11-19T13:58:57.788815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435690383063178:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37999
2025-11-19T13:58:57.789084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435690471681619:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14670
2025-11-19T13:58:57.790689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435690543672274:2078] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.790706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:57.790714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435690543672274:2078] [node 4] ICP22 created new session: [3:7574435690543672335:2078]
2025-11-19T13:58:57.790720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435690543672335:2078] [node 4] ICS09 handshake done sender: [3:7574435690543672280:2082] self: [3:7574435690543672279:0] peer: [4:7574435690471681617:0] socket: 168 qp: -1
2025-11-19T13:58:57.790726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435690543672335:2078] [node 4] ICS10 traffic start
2025-11-19T13:58:57.790735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:57.789257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435693346556565:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.789283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:57.789291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435693346556565:2048] [node 1] ICP22 created new session: [5:7574435693346556648:2048]
2025-11-19T13:58:57.789298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435693346556648:2048] [node 1] ICS09 handshake done sender: [5:7574435693346556603:2083] self: [5:7574435693346556640:0] peer: [1:7574435693637446540:0] socket: 163 qp: -1
2025-11-19T13:58:57.789306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435693346556648:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:57.789313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:57.792642Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435693637446558:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.792818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435693637446539:2107] [node 5] ICP18 handshake (actor [1:7574435693637446558:2116]) is too old
2025-11-19T13:58:57.793686Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435693637446550:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:57.793861Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435693346556620:2096] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.794517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435693346556616:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.794530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:57.794536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435693346556616:2088] [node 6] ICP22 created new session: [5:7574435693346556652:2088]
2025-11-19T13:58:57.794541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435693346556652:2088] [node 6] ICS09 handshake done sender: [5:7574435693346556620:2096] self: [5:7574435693346556619:0] peer: [6:7574435692348003300:0] socket: 182 qp: -1
2025-11-19T13:58:57.794546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435693346556652:2088] [node 6] ICS10 traffic start
2025-11-19T13:58:57.794552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:57.794680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435693346556650:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50269
2025-11-19T13:58:57.794692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435693346556651:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50269
2025-11-19T13:58:57.794973Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435690383063177:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.795019Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435690383063156:2109] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.795158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435690383063153:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.795171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:57.795176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435690383063153:2107] [node 4] ICP22 created new session: [2:7574435690383063182:2107]
2025-11-19T13:58:57.795180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435690383063182:2107] [node 4] ICS09 handshake done sender: [2:7574435690383063156:2109] self: [2:7574435690383063155:0] peer: [4:7574435690471681615:0] socket: 173 qp: -1
2025-11-19T13:58:57.795185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435690383063182:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:57.795190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:57.795324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435690383063092:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.795348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435690383063181:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37999
2025-11-19T13:58:57.795720Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435693637446557:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.795727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435693637446557:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.796346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435693637446548:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.796355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:57.796360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435693637446548:2111] [node 6] ICP22 created new session: [1:7574435693637446571:2111]
2025-11-19T13:58:57.796365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435693637446571:2111] [node 6] ICS09 handshake done sender: [1:7574435693637446550:2112] self: [1:7574435693637446549:0] peer: [6:7574435692348003304:0] socket: 174 qp: -1
2025-11-19T13:58:57.796372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435693637446571:2111] [node 6] ICS10 traffic start
2025-11-19T13:58:57.796380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.797578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435690543672337:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57206
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.800429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435690383063186:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37999
2025-11-19T13:58:57.800644Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435693346556651:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.800690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435693346556566:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.801072Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435690543672337:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.801214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435692348003309:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45112
2025-11-19T13:58:57.801076Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435690471681605:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:57.800948Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435693637446567:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:57.801360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435690471681534:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.801373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:57.801378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435690471681534:2050] [node 3] ICP22 created new session: [4:7574435690471681622:2050]
2025-11-19T13:58:57.801384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435690471681622:2050] [node 3] ICS09 handshake done sender: [4:7574435690471681605:2114] self: [4:7574435690471681617:0] peer: [3:7574435690543672279:0] socket: 187 qp: -1
2025-11-19T13:58:57.801389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435690471681622:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:57.801408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:57.801501Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435690383063181:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.801368Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435690543672287:2087] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.801375Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435690543672287:2087] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.801239Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435693346556650:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.801267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435693346556567:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.802133Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435690543672287:2087] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:57.802241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435690543672282:2080] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.802259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:57.802265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435690543672282:2080] [node 5] ICP22 created new session: [3:7574435690543672340:2080]
2025-11-19T13:58:57.802272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435690543672340:2080] [node 5] ICS09 handshake done sender: [3:7574435690543672287:2087] self: [3:7574435690543672284:0] peer: [5:7574435693346556657:0] socket: 196 qp: -1
2025-11-19T13:58:57.802278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435690543672340:2080] [node 5] ICS10 traffic start
2025-11-19T13:58:57.802285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.802631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435693346556658:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50269
2025-11-19T13:58:57.803994Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435693637446557:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:57.804103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435693637446553:2113] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.804118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:57.804124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435693637446553:2113] [node 4] ICP22 created new session: [1:7574435693637446574:2113]
2025-11-19T13:58:57.804129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435693637446574:2113] [node 4] ICS09 handshake done sender: [1:7574435693637446567:2117] self: [1:7574435693637446555:0] peer: [4:7574435690471681618:0] socket: 205 qp: -1
2025-11-19T13:58:57.804134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435693637446574:2113] [node 4] ICS10 traffic start
2025-11-19T13:58:57.804141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:57.804350Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435693346556567:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30431) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435693346556650:2118] held: no
2025-11-19T13:58:57.804377Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435693346556650:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:57.804477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435693346556567:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.804491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:57.804495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435693346556567:2050] [node 3] ICP22 created new session: [5:7574435693346556659:2050]
2025-11-19T13:58:57.804500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435693346556659:2050] [node 3] ICS09 handshake done sender: [5:7574435693346556650:2118] self: [5:7574435693346556657:0] peer: [3:7574435690543672284:0] socket: 214 qp: -1
2025-11-19T13:58:57.804506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435693346556659:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:57.804512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:57.809135Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435690383063166:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.809152Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435690383063166:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.811997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435693637446478:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.812024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:57.812031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435693637446478:2048] [node 2] ICP22 created new session: [1:7574435693637446576:2048]
2025-11-19T13:58:57.812037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435693637446576:2048] [node 2] ICS09 handshake done sender: [1:7574435693637446557:2115] self: [1:7574435693637446556:0] peer: [2:7574435690383063184:0] socket: 203 qp: -1
2025-11-19T13:58:57.812044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435693637446576:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:57.812052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.819971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435690471681534:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(18558482ns) {HandleHandshakeStatus:329(18557601ns) {DropIncomingHandshake:441(196ns) DropOutgoingHandshake:456(1459ns) SwitchToState:178(57ns) ProcessPendingSessionEvents:495(10883189ns) {ForwardSessionEventToSession:612(10866448ns) ForwardSessionEventToSession:612(2159ns) ForwardSessionEventToSession:612(1473ns) ForwardSessionEventToSession:612(789ns) ForwardSessionEventToSession:612(527ns) ForwardSessionEventToSession:612(1281ns) ForwardSessionEventToSession:612(846ns) ForwardSessionEventToSession:612(494ns) ForwardSessionEventToSession:612(784ns) ForwardSessionEventToSession:612(927ns)}}}
2025-11-19T13:58:57.820412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435690471681620:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14670
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.821076Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435690383063177:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:57.821594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435690383063092:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.821611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:57.821623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435690383063092:2048] [node 1] ICP22 created new session: [2:7574435690383063188:2048]
2025-11-19T13:58:57.821630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435690383063188:2048] [node 1] ICS09 handshake done sender: [2:7574435690383063177:2117] self: [2:7574435690383063184:0] peer: [1:7574435693637446556:0] socket: 208 qp: -1
2025-11-19T13:58:57.821639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435690383063188:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:57.821648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:57.820623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435692348003310:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45112
2025-11-19T13:58:57.821293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435692348003311:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45112
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.825382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435690471681621:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14670
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.828070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435693346556661:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50269
2025-11-19T13:58:57.828495Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435690471681606:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:57.829355Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435692348003289:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:57.828787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435690471681533:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.828809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:57.828817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435690471681533:2049] [node 2] ICP22 created new session: [4:7574435690471681625:2049]
2025-11-19T13:58:57.828823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435690471681625:2049] [node 2] ICS09 handshake done sender: [4:7574435690471681606:2115] self: [4:7574435690471681615:0] peer: [2:7574435690383063155:0] socket: 188 qp: -1
2025-11-19T13:58:57.828831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435690471681625:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:57.828838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:57.829134Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435690471681616:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:57.829340Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435690383063166:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:57.829805Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435692348003311:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.829827Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435693346556651:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:57.830008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435690383063162:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.830028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:57.830034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435690383063162:2111] [node 5] ICP22 created new session: [2:7574435690383063191:2111]
2025-11-19T13:58:57.830039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435690383063191:2111] [node 5] ICS09 handshake done sender: [2:7574435690383063166:2113] self: [2:7574435690383063165:0] peer: [5:7574435693346556656:0] socket: 211 qp: -1
2025-11-19T13:58:57.830045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435690383063191:2111] [node 5] ICS10 traffic start
2025-11-19T13:58:57.830055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:57.830428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435693346556566:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.830445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:57.830451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435693346556566:2049] [node 2] ICP22 created new session: [5:7574435693346556662:2049]
2025-11-19T13:58:57.830456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435693346556662:2049] [node 2] ICS09 handshake done sender: [5:7574435693346556651:2119] self: [5:7574435693346556656:0] peer: [2:7574435690383063165:0] socket: 215 qp: -1
2025-11-19T13:58:57.830462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435693346556662:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:57.830469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:57.832052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435693637446545:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.832096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435693637446553:2113] [node 4] ICP18 handshake (actor [1:7574435693637446545:2110]) is too old
2025-11-19T13:58:57.837534Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435692348003288:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:57.837750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435692348003235:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.837764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:57.837772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435692348003235:2059] [node 5] ICP22 created new session: [6:7574435692348003313:2059]
2025-11-19T13:58:57.837779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435692348003313:2059] [node 5] ICS09 handshake done sender: [6:7574435692348003289:2111] self: [6:7574435692348003300:0] peer: [5:7574435693346556619:0] socket: 190 qp: -1
2025-11-19T13:58:57.837785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435692348003313:2059] [node 5] ICS10 traffic start
2025-11-19T13:58:57.837795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:57.837983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435692348003221:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.837990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:57.837997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435692348003221:2048] [node 1] ICP22 created new session: [6:7574435692348003315:2048]
2025-11-19T13:58:57.838001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435692348003315:2048] [node 1] ICS09 handshake done sender: [6:7574435692348003288:2110] self: [6:7574435692348003304:0] peer: [1:7574435693637446549:0] socket: 189 qp: -1
2025-11-19T13:58:57.838006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435692348003315:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:57.838012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:57.839330Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435693346556646:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.839460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435693346556616:2088] [node 6] ICP18 handshake (actor [5:7574435693346556646:2117]) is too old
2025-11-19T13:58:57.840258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435690471681532:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.840280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:57.840286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435690471681532:2048] [node 1] ICP22 created new session: [4:7574435690471681627:2048]
2025-11-19T13:58:57.840293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435690471681627:2048] [node 1] ICS09 handshake done sender: [4:7574435690471681616:2119] self: [4:7574435690471681618:0] peer: [1:7574435693637446555:0] socket: 206 qp: -1
2025-11-19T13:58:57.840299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435690471681627:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:57.840306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:57.840378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435690471681532:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:57.840525Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435690471681613:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.842534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435693346556664:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50269
2025-11-19T13:58:57.843226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435692348003222:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.843388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435692348003312:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45112
2025-11-19T13:58:57.844625Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435690383063171:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.844639Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435690383063171:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.847150Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435690383063171:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:57.847450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435690383063169:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.847466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:57.847473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435690383063169:2114] [node 6] ICP22 created new session: [2:7574435690383063194:2114]
2025-11-19T13:58:57.847480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435690383063194:2114] [node 6] ICS09 handshake done sender: [2:7574435690383063171:2115] self: [2:7574435690383063170:0] peer: [6:7574435692348003317:0] socket: 220 qp: -1
2025-11-19T13:58:57.847487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435690383063194:2114] [node 6] ICS10 traffic start
2025-11-19T13:58:57.847497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:57.847615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435690383063153:2107] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.847908Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435692348003312:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.847965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435692348003318:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45112
2025-11-19T13:58:57.848050Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435690383063178:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.848074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435690383063169:2114] [node 6] ICP18 handshake (actor [2:7574435690383063178:2118]) is too old
2025-11-19T13:58:57.848431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435692348003239:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:57.849236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435692348003222:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28564) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435692348003311:2121] held: no
2025-11-19T13:58:57.849278Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435692348003311:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:57.849427Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435690471681609:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.849439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435690471681609:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.850380Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435693346556664:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.850433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435693346556599:2080] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.850652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435692348003222:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.850671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:57.850678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435692348003222:2049] [node 2] ICP22 created new session: [6:7574435692348003321:2049]
2025-11-19T13:58:57.850684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435692348003321:2049] [node 2] ICS09 handshake done sender: [6:7574435692348003311:2121] self: [6:7574435692348003317:0] peer: [2:7574435690383063170:0] socket: 227 qp: -1
2025-11-19T13:58:57.850690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435692348003321:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:57.850697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:57.850913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435692348003320:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45112
2025-11-19T13:58:57.851286Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435690471681609:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:57.851472Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435692348003312:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:57.851623Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435690471681612:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:57.851630Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435690471681612:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:57.851855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435692348003239:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.851872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:57.851877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435692348003239:2062] [node 4] ICP22 created new session: [6:7574435692348003323:2062]
2025-11-19T13:58:57.851883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435692348003323:2062] [node 4] ICS09 handshake done sender: [6:7574435692348003312:2122] self: [6:7574435692348003319:0] peer: [4:7574435690471681608:0] socket: 202 qp: -1
2025-11-19T13:58:57.851888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435692348003323:2062] [node 4] ICS10 traffic start
2025-11-19T13:58:57.851896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:57.852606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435690471681607:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.852623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:57.852629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435690471681607:2079] [node 6] ICP22 created new session: [4:7574435690471681632:2079]
2025-11-19T13:58:57.852635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435690471681632:2079] [node 6] ICS09 handshake done sender: [4:7574435690471681609:2116] self: [4:7574435690471681608:0] peer: [6:7574435692348003319:0] socket: 179 qp: -1
2025-11-19T13:58:57.852640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435690471681632:2079] [node 6] ICS10 traffic start
2025-11-19T13:58:57.852649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:57.854690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435693346556666:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50269
2025-11-19T13:58:57.854835Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435690471681612:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:57.855048Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435693346556664:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:57.855156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435693346556599:2080] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:57.855171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:57.855177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435693346556599:2080] [node 4] ICP22 created new session: [5:7574435693346556667:2080]
2025-11-19T13:58:57.855182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435693346556667:2080] [node 4] ICS09 handshake done sender: [5:7574435693346556664:2122] self: [5:7574435693346556665:0] peer: [4:7574435690471681572:0] socket: 171 qp: -1
2025-11-19T13:58:57.855188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435693346556667:2080] [node 4] ICS10 traffic start
2025-11-19T13:58:57.855195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:57.856600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435690471681561:2073] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:57.856618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:57.856624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435690471681561:2073] [node 5] ICP22 created new session: [4:7574435690471681635:2073]
2025-11-19T13:58:57.856630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435690471681635:2073] [node 5] ICS09 handshake done sender: [4:7574435690471681612:2117] self: [4:7574435690471681572:0] peer: [5:7574435693346556665:0] socket: 166 qp: -1
2025-11-19T13:58:57.856635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435690471681635:2073] [node 5] ICS10 traffic start
2025-11-19T13:58:57.856643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:57.883897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:57.884002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435690383063188:2048] [node 1] ICS01 socket: 208 reason# EndOfStream
2025-11-19T13:58:57.884014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435690383063092:2048] [node 1] ICP30 unregister session Session# [2:7574435690383063188:2048] VirtualId# [2:7574435690383063184:0]
2025-11-19T13:58:57.884023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435690383063188:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.884080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:57.884112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:57.884241Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:57.884309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435690543672327:2048] [node 1] ICS01 socket: 175 reason# EndOfStream
2025-11-19T13:58:57.884314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435690543672239:2048] [node 1] ICP30 unregister session Session# [3:7574435690543672327:2048] VirtualId# [3:7574435690543672320:0]
2025-11-19T13:58:57.884319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435690543672327:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.884354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:57.884380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:57.891938Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:57.892040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435693346556648:2048] [node 1] ICS01 socket: 163 reason# EndOfStream
2025-11-19T13:58:57.892050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435693346556565:2048] [node 1] ICP30 unregister session Session# [5:7574435693346556648:2048] VirtualId# [5:7574435693346556640:0]
2025-11-19T13:58:57.892059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435693346556648:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.892138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:57.892167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:57.892182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:57.892255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435690471681627:2048] [node 1] ICS01 socket: 206 reason# EndOfStream
2025-11-19T13:58:57.892262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435690471681532:2048] [node 1] ICP30 unregister session Session# [4:7574435690471681627:2048] VirtualId# [4:7574435690471681618:0]
2025-11-19T13:58:57.892267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435690471681627:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.892316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:57.892341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:57.892396Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:57.892469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435692348003315:2048] [node 1] ICS01 socket: 189 reason# EndOfStream
2025-11-19T13:58:57.892475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435692348003221:2048] [node 1] ICP30 unregister session Session# [6:7574435692348003315:2048] VirtualId# [6:7574435692348003304:0]
2025-11-19T13:58:57.892479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435692348003315:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.892509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:57.892529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:57.895002Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:57.895094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435690543672329:2049] [node 2] ICS01 socket: 178 reason# EndOfStream
2025-11-19T13:58:57.895102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435690543672240:2049] [node 2] ICP30 unregister session Session# [3:7574435690543672329:2049] VirtualId# [3:7574435690543672311:0]
2025-11-19T13:58:57.895110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435690543672329:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.895175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:57.895202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:57.899785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:57.899863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435690471681625:2049] [node 2] ICS01 socket: 188 reason# EndOfStream
2025-11-19T13:58:57.899870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435690471681533:2049] [node 2] ICP30 unregister session Session# [4:7574435690471681625:2049] VirtualId# [4:7574435690471681615:0]
2025-11-19T13:58:57.899875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435690471681625:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.899924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:57.899950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:57.899984Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:57.900047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435693346556662:2049] [node 2] ICS01 socket: 215 reason# EndOfStream
2025-11-19T13:58:57.900052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435693346556566:2049] [node 2] ICP30 unregister session Session# [5:7574435693346556662:2049] VirtualId# [5:7574435693346556656:0]
2025-11-19T13:58:57.900056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435693346556662:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.900088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:57.900111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:57.898401Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:57.898502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435692348003321:2049] [node 2] ICS01 socket: 227 reason# EndOfStream
2025-11-19T13:58:57.898510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435692348003222:2049] [node 2] ICP30 unregister session Session# [6:7574435692348003321:2049] VirtualId# [6:7574435692348003317:0]
2025-11-19T13:58:57.898516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435692348003321:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.898564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:57.898587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:57.912146Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:57.912229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435693346556659:2050] [node 3] ICS01 socket: 214 reason# EndOfStream
2025-11-19T13:58:57.912237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435693346556567:2050] [node 3] ICP30 unregister session Session# [5:7574435693346556659:2050] VirtualId# [5:7574435693346556657:0]
2025-11-19T13:58:57.912242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435693346556659:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.912295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:57.912317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:57.911797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:57.911897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435692348003306:2050] [node 3] ICS01 socket: 169 reason# EndOfStream
2025-11-19T13:58:57.911907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435692348003223:2050] [node 3] ICP30 unregister session Session# [6:7574435692348003306:2050] VirtualId# [6:7574435692348003303:0]
2025-11-19T13:58:57.911915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435692348003306:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.911967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:57.911993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:57.932730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T13:58:57.932825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435693346556667:2080] [node 4] ICS01 socket: 171 reason# EndOfStream
2025-11-19T13:58:57.932834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435693346556599:2080] [node 4] ICP30 unregister session Session# [5:7574435693346556667:2080] VirtualId# [5:7574435693346556665:0]
2025-11-19T13:58:57.932842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435693346556667:2080] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.932902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:57.932925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:58:57.934472Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:57.934554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435692348003323:2062] [node 4] ICS07 socket disconnect 202 reason# EndOfStream
2025-11-19T13:58:57.934559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435692348003323:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:57.934606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:57.934624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435692348003323:2062] [node 4] ICS15 start handshake
2025-11-19T13:58:57.935054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435692348003239:2062] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20025) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:57.935067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435692348003323:2062] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:57.935071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435692348003239:2062] [node 4] ICP30 unregister session Session# [6:7574435692348003323:2062] VirtualId# [6:7574435692348003319:0]
2025-11-19T13:58:57.935086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:58:57.935091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435692348003239:2062] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20025) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:57.935093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20025) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:58.123874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435698461318827:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:58.123892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435698461318828:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:58.169735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435695409840769:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:58.169750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435695409840770:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:58.210542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435698118135695:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:58.210569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435698118135696:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:58.273162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435696950411307:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:58.273174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435696950411308:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:58.273179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435696950411309:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:58.334960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435695600720152:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:58.334975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435695600720153:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:58.334980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435695600720154:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:58.371884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435695044076822:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:58.371899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435695044076823:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:58.371904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435695044076824:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:58.411434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435698461318888:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:58.412013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435695600720186:2080] [node 6] ICP01 ready to work
2025-11-19T13:58:58.412641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435695409840830:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:58.415276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435695044076853:2064] [node 5] ICP01 ready to work
2025-11-19T13:58:58.416788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435695044076846:2062] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.424974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435696950411368:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2591
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.426244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435698118135755:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-19T13:58:58.426266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435698118135756:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-19T13:58:58.426481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435698461318895:2108] [node 6] ICP01 ready to work
2025-11-19T13:58:58.426651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435698461318896:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:58.428060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435698461318897:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
2025-11-19T13:58:58.431831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435698461318827:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20165756ns) {Configure:104(20161967ns) {ProcessConfigured:135(16831964ns) {StartInitialHandshake:157(16831535ns) {DropHandshakes:469(382ns) {DropIncomingHandshake:441(33ns) DropOutgoingHandshake:456(112ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:476(1074ns) {SwitchToState:178(113ns)}}}}}
2025-11-19T13:58:58.431672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435695600720216:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49699
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.433057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435696950411372:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2591
2025-11-19T13:58:58.432137Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435698461318828:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(20163989ns) {Configure:104(20163371ns) {ProcessConfigured:135(17022834ns) {StartInitialHandshake:157(17022233ns) {DropHandshakes:469(259ns) {DropIncomingHandshake:441(25ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:476(547ns) {SwitchToState:178(402ns)}}}}}
2025-11-19T13:58:58.435989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435698118135759:2109] [node 4] ICP01 ready to work
2025-11-19T13:58:58.436077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435698118135760:2110] [node 6] ICP01 ready to work
2025-11-19T13:58:58.437046Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435696950411372:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.439585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435695600720192:2082] [node 4] ICP01 ready to work
2025-11-19T13:58:58.442080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435696950411378:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.441506Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435695600720186:2080] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(29429301ns) {Configure:104(29428830ns) {ProcessConfigured:135(29023075ns) {StartInitialHandshake:157(29022622ns) {DropHandshakes:469(869ns) {DropIncomingHandshake:441(164ns) DropOutgoingHandshake:456(133ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(1119ns) {SwitchToState:178(598ns)}}}}}
2025-11-19T13:58:58.441745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435695044076846:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24853812ns) {Configure:104(24853202ns) {ProcessConfigured:135(24588194ns) {StartInitialHandshake:157(24587632ns) {DropHandshakes:469(467ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:476(825ns) {SwitchToState:178(592ns)}}}}}
2025-11-19T13:58:58.444793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435695044076897:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29271
2025-11-19T13:58:58.444833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435695044076898:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29271
2025-11-19T13:58:58.445573Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435698461318894:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.445589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435698461318894:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.452539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435698118135766:2111] [node 5] ICP01 ready to work
2025-11-19T13:58:58.453158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435698118135767:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-19T13:58:58.453170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435698118135768:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-19T13:58:58.453443Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435698118135696:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17325208ns) {Configure:104(17324368ns) {ProcessConfigured:135(16931903ns) {StartInitialHandshake:157(16931537ns) {DropHandshakes:469(1282ns) {DropIncomingHandshake:441(565ns) DropOutgoingHandshake:456(164ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(499ns) {SwitchToState:178(64ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-19T13:58:58.456201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435695600720225:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49699
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.456230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435698461318907:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
2025-11-19T13:58:58.456622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435698461318909:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.457317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435695409840845:2112] [node 5] ICP01 ready to work
2025-11-19T13:58:58.457755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435695409840843:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30817
2025-11-19T13:58:58.457817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435695409840844:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30817
2025-11-19T13:58:58.456437Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435698118135756:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.456723Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435698118135760:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20061435ns) {Configure:104(20059801ns) {ProcessConfigured:135(19789606ns) {StartInitialHandshake:157(19788832ns) {DropHandshakes:469(672ns) {DropIncomingHandshake:441(142ns) DropOutgoingHandshake:456(100ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:476(872ns) {SwitchToState:178(215ns)}}}}}
2025-11-19T13:58:58.457919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435695600720192:2082] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18262179ns) {Configure:104(18261733ns) {ProcessConfigured:135(17943224ns) {StartInitialHandshake:157(17942016ns) {DropHandshakes:469(806ns) {DropIncomingHandshake:441(138ns) DropOutgoingHandshake:456(89ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:476(764ns) {SwitchToState:178(467ns)}}}}}
2025-11-19T13:58:58.463654Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435698461318909:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.464731Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435698461318897:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.464748Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435698118135755:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.464784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435698118135696:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:58.463886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435695409840841:2111] [node 6] ICP01 ready to work
2025-11-19T13:58:58.464493Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435695409840843:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.465141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435696950411307:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:58.465179Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435696950411307:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(28077120ns) {IncomingHandshake:261(28076617ns) {DropIncomingHandshake:441(237ns) PrepareNewSessionHandshake:476(369ns) {SwitchToState:178(123ns)} GenerateSessionVirtualId:988(510ns) IssueIncomingHandshakeReply:193(28071930ns) {SwitchToState:178(553ns)}}}
2025-11-19T13:58:58.463885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435695044076853:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(48372746ns) {Configure:104(48371516ns) {ProcessConfigured:135(48039885ns) {StartInitialHandshake:157(48039203ns) {DropHandshakes:469(507ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:476(1359ns) {SwitchToState:178(399ns)}}}}}
2025-11-19T13:58:58.463970Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435695044076897:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.468159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435695044076824:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(25221930ns) {Configure:104(25220859ns) {ProcessConfigured:135(24948373ns) {StartInitialHandshake:157(24948229ns) {DropHandshakes:469(378ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(64ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:476(805ns) {SwitchToState:178(20ns)}}}}}
2025-11-19T13:58:58.469402Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435695044076898:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.469428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435695044076822:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:58.469484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435695044076853:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:58.470526Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435695409840837:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.470547Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435695409840837:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:58.466556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435698461318908:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
2025-11-19T13:58:58.472362Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435698118135768:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.476324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435696950411385:2115] [node 6] ICP01 ready to work
2025-11-19T13:58:58.477006Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435695409840844:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.472575Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435698461318894:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:58.473352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435698461318888:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.473370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:58.473381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435698461318888:2107] [node 4] ICP22 created new session: [1:7574435698461318913:2107]
2025-11-19T13:58:58.473388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435698461318913:2107] [node 4] ICS09 handshake done sender: [1:7574435698461318894:2110] self: [1:7574435698461318891:0] peer: [4:7574435696950411373:0] socket: 178 qp: -1
2025-11-19T13:58:58.473397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435698461318913:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:58.473408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:58.480141Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435696950411368:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.480185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435696950411308:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.480329Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435695409840835:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.480340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435695409840835:2109] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.481404Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435695600720216:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.481436Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435695600720225:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.481461Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435695600720223:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.481463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435695600720223:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:58.482038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435695600720154:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.482341Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435695600720223:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:58.483613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435695600720152:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:58.483760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435695600720186:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.483774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:58.483780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435695600720186:2080] [node 6] ICP22 created new session: [5:7574435695600720235:2080]
2025-11-19T13:58:58.483785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435695600720235:2080] [node 6] ICS09 handshake done sender: [5:7574435695600720223:2112] self: [5:7574435695600720190:0] peer: [6:7574435695044076903:0] socket: 186 qp: -1
2025-11-19T13:58:58.483791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435695600720235:2080] [node 6] ICS10 traffic start
2025-11-19T13:58:58.483825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:58.483937Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435695409840835:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:58.484035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435695409840830:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.484051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:58.484055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435695409840830:2107] [node 4] ICP22 created new session: [2:7574435695409840853:2107]
2025-11-19T13:58:58.484060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435695409840853:2107] [node 4] ICS09 handshake done sender: [2:7574435695409840835:2109] self: [2:7574435695409840832:0] peer: [4:7574435696950411388:0] socket: 171 qp: -1
2025-11-19T13:58:58.484065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435695409840853:2107] [node 4] ICS10 traffic start
2025-11-19T13:58:58.484072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:58.482776Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435698461318899:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.482804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435698461318899:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:58.484855Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435698118135772:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.484871Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435698118135772:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:58.488111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435696950411389:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2591
2025-11-19T13:58:58.488211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435696950411390:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2591
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.490653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435695409840855:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30817
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.490974Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435695409840837:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:58.490294Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435698118135767:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.490873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435698118135782:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-19T13:58:58.491218Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435698118135755:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:58.492307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435696950411391:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2591
2025-11-19T13:58:58.491331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435695409840770:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.491352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:58.491361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435695409840770:2049] [node 3] ICP22 created new session: [2:7574435695409840858:2049]
2025-11-19T13:58:58.491368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435695409840858:2049] [node 3] ICS09 handshake done sender: [2:7574435695409840837:2110] self: [2:7574435695409840836:0] peer: [3:7574435698118135779:0] socket: 172 qp: -1
2025-11-19T13:58:58.491375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435695409840858:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:58.491384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:58.495387Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435696950411389:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.492038Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435698461318907:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.495873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435698118135696:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.495900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:58.495913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435698118135696:2049] [node 2] ICP22 created new session: [3:7574435698118135783:2049]
2025-11-19T13:58:58.495922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435698118135783:2049] [node 2] ICS09 handshake done sender: [3:7574435698118135755:2107] self: [3:7574435698118135779:0] peer: [2:7574435695409840836:0] socket: 175 qp: -1
2025-11-19T13:58:58.495931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435698118135783:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:58.495941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:58.499948Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435698461318908:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.500449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435695600720238:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49699
2025-11-19T13:58:58.504095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435695409840861:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30817
2025-11-19T13:58:58.506026Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435698118135772:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:58.506163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435698118135766:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.506185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:58.506193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435698118135766:2111] [node 5] ICP22 created new session: [3:7574435698118135787:2111]
2025-11-19T13:58:58.506199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435698118135787:2111] [node 5] ICS09 handshake done sender: [3:7574435698118135772:2116] self: [3:7574435698118135771:0] peer: [5:7574435695600720231:0] socket: 196 qp: -1
2025-11-19T13:58:58.506206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435698118135787:2111] [node 5] ICS10 traffic start
2025-11-19T13:58:58.506243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:58.503883Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435698461318899:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.516144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435695044076904:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29271
2025-11-19T13:58:58.517610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435695044076905:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29271
2025-11-19T13:58:58.517753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435698461318888:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(44370609ns) {HandleHandshakeStatus:329(44369367ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(249ns) SwitchToState:178(72ns) ProcessPendingSessionEvents:495(29627036ns) {ForwardSessionEventToSession:612(29613610ns) ForwardSessionEventToSession:612(1926ns) ForwardSessionEventToSession:612(1546ns) ForwardSessionEventToSession:612(880ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(1267ns) ForwardSessionEventToSession:612(571ns) ForwardSessionEventToSession:612(519ns) ForwardSessionEventToSession:612(551ns) ForwardSessionEventToSession:612(921ns)}}}
2025-11-19T13:58:58.517787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435698461318895:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.517807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:58.517813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435698461318895:2108] [node 6] ICP22 created new session: [1:7574435698461318918:2108]
2025-11-19T13:58:58.517818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435698461318918:2108] [node 6] ICS09 handshake done sender: [1:7574435698461318899:2112] self: [1:7574435698461318898:0] peer: [6:7574435695044076902:0] socket: 179 qp: -1
2025-11-19T13:58:58.517824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435698461318918:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:58.517833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:58.517980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435698461318888:2107] [node 4] ICP18 handshake (actor [1:7574435698461318907:2116]) is too old
2025-11-19T13:58:58.518329Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435698461318901:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.518340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435698461318901:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.528407Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435695600720186:2080] [node 6] ICP35 event processing took too much time PendingConnection:278(44609368ns) {HandleHandshakeStatus:329(44594871ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(299ns) SwitchToState:178(46ns) ProcessPendingSessionEvents:495(35770424ns) {ForwardSessionEventToSession:612(35750734ns) ForwardSessionEventToSession:612(2936ns) ForwardSessionEventToSession:612(1065ns) ForwardSessionEventToSession:612(682ns) ForwardSessionEventToSession:612(699ns) ForwardSessionEventToSession:612(1374ns) ForwardSessionEventToSession:612(538ns) ForwardSessionEventToSession:612(873ns) ForwardSessionEventToSession:612(722ns) ForwardSessionEventToSession:612(1855ns)}}}
2025-11-19T13:58:58.528439Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435695600720154:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18488) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435695600720225:2113] held: no
2025-11-19T13:58:58.528863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435695409840830:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(44799337ns) {HandleHandshakeStatus:329(44798665ns) {DropIncomingHandshake:441(4051ns) DropOutgoingHandshake:456(128ns) SwitchToState:178(39ns) ProcessPendingSessionEvents:495(36205528ns) {ForwardSessionEventToSession:612(36193681ns) ForwardSessionEventToSession:612(1375ns) ForwardSessionEventToSession:612(931ns) ForwardSessionEventToSession:612(834ns) ForwardSessionEventToSession:612(507ns) ForwardSessionEventToSession:612(1535ns) ForwardSessionEventToSession:612(756ns) ForwardSessionEventToSession:612(775ns) ForwardSessionEventToSession:612(448ns) ForwardSessionEventToSession:612(676ns)}}}
2025-11-19T13:58:58.531884Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435695409840855:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.532472Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435695600720238:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.532523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435695600720153:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.536913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435695600720241:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49699
2025-11-19T13:58:58.540432Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435695600720225:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:58.540696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435695600720154:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.540714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:58.540721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435695600720154:2050] [node 3] ICP22 created new session: [5:7574435695600720242:2050]
2025-11-19T13:58:58.540728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435695600720242:2050] [node 3] ICS09 handshake done sender: [5:7574435695600720225:2113] self: [5:7574435695600720231:0] peer: [3:7574435698118135771:0] socket: 199 qp: -1
2025-11-19T13:58:58.540736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435695600720242:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:58.540744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.542412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435698461318920:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13580
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.542573Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435695409840861:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.542608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435695409840769:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:58.542727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435698118135789:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-19T13:58:58.542859Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435698461318905:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.542865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435698461318905:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:58.543254Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435698461318920:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.544199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435695044076906:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29271
2025-11-19T13:58:58.544165Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435695409840847:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.544177Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435695409840847:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.549017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435695409840864:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30817
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.551994Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435698461318905:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:58.552106Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435698461318901:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:58.552055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435695600720244:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49699
2025-11-19T13:58:58.552279Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435698118135696:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(56372620ns) {HandleHandshakeStatus:329(56371256ns) {DropIncomingHandshake:441(395ns) DropOutgoingHandshake:456(3142ns) SwitchToState:178(172ns) ProcessPendingSessionEvents:495(32188915ns) {ForwardSessionEventToSession:612(32169826ns) ForwardSessionEventToSession:612(2794ns) ForwardSessionEventToSession:612(1204ns) ForwardSessionEventToSession:612(510ns) ForwardSessionEventToSession:612(4330ns) ForwardSessionEventToSession:612(2069ns) ForwardSessionEventToSession:612(612ns) ForwardSessionEventToSession:612(645ns) ForwardSessionEventToSession:612(754ns) ForwardSessionEventToSession:612(808ns)}}}
2025-11-19T13:58:58.552713Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435695600720216:2106] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:58.552721Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435695409840861:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:58.552792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435695600720152:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.552811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:58.552817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435695600720152:2048] [node 1] ICP22 created new session: [5:7574435695600720245:2048]
2025-11-19T13:58:58.552823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435695600720245:2048] [node 1] ICS09 handshake done sender: [5:7574435695600720216:2106] self: [5:7574435695600720234:0] peer: [1:7574435698461318900:0] socket: 181 qp: -1
2025-11-19T13:58:58.552828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435695600720245:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:58.552835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:58.552272Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435696950411308:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435696950411368:2108] held: no
2025-11-19T13:58:58.552332Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435696950411390:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.555845Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435696950411372:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.558340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435698461318896:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.558368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:58.558378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435698461318896:2109] [node 5] ICP22 created new session: [1:7574435698461318925:2109]
2025-11-19T13:58:58.558385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435698461318925:2109] [node 5] ICS09 handshake done sender: [1:7574435698461318901:2113] self: [1:7574435698461318900:0] peer: [5:7574435695600720234:0] socket: 180 qp: -1
2025-11-19T13:58:58.558392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435698461318925:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:58.558402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:58.558524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435698461318827:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.558535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:58.558541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435698461318827:2048] [node 2] ICP22 created new session: [1:7574435698461318927:2048]
2025-11-19T13:58:58.558545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435698461318927:2048] [node 2] ICS09 handshake done sender: [1:7574435698461318905:2114] self: [1:7574435698461318892:0] peer: [2:7574435695409840863:0] socket: 215 qp: -1
2025-11-19T13:58:58.558549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435698461318927:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:58.558554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:58.558857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435696950411307:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3125) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435696950411372:2110] held: no
2025-11-19T13:58:58.558912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435696950411309:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:58.559115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435696950411392:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2591
2025-11-19T13:58:58.559386Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435696950411368:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:58.559277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435698118135765:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.559290Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435698118135765:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:58.560240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435695044076907:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29271
2025-11-19T13:58:58.559455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435696950411308:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.559467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:58.559471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435696950411308:2049] [node 2] ICP22 created new session: [4:7574435696950411396:2049]
2025-11-19T13:58:58.559476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435696950411396:2049] [node 2] ICS09 handshake done sender: [4:7574435696950411368:2108] self: [4:7574435696950411388:0] peer: [2:7574435695409840832:0] socket: 174 qp: -1
2025-11-19T13:58:58.559481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435696950411396:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:58.559499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:58.561629Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435695044076906:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.561658Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435695044076898:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:58.561728Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435695044076897:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:58.562081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435695044076822:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3125) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435695044076897:2114] held: no
2025-11-19T13:58:58.562189Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435695044076907:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.562210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435695044076853:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.562238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:58.562245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435695044076853:2064] [node 5] ICP22 created new session: [6:7574435695044076908:2064]
2025-11-19T13:58:58.562255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435695044076908:2064] [node 5] ICS09 handshake done sender: [6:7574435695044076898:2115] self: [6:7574435695044076903:0] peer: [5:7574435695600720190:0] socket: 188 qp: -1
2025-11-19T13:58:58.562264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435695044076908:2064] [node 5] ICS10 traffic start
2025-11-19T13:58:58.562276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:58.562728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435695044076823:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:58.562749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435695044076824:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.564358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435695600720247:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49699
2025-11-19T13:58:58.563140Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435698118135777:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.563152Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435698118135777:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:58.564466Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435698118135777:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:58.564543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435698118135760:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.564557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:58.564564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435698118135760:2110] [node 6] ICP22 created new session: [3:7574435698118135791:2110]
2025-11-19T13:58:58.564569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435698118135791:2110] [node 6] ICS09 handshake done sender: [3:7574435698118135777:2118] self: [3:7574435698118135762:0] peer: [6:7574435695044076911:0] socket: 220 qp: -1
2025-11-19T13:58:58.564573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435698118135791:2110] [node 6] ICS10 traffic start
2025-11-19T13:58:58.564579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.564834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435695044076822:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.564848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:58.564854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435695044076822:2048] [node 1] ICP22 created new session: [6:7574435695044076913:2048]
2025-11-19T13:58:58.564864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435695044076913:2048] [node 1] ICS09 handshake done sender: [6:7574435695044076897:2114] self: [6:7574435695044076902:0] peer: [1:7574435698461318898:0] socket: 187 qp: -1
2025-11-19T13:58:58.564870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435695044076913:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:58.564876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:58.566331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435695044076915:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29271
2025-11-19T13:58:58.566403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435695409840769:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.566423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:58.566430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435695409840769:2048] [node 1] ICP22 created new session: [2:7574435695409840865:2048]
2025-11-19T13:58:58.566436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435695409840865:2048] [node 1] ICS09 handshake done sender: [2:7574435695409840861:2118] self: [2:7574435695409840863:0] peer: [1:7574435698461318892:0] socket: 217 qp: -1
2025-11-19T13:58:58.566441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435695409840865:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:58.566448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:58.566632Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435695409840849:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.566635Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435695409840849:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:58.568382Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435698461318906:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.568396Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435698461318906:2115] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.569310Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435698461318906:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:58.568185Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435698118135789:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.568221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435698118135695:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:58.569202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435698118135794:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14408
2025-11-19T13:58:58.569412Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435698118135789:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:58.569699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435698118135695:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.569724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:58.569731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435698118135695:2048] [node 1] ICP22 created new session: [3:7574435698118135795:2048]
2025-11-19T13:58:58.569737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435698118135795:2048] [node 1] ICS09 handshake done sender: [3:7574435698118135789:2120] self: [3:7574435698118135793:0] peer: [1:7574435698461318893:0] socket: 224 qp: -1
2025-11-19T13:58:58.569744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435698118135795:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:58.569750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.571965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435696950411307:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.571986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:58.571994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435696950411307:2048] [node 1] ICP22 created new session: [4:7574435696950411398:2048]
2025-11-19T13:58:58.572002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435696950411398:2048] [node 1] ICS09 handshake done sender: [4:7574435696950411372:2110] self: [4:7574435696950411373:0] peer: [1:7574435698461318891:0] socket: 183 qp: -1
2025-11-19T13:58:58.572008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435696950411398:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:58.572016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:58.572261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435696950411400:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2591
2025-11-19T13:58:58.576091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435695409840769:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:58.578898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435698461318828:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.578925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:58.578933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435698461318828:2049] [node 3] ICP22 created new session: [1:7574435698461318930:2049]
2025-11-19T13:58:58.578941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435698461318930:2049] [node 3] ICS09 handshake done sender: [1:7574435698461318906:2115] self: [1:7574435698461318893:0] peer: [3:7574435698118135793:0] socket: 197 qp: -1
2025-11-19T13:58:58.578948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435698461318930:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:58.578958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:58.579330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435695044076824:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18488) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435695044076907:2120] held: no
2025-11-19T13:58:58.579421Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435695044076907:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:58.578249Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435695600720152:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(25405025ns) {HandleHandshakeStatus:329(25404258ns) {DropIncomingHandshake:441(338ns) DropOutgoingHandshake:456(1429ns) SwitchToState:178(219ns) ProcessPendingSessionEvents:495(17627103ns) {ForwardSessionEventToSession:612(17610736ns) ForwardSessionEventToSession:612(2669ns) ForwardSessionEventToSession:612(836ns) ForwardSessionEventToSession:612(603ns) ForwardSessionEventToSession:612(975ns) ForwardSessionEventToSession:612(1278ns) ForwardSessionEventToSession:612(710ns) ForwardSessionEventToSession:612(483ns) ForwardSessionEventToSession:612(566ns) ForwardSessionEventToSession:612(865ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.580127Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435695409840849:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:58.580081Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435696950411400:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.580214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435696950411401:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2591
2025-11-19T13:58:58.579995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435695600720248:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49699
2025-11-19T13:58:58.580400Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435695409840847:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.580372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435695044076824:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.580387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:58.580393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435695044076824:2050] [node 3] ICP22 created new session: [6:7574435695044076917:2050]
2025-11-19T13:58:58.580397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435695044076917:2050] [node 3] ICS09 handshake done sender: [6:7574435695044076907:2120] self: [6:7574435695044076911:0] peer: [3:7574435698118135762:0] socket: 223 qp: -1
2025-11-19T13:58:58.580402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435695044076917:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:58.580408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:58.580529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435695044076916:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29271
2025-11-19T13:58:58.580617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435695409840845:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.580883Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435695044076906:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:58.580636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:58.580642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435695409840845:2112] [node 5] ICP22 created new session: [2:7574435695409840869:2112]
2025-11-19T13:58:58.580648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435695409840869:2112] [node 5] ICS09 handshake done sender: [2:7574435695409840847:2115] self: [2:7574435695409840846:0] peer: [5:7574435695600720240:0] socket: 212 qp: -1
2025-11-19T13:58:58.580654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435695409840869:2112] [node 5] ICS10 traffic start
2025-11-19T13:58:58.580662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:58.580973Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435695600720247:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.581005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435695600720192:2082] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:58.581084Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435695600720238:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:58.580487Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435698118135765:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:58.582516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435698118135759:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.582545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:58.582558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435698118135759:2109] [node 4] ICP22 created new session: [3:7574435698118135798:2109]
2025-11-19T13:58:58.582573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435698118135798:2109] [node 4] ICS09 handshake done sender: [3:7574435698118135765:2112] self: [3:7574435698118135764:0] peer: [4:7574435696950411393:0] socket: 194 qp: -1
2025-11-19T13:58:58.582609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435698118135798:2109] [node 4] ICS10 traffic start
2025-11-19T13:58:58.582627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:58.581731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435695409840841:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.581768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:58.581781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435695409840841:2111] [node 6] ICP22 created new session: [2:7574435695409840871:2111]
2025-11-19T13:58:58.581792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435695409840871:2111] [node 6] ICS09 handshake done sender: [2:7574435695409840849:2116] self: [2:7574435695409840848:0] peer: [6:7574435695044076910:0] socket: 218 qp: -1
2025-11-19T13:58:58.581806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435695409840871:2111] [node 6] ICS10 traffic start
2025-11-19T13:58:58.581819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:58.583757Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435696950411382:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.583774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435696950411382:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:58.583777Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435696950411389:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:58.584637Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435696950411382:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:58.584715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435696950411309:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.584731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:58.584737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435696950411309:2050] [node 3] ICP22 created new session: [4:7574435696950411404:2050]
2025-11-19T13:58:58.584742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435696950411404:2050] [node 3] ICS09 handshake done sender: [4:7574435696950411389:2117] self: [4:7574435696950411393:0] peer: [3:7574435698118135764:0] socket: 202 qp: -1
2025-11-19T13:58:58.584747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435696950411404:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:58.584753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:58.584892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435696950411378:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.584903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:58.584907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435696950411378:2112] [node 5] ICP22 created new session: [4:7574435696950411406:2112]
2025-11-19T13:58:58.584910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435696950411406:2112] [node 5] ICS09 handshake done sender: [4:7574435696950411382:2114] self: [4:7574435696950411381:0] peer: [5:7574435695600720249:0] socket: 185 qp: -1
2025-11-19T13:58:58.584914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435696950411406:2112] [node 5] ICS10 traffic start
2025-11-19T13:58:58.584919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:58.584612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435695600720153:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.584630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:58.584640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435695600720153:2049] [node 2] ICP22 created new session: [5:7574435695600720250:2049]
2025-11-19T13:58:58.584646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435695600720250:2049] [node 2] ICS09 handshake done sender: [5:7574435695600720238:2117] self: [5:7574435695600720240:0] peer: [2:7574435695409840846:0] socket: 216 qp: -1
2025-11-19T13:58:58.584653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435695600720250:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:58.584660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:58.585534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435695044076823:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.585552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:58.585559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435695044076823:2049] [node 2] ICP22 created new session: [6:7574435695044076919:2049]
2025-11-19T13:58:58.585565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435695044076919:2049] [node 2] ICS09 handshake done sender: [6:7574435695044076906:2119] self: [6:7574435695044076910:0] peer: [2:7574435695409840848:0] socket: 222 qp: -1
2025-11-19T13:58:58.585571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435695044076919:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:58.585578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:58.585857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435695044076823:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.589834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435695600720252:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49699
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.590034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435695044076921:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29271
2025-11-19T13:58:58.590222Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435695600720247:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:58.592536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435695600720192:2082] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7946) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435695600720247:2120] held: no
2025-11-19T13:58:58.592764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435695600720192:2082] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.592780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:58.592786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435695600720192:2082] [node 4] ICP22 created new session: [5:7574435695600720253:2082]
2025-11-19T13:58:58.592792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435695600720253:2082] [node 4] ICS09 handshake done sender: [5:7574435695600720247:2120] self: [5:7574435695600720249:0] peer: [4:7574435696950411381:0] socket: 173 qp: -1
2025-11-19T13:58:58.592800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435695600720253:2082] [node 4] ICS10 traffic start
2025-11-19T13:58:58.592810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:58.593373Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435696950411387:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.593384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435696950411387:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:58.594559Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435696950411387:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:58.593228Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435695044076921:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:58.593269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435695044076846:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:58.594255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435695044076923:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29271
2025-11-19T13:58:58.594733Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435695044076921:2123] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:58.594803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435695044076846:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:58.594812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:58.594817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435695044076846:2062] [node 4] ICP22 created new session: [6:7574435695044076924:2062]
2025-11-19T13:58:58.594822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435695044076924:2062] [node 4] ICS09 handshake done sender: [6:7574435695044076921:2123] self: [6:7574435695044076922:0] peer: [4:7574435696950411386:0] socket: 182 qp: -1
2025-11-19T13:58:58.594828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435695044076924:2062] [node 4] ICS10 traffic start
2025-11-19T13:58:58.594836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:58.596005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435696950411385:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:58.596025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:58.596030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435696950411385:2115] [node 6] ICP22 created new session: [4:7574435696950411410:2115]
2025-11-19T13:58:58.596036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435696950411410:2115] [node 6] ICS09 handshake done sender: [4:7574435696950411387:2116] self: [4:7574435696950411386:0] peer: [6:7574435695044076922:0] socket: 169 qp: -1
2025-11-19T13:58:58.596041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435696950411410:2115] [node 6] ICS10 traffic start
2025-11-19T13:58:58.596049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:58.626141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:58.626245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435698118135795:2048] [node 1] ICS01 socket: 224 reason# EndOfStream
2025-11-19T13:58:58.626257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435698118135695:2048] [node 1] ICP30 unregister session Session# [3:7574435698118135795:2048] VirtualId# [3:7574435698118135793:0]
2025-11-19T13:58:58.626264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435698118135795:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:58.626299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:58.626319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:58.626196Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:58.626297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435695409840865:2048] [node 1] ICS01 socket: 217 reason# EndOfStream
2025-11-19T13:58:58.626305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435695409840769:2048] [node 1] ICP30 unregister session Session# [2:7574435695409840865:2048] VirtualId# [2:7574435695409840863:0]
2025-11-19T13:58:58.626311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435695409840865:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:58.626352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:58.626373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:58.640889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:58.640991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435696950411398:2048] [node 1] ICS01 socket: 183 reason# EndOfStream
2025-11-19T13:58:58.641003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435696950411307:2048] [node 1] ICP30 unregister session Session# [4:7574435696950411398:2048] VirtualId# [4:7574435696950411373:0]
2025-11-19T13:58:58.641012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435696950411398:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:58.641077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:58.641108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:58.641256Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:58.641330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435695044076913:2048] [node 1] ICS01 socket: 187 reason# EndOfStream
2025-11-19T13:58:58.641336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435695044076822:2048] [node 1] ICP30 unregister session Session# [6:7574435695044076913:2048] VirtualId# [6:7574435695044076902:0]
2025-11-19T13:58:58.641341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435695044076913:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:58.641382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:58.641405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:58.641440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:58.641507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435695600720245:2048] [node 1] ICS07 socket disconnect 181 reason# EndOfStream
2025-11-19T13:58:58.641513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435695600720245:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:58.641545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:58.641567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435695600720245:2048] [node 1] ICS15 start handshake
2025-11-19T13:58:58.642298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435695600720152:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3125) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:58:58.642312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435695600720245:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:58:58.642318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435695600720152:2048] [node 1] ICP30 unregister session Session# [5:7574435695600720245:2048] VirtualId# [5:7574435695600720234:0]
2025-11-19T13:58:58.642333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:58.642370Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435695600720152:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3125) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:58.642373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3125) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:58:58.649873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:58.649963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435698118135783:2049] [node 2] ICS07 socket disconnect 175 reason# EndOfStream
2025-11-19T13:58:58.649969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435698118135783:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:58.650008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:58.650027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435698118135783:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:58.653393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435698118135696:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24952) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:58:58.653410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435698118135783:2049] [node 2] ICS15 start handshake
2025-11-19T13:58:58.654844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:58.654926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435696950411396:2049] [node 2] ICS01 socket: 174 reason# EndOfStream
2025-11-19T13:58:58.654932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435696950411308:2049] [node 2] ICP30 unregister session Session# [4:7574435696950411396:2049] VirtualId# [4:7574435696950411388:0]
2025-11-19T13:58:58.654938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435696950411396:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:58.654986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:58.655009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:58.655034Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:58.655113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435695044076919:2049] [node 2] ICS01 socket: 222 reason# EndOfStream
2025-11-19T13:58:58.655120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435695044076823:2049] [node 2] ICP30 unregister session Session# [6:7574435695044076919:2049] VirtualId# [6:7574435695044076910:0]
2025-11-19T13:58:58.655126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435695044076919:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:58.655171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:58.655193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:58.658460Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:58.658548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435695600720250:2049] [node 2] ICS01 socket: 216 reason# EndOfStream
2025-11-19T13:58:58.658557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435695600720153:2049] [node 2] ICP30 unregister session Session# [5:7574435695600720250:2049] VirtualId# [5:7574435695600720240:0]
2025-11-19T13:58:58.658562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435695600720250:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:58.658616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:58.658639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:58.658835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435695600720152:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:58:58.668306Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:58.668389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435695044076917:2050] [node 3] ICS01 socket: 223 reason# EndOfStream
2025-11-19T13:58:58.668397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435695044076824:2050] [node 3] ICP30 unregister session Session# [6:7574435695044076917:2050] VirtualId# [6:7574435695044076911:0]
2025-11-19T13:58:58.668402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435695044076917:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:58.668483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:58.668526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:58.668077Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:58.668135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435695600720242:2050] [node 3] ICS01 socket: 199 reason# EndOfStream
2025-11-19T13:58:58.668140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435695600720154:2050] [node 3] ICP30 unregister session Session# [5:7574435695600720242:2050] VirtualId# [5:7574435695600720231:0]
2025-11-19T13:58:58.668144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435695600720242:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:58.668168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:58.668188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:58.691903Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:58.691998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435695044076924:2062] [node 4] ICS01 socket: 182 reason# EndOfStream
2025-11-19T13:58:58.692010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435695044076846:2062] [node 4] ICP30 unregister session Session# [6:7574435695044076924:2062] VirtualId# [6:7574435695044076922:0]
2025-11-19T13:58:58.692018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435695044076924:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:58.692083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:58.692110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:58.961906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435694947296963:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:58.961925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435694947296964:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:59.004087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435700813185804:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:59.004100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435700813185805:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:59.048160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435701718797400:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:59.048171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435701718797401:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:59.104431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435702332487376:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:59.104442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435702332487377:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:59.104447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435702332487378:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:59.137776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435699166098317:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:59.137792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435699166098318:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:59.137796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435699166098319:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:59.187967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435699956755082:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:59.187982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435699956755083:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:59.187986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435699956755084:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:59.228693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435699956755123:2087] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.230350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435701718797460:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52568
2025-11-19T13:58:59.230123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435699242264320:2107] [node 6] ICP01 ready to work
2025-11-19T13:58:59.231406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435699956755134:2093] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.232068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435699956755138:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51495
2025-11-19T13:58:59.232005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435699166098377:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17242
2025-11-19T13:58:59.239271Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435701718797460:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.238337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435699242264331:2111] [node 5] ICP01 ready to work
2025-11-19T13:58:59.243827Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435699166098377:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.248116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435702332487435:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.248976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435699956755154:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51495
2025-11-19T13:58:59.251568Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435699956755154:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.251606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435699956755134:2093] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.255238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435700813185870:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:59.255346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435700813185871:2110] [node 6] ICP01 ready to work
2025-11-19T13:58:59.259780Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435702332487437:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.259802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435702332487437:2106] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.261517Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435702332487437:2106] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.265579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435699166098384:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17242
2025-11-19T13:58:59.264731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435700813185873:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23343
2025-11-19T13:58:59.269539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435699242264327:2109] [node 4] ICP01 ready to work
2025-11-19T13:58:59.269050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435699166098379:2108] [node 6] ICP01 ready to work
2025-11-19T13:58:59.269438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435699166098379:2108] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.269851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435701718797462:2108] [node 6] ICP01 ready to work
2025-11-19T13:58:59.269982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435701718797467:2110] [node 4] ICP01 ready to work
2025-11-19T13:58:59.271604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435702332487435:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.271627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:59.271638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435702332487435:2105] [node 6] ICP22 created new session: [4:7574435702332487447:2105]
2025-11-19T13:58:59.271646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435702332487447:2105] [node 6] ICS09 handshake done sender: [4:7574435702332487437:2106] self: [4:7574435702332487436:0] peer: [6:7574435699956755155:0] socket: 187 qp: -1
2025-11-19T13:58:59.271655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435702332487447:2105] [node 6] ICS10 traffic start
2025-11-19T13:58:59.271679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.275056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435701718797474:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52568
2025-11-19T13:58:59.272768Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435700813185805:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17380183ns) {Configure:104(17379586ns) {ProcessConfigured:135(16957501ns) {StartInitialHandshake:157(16956964ns) {DropHandshakes:469(760ns) {DropIncomingHandshake:441(45ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(342ns) PrepareNewSessionHandshake:476(664ns) {SwitchToState:178(530ns)}}}}}
2025-11-19T13:58:59.274229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435700813185882:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23343
2025-11-19T13:58:59.272272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435702332487450:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:59.272325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435702332487449:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44558
2025-11-19T13:58:59.273260Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435702332487449:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.274316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435702332487435:2105] [node 6] ICP18 handshake (actor [4:7574435702332487449:2112]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.277631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435699956755159:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51495
2025-11-19T13:58:59.278319Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435699956755127:2089] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.278332Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435699956755127:2089] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.280344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435700813185883:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23343
2025-11-19T13:58:59.280439Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435700813185882:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.280560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435699166098387:2112] [node 4] ICP01 ready to work
2025-11-19T13:58:59.282349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435700813185883:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.283267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435699242264332:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40051
2025-11-19T13:58:59.283329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435699242264335:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40051
2025-11-19T13:58:59.283456Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435694947296964:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(50174384ns) {Configure:104(50173533ns) {ProcessConfigured:135(49935471ns) {StartInitialHandshake:157(49934888ns) {DropHandshakes:469(399ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(79ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(955ns) {SwitchToState:178(329ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.286371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435702332487459:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44558
2025-11-19T13:58:59.284987Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435699242264335:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.285261Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435701718797474:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.285371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435701718797478:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52568
2025-11-19T13:58:59.290371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435701718797469:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.291200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435699956755160:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51495
2025-11-19T13:58:59.289736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435700813185880:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:59.287981Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435699242264332:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.295879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435699956755134:2093] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3758) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435699956755154:2111] held: no
2025-11-19T13:58:59.295934Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435699956755138:2095] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.295969Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435699956755154:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.296595Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435701718797400:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(36688220ns) {Configure:104(36686925ns) {ProcessConfigured:135(36247647ns) {StartInitialHandshake:157(36246663ns) {DropHandshakes:469(1121ns) {DropIncomingHandshake:441(90ns) DropOutgoingHandshake:456(116ns)} GenerateSessionVirtualId:988(259ns) PrepareNewSessionHandshake:476(586ns) {SwitchToState:178(255ns)}}}}}
2025-11-19T13:58:59.296634Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435701718797478:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.297709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435701718797400:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.298145Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435699242264326:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.298155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435699242264326:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.298586Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435699242264322:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.298590Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435699242264322:2108] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.299073Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435699242264336:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.299076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435699242264336:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.293445Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435700813185873:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.296503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435700813185887:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23343
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.297861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435699166098387:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(17239981ns) {Configure:104(17239107ns) {ProcessConfigured:135(16893671ns) {StartInitialHandshake:157(16893112ns) {DropHandshakes:469(737ns) {DropIncomingHandshake:441(102ns) DropOutgoingHandshake:456(190ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:476(862ns) {SwitchToState:178(21ns)}}}}}
2025-11-19T13:58:59.297962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435699166098392:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17242
2025-11-19T13:58:59.300004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435699166098317:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18949626ns) {Configure:104(18948559ns) {ProcessConfigured:135(18736110ns) {StartInitialHandshake:157(18735679ns) {DropHandshakes:469(253ns) {DropIncomingHandshake:441(21ns) DropOutgoingHandshake:456(28ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:476(1198ns) {SwitchToState:178(503ns)}}}}}
2025-11-19T13:58:59.300753Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435699242264326:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.301213Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435699242264322:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:59.301361Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435699242264336:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:59.300061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435702332487460:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44558
2025-11-19T13:58:59.300508Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435700813185877:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.300519Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435700813185877:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.301179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435700813185888:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23343
2025-11-19T13:58:59.297027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435699956755082:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.297169Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435699956755160:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.297175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435699956755134:2093] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.297184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:59.297190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435699956755134:2093] [node 4] ICP22 created new session: [6:7574435699956755168:2093]
2025-11-19T13:58:59.297195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435699956755168:2093] [node 4] ICS09 handshake done sender: [6:7574435699956755154:2111] self: [6:7574435699956755155:0] peer: [4:7574435702332487436:0] socket: 188 qp: -1
2025-11-19T13:58:59.297202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435699956755168:2093] [node 4] ICS10 traffic start
2025-11-19T13:58:59.297209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:59.297379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435699956755083:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.303033Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435700813185877:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:59.306270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435699956755161:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51495
2025-11-19T13:58:59.303777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435694947296963:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.303805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:59.303812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435694947296963:2048] [node 2] ICP22 created new session: [1:7574435699242264345:2048]
2025-11-19T13:58:59.303819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435699242264345:2048] [node 2] ICS09 handshake done sender: [1:7574435699242264326:2110] self: [1:7574435699242264325:0] peer: [2:7574435700813185886:0] socket: 182 qp: -1
2025-11-19T13:58:59.303826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435699242264345:2048] [node 2] ICS10 traffic start
2025-11-19T13:58:59.303835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:59.303872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435699242264320:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.303878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:59.303883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435699242264320:2107] [node 6] ICP22 created new session: [1:7574435699242264347:2107]
2025-11-19T13:58:59.303889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435699242264347:2107] [node 6] ICS09 handshake done sender: [1:7574435699242264322:2108] self: [1:7574435699242264321:0] peer: [6:7574435699956755167:0] socket: 177 qp: -1
2025-11-19T13:58:59.303893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435699242264347:2107] [node 6] ICS10 traffic start
2025-11-19T13:58:59.303910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: 2025-11-19T13:58:59.306366Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435699956755127:2089] [node 5] ICH04 handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.306600Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435701718797469:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16163465ns) {Configure:104(16162303ns) {ProcessConfigured:135(15885939ns) {StartInitialHandshake:157(15885057ns) {DropHandshakes:469(242ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:476(595ns) {SwitchToState:178(300ns)}}}}}
2025-11-19T13:58:59.307273Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435702332487459:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.307314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435702332487378:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.308687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435694947296964:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.308711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:59.308719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435694947296964:2049] [node 3] ICP22 created new session: [1:7574435699242264349:2049]
2025-11-19T13:58:59.308725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435699242264349:2049] [node 3] ICS09 handshake done sender: [1:7574435699242264336:2114] self: [1:7574435699242264328:0] peer: [3:7574435701718797483:0] socket: 205 qp: -1
2025-11-19T13:58:59.308733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435699242264349:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:59.308753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:59.307628Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435699166098319:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19200076ns) {Configure:104(19199275ns) {ProcessConfigured:135(18775285ns) {StartInitialHandshake:157(18774806ns) {DropHandshakes:469(668ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(174ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:476(806ns) {SwitchToState:178(633ns)}}}}}
2025-11-19T13:58:59.307690Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435699166098377:2107] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:59.309254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435700813185804:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.309352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435700813185871:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.309361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:59.309368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435700813185871:2110] [node 6] ICP22 created new session: [2:7574435700813185890:2110]
2025-11-19T13:58:59.309372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435700813185890:2110] [node 6] ICS09 handshake done sender: [2:7574435700813185877:2113] self: [2:7574435700813185876:0] peer: [6:7574435699956755170:0] socket: 193 qp: -1
2025-11-19T13:58:59.309377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435700813185890:2110] [node 6] ICS10 traffic start
2025-11-19T13:58:59.309384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:59.312453Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435699166098384:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.312471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435699166098379:2108] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.312480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:59.312489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435699166098379:2108] [node 6] ICP22 created new session: [5:7574435699166098397:2108]
2025-11-19T13:58:59.312495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435699166098397:2108] [node 6] ICS09 handshake done sender: [5:7574435699166098377:2107] self: [5:7574435699166098385:0] peer: [6:7574435699956755126:0] socket: 179 qp: -1
2025-11-19T13:58:59.312501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435699166098397:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:59.312519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:59.312702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435699166098318:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.313920Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435700813185873:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:59.313734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435701718797485:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52568
2025-11-19T13:58:59.314103Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435701718797478:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:59.314168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435701718797400:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.314179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:59.314185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435701718797400:2048] [node 1] ICP22 created new session: [3:7574435701718797486:2048]
2025-11-19T13:58:59.314190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435701718797486:2048] [node 1] ICS09 handshake done sender: [3:7574435701718797478:2115] self: [3:7574435701718797483:0] peer: [1:7574435699242264328:0] socket: 206 qp: -1
2025-11-19T13:58:59.314197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435701718797486:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:59.314225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:59.315941Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435701718797477:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.315951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435701718797477:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.306557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435699956755172:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51495
2025-11-19T13:58:59.306631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435699956755123:2087] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.306639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:59.306645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435699956755123:2087] [node 5] ICP22 created new session: [6:7574435699956755173:2087]
2025-11-19T13:58:59.306650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435699956755173:2087] [node 5] ICS09 handshake done sender: [6:7574435699956755127:2089] self: [6:7574435699956755126:0] peer: [5:7574435699166098385:0] socket: 175 qp: -1
2025-11-19T13:58:59.306654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435699956755173:2087] [node 5] ICS10 traffic start
2025-11-19T13:58:59.306673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:59.312177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435699956755171:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51495
2025-11-19T13:58:59.317117Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435699956755161:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.317133Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435699956755160:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.320011Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435702332487460:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.317589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435700813185880:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(27764079ns) {Configure:104(27763275ns) {ProcessConfigured:135(27428236ns) {StartInitialHandshake:157(27411207ns) {DropHandshakes:469(431ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(35ns)} GenerateSessionVirtualId:988(394ns) PrepareNewSessionHandshake:476(1010ns) {SwitchToState:178(633ns)}} IncomingHandshake:261(7668ns) {DropIncomingHandshake:441(224ns) PrepareNewSessionHandshake:476(193ns) {SwitchToState:178(156ns)} GenerateSessionVirtualId:988(226ns) IssueIncomingHandshakeReply:193(264ns)}}}}
2025-11-19T13:58:59.319423Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435700813185875:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.319429Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435700813185875:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.319828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435700813185804:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.319842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:59.319848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435700813185804:2048] [node 1] ICP22 created new session: [2:7574435700813185896:2048]
2025-11-19T13:58:59.319857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435700813185896:2048] [node 1] ICS09 handshake done sender: [2:7574435700813185873:2111] self: [2:7574435700813185886:0] peer: [1:7574435699242264325:0] socket: 183 qp: -1
2025-11-19T13:58:59.319864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435700813185896:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:59.319883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:59.319963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435700813185804:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:59.320160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435700813185892:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23343
2025-11-19T13:58:59.320195Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435700813185887:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.324691Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435700813185875:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:59.324849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435700813185870:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.324869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:59.324876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435700813185870:2109] [node 5] ICP22 created new session: [2:7574435700813185900:2109]
2025-11-19T13:58:59.324882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435700813185900:2109] [node 5] ICS09 handshake done sender: [2:7574435700813185875:2112] self: [2:7574435700813185874:0] peer: [5:7574435699166098399:0] socket: 192 qp: -1
2025-11-19T13:58:59.324888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435700813185900:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:59.324895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.329068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435699242264353:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40051
2025-11-19T13:58:59.329231Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435700813185871:2110] [node 6] ICP35 event processing took too much time PendingConnection:278(19846386ns) {HandleHandshakeStatus:329(19845414ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(124ns) SwitchToState:178(76ns) ProcessPendingSessionEvents:495(13510249ns) {ForwardSessionEventToSession:612(13496315ns) ForwardSessionEventToSession:612(2039ns) ForwardSessionEventToSession:612(1042ns) ForwardSessionEventToSession:612(696ns) ForwardSessionEventToSession:612(528ns) ForwardSessionEventToSession:612(984ns) ForwardSessionEventToSession:612(577ns) ForwardSessionEventToSession:612(533ns) ForwardSessionEventToSession:612(645ns) ForwardSessionEventToSession:612(1068ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.330906Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435699242264353:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.331720Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435699166098318:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435699166098384:2111] held: no
2025-11-19T13:58:59.330367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435702332487463:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44558
2025-11-19T13:58:59.330427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435702332487464:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44558
2025-11-19T13:58:59.330463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435702332487465:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44558
2025-11-19T13:58:59.331191Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435702332487463:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.331231Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435702332487464:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.331254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435702332487376:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.331265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435702332487377:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.332202Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435699956755138:2095] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:59.332390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435699166098400:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17242
2025-11-19T13:58:59.332429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435699166098401:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17242
2025-11-19T13:58:59.332439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435699166098402:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17242
2025-11-19T13:58:59.333859Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435699166098400:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.333893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435699166098319:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.333865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435699956755083:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.333883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:59.333890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435699956755083:2049] [node 2] ICP22 created new session: [6:7574435699956755176:2049]
2025-11-19T13:58:59.333896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435699956755176:2049] [node 2] ICS09 handshake done sender: [6:7574435699956755160:2116] self: [6:7574435699956755170:0] peer: [2:7574435700813185876:0] socket: 202 qp: -1
2025-11-19T13:58:59.333903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435699956755176:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:59.333909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:59.334057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435699956755082:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.334064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:59.334067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435699956755082:2048] [node 1] ICP22 created new session: [6:7574435699956755178:2048]
2025-11-19T13:58:59.334071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435699956755178:2048] [node 1] ICS09 handshake done sender: [6:7574435699956755138:2095] self: [6:7574435699956755167:0] peer: [1:7574435699242264321:0] socket: 180 qp: -1
2025-11-19T13:58:59.334075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435699956755178:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:59.334079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:59.333679Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435702332487459:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:59.335531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435702332487378:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.335549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:59.335555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435702332487378:2050] [node 3] ICP22 created new session: [4:7574435702332487468:2050]
2025-11-19T13:58:59.335560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435702332487468:2050] [node 3] ICS09 handshake done sender: [4:7574435702332487459:2116] self: [4:7574435702332487461:0] peer: [3:7574435701718797468:0] socket: 208 qp: -1
2025-11-19T13:58:59.335565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435702332487468:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:59.335571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:59.331989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435701718797489:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52568
2025-11-19T13:58:59.332097Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435701718797477:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:59.334696Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435701718797484:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.334702Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435701718797484:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.335240Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435701718797489:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.335263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435701718797401:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.335474Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435701718797484:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:59.335632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435701718797467:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.335642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:59.335650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435701718797467:2110] [node 4] ICP22 created new session: [3:7574435701718797493:2110]
2025-11-19T13:58:59.335655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435701718797493:2110] [node 4] ICS09 handshake done sender: [3:7574435701718797477:2114] self: [3:7574435701718797468:0] peer: [4:7574435702332487461:0] socket: 207 qp: -1
2025-11-19T13:58:59.335658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435701718797493:2110] [node 4] ICS10 traffic start
2025-11-19T13:58:59.335679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:59.335498Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435700813185881:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.335508Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435700813185881:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.336603Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435699242264339:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.336624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435699242264339:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.336671Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435700813185893:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.336681Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435700813185893:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.337115Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435699166098401:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.337143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435699166098317:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.337374Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435699166098384:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:59.337788Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435700813185881:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:59.337542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435701718797469:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.337558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:59.337566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435701718797469:2109] [node 5] ICP22 created new session: [3:7574435701718797495:2109]
2025-11-19T13:58:59.337572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435701718797495:2109] [node 5] ICS09 handshake done sender: [3:7574435701718797484:2117] self: [3:7574435701718797481:0] peer: [5:7574435699166098403:0] socket: 224 qp: -1
2025-11-19T13:58:59.339766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435700813185805:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.339794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:59.339802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435700813185805:2049] [node 3] ICP22 created new session: [2:7574435700813185904:2049]
2025-11-19T13:58:59.339807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435700813185904:2049] [node 3] ICS09 handshake done sender: [2:7574435700813185881:2114] self: [2:7574435700813185872:0] peer: [3:7574435701718797492:0] socket: 229 qp: -1
2025-11-19T13:58:59.339813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435700813185904:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:59.339819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:59.337578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435701718797495:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:59.337585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:59.340106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435699956755084:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.344752Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435699242264338:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.344771Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435699242264338:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.344848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435699166098404:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17242
2025-11-19T13:58:59.345108Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435701718797472:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.345125Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435701718797472:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.347236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435701718797401:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435701718797489:2119] held: no
2025-11-19T13:58:59.348698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435700813185871:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:58:59.349471Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435699242264339:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:59.349607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435699242264327:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.349630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:59.349639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435699242264327:2109] [node 4] ICP22 created new session: [1:7574435699242264356:2109]
2025-11-19T13:58:59.349645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435699242264356:2109] [node 4] ICS09 handshake done sender: [1:7574435699242264339:2116] self: [1:7574435699242264334:0] peer: [4:7574435702332487466:0] socket: 226 qp: -1
2025-11-19T13:58:59.349651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435699242264356:2109] [node 4] ICS10 traffic start
2025-11-19T13:58:59.349658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:59.349889Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435699166098400:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.350548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435699956755182:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51495
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.350678Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435701718797472:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:59.350770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435701718797462:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.350782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:59.350787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435701718797462:2108] [node 6] ICP22 created new session: [3:7574435701718797498:2108]
2025-11-19T13:58:59.350796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435701718797498:2108] [node 6] ICS09 handshake done sender: [3:7574435701718797472:2112] self: [3:7574435701718797471:0] peer: [6:7574435699956755180:0] socket: 201 qp: -1
2025-11-19T13:58:59.350802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435701718797498:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:59.351872Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435699956755084:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22733) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435699956755161:2117] held: no
2025-11-19T13:58:59.351897Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435699956755161:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:58:59.350808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:59.350940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435701718797500:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52568
2025-11-19T13:58:59.352000Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435700813185893:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:59.352084Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435701718797489:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:59.352990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435699956755084:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.353010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:59.353016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435699956755084:2050] [node 3] ICP22 created new session: [6:7574435699956755183:2050]
2025-11-19T13:58:59.353022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435699956755183:2050] [node 3] ICS09 handshake done sender: [6:7574435699956755161:2117] self: [6:7574435699956755180:0] peer: [3:7574435701718797471:0] socket: 203 qp: -1
2025-11-19T13:58:59.353029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435699956755183:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:59.353035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:59.353294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435700813185880:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.353313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:59.353318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435700813185880:2108] [node 4] ICP22 created new session: [2:7574435700813185907:2108]
2025-11-19T13:58:59.353323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435700813185907:2108] [node 4] ICS09 handshake done sender: [2:7574435700813185893:2120] self: [2:7574435700813185885:0] peer: [4:7574435702332487467:0] socket: 184 qp: -1
2025-11-19T13:58:59.353328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435700813185907:2108] [node 4] ICS10 traffic start
2025-11-19T13:58:59.353333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.356209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435699166098319:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.356225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:59.356240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435699166098319:2050] [node 3] ICP22 created new session: [5:7574435699166098409:2050]
2025-11-19T13:58:59.356247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435699166098409:2050] [node 3] ICS09 handshake done sender: [5:7574435699166098400:2117] self: [5:7574435699166098403:0] peer: [3:7574435701718797481:0] socket: 228 qp: -1
2025-11-19T13:58:59.356252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435699166098409:2050] [node 3] ICS10 traffic start
2025-11-19T13:58:59.356259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:59.356258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435699166098406:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17242
2025-11-19T13:58:59.356386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435699166098407:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17242
2025-11-19T13:58:59.356526Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435699242264338:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:59.356817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435699242264331:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.356832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:59.356836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435699242264331:2111] [node 5] ICP22 created new session: [1:7574435699242264359:2111]
2025-11-19T13:58:59.356841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435699242264359:2111] [node 5] ICS09 handshake done sender: [1:7574435699242264338:2115] self: [1:7574435699242264333:0] peer: [5:7574435699166098405:0] socket: 225 qp: -1
2025-11-19T13:58:59.356845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435699242264359:2111] [node 5] ICS10 traffic start
2025-11-19T13:58:59.356851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:59.358694Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435702332487376:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29596) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435702332487463:2118] held: no
2025-11-19T13:58:59.358857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435699166098317:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29596) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435699166098401:2118] held: no
2025-11-19T13:58:59.358891Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435699166098401:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:59.359004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435699166098317:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.359016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:59.359022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435699166098317:2048] [node 1] ICP22 created new session: [5:7574435699166098411:2048]
2025-11-19T13:58:59.359026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435699166098411:2048] [node 1] ICS09 handshake done sender: [5:7574435699166098401:2118] self: [5:7574435699166098405:0] peer: [1:7574435699242264333:0] socket: 178 qp: -1
2025-11-19T13:58:59.359031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435699166098411:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:59.359037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:59.359969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435701718797401:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.359983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:59.359988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435701718797401:2049] [node 2] ICP22 created new session: [3:7574435701718797501:2049]
2025-11-19T13:58:59.359992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435701718797501:2049] [node 2] ICS09 handshake done sender: [3:7574435701718797489:2119] self: [3:7574435701718797492:0] peer: [2:7574435700813185872:0] socket: 231 qp: -1
2025-11-19T13:58:59.359997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435701718797501:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:59.360003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:59.360378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435699166098318:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.360399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:59.360408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435699166098318:2049] [node 2] ICP22 created new session: [5:7574435699166098413:2049]
2025-11-19T13:58:59.360414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435699166098413:2049] [node 2] ICS09 handshake done sender: [5:7574435699166098384:2111] self: [5:7574435699166098399:0] peer: [2:7574435700813185874:0] socket: 194 qp: -1
2025-11-19T13:58:59.360421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435699166098413:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:59.360443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.367740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435702332487471:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44558
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.368413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435702332487377:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435702332487464:2119] held: no
2025-11-19T13:58:59.368447Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435702332487463:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.371878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435702332487376:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.371904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:59.371913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435702332487376:2048] [node 1] ICP22 created new session: [4:7574435702332487473:2048]
2025-11-19T13:58:59.371919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435702332487473:2048] [node 1] ICS09 handshake done sender: [4:7574435702332487463:2118] self: [4:7574435702332487466:0] peer: [1:7574435699242264334:0] socket: 227 qp: -1
2025-11-19T13:58:59.371927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435702332487473:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:59.371935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:59.371902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435701718797503:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52568
2025-11-19T13:58:59.372348Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435699166098318:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(27227640ns) {HandleHandshakeStatus:329(27226970ns) {DropIncomingHandshake:441(464ns) DropOutgoingHandshake:456(24ns) SwitchToState:178(168ns) ProcessPendingSessionEvents:495(372ns)}}
2025-11-19T13:58:59.372872Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435699166098406:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.372905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435699166098387:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.375680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435702332487472:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44558
2025-11-19T13:58:59.376174Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435702332487464:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:59.376253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435702332487377:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.376271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:59.376279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435702332487377:2049] [node 2] ICP22 created new session: [4:7574435702332487475:2049]
2025-11-19T13:58:59.376285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435702332487475:2049] [node 2] ICS09 handshake done sender: [4:7574435702332487464:2119] self: [4:7574435702332487467:0] peer: [2:7574435700813185885:0] socket: 233 qp: -1
2025-11-19T13:58:59.376292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435702332487475:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:59.376299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:59.380513Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435702332487458:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.380533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435702332487458:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.381499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435699166098416:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17242
2025-11-19T13:58:59.384041Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435702332487458:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:59.384258Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435699166098406:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:59.384353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435699166098387:2112] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.384369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:59.384375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435699166098387:2112] [node 4] ICP22 created new session: [5:7574435699166098417:2112]
2025-11-19T13:58:59.384380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435699166098417:2112] [node 4] ICS09 handshake done sender: [5:7574435699166098406:2121] self: [5:7574435699166098415:0] peer: [4:7574435702332487457:0] socket: 239 qp: -1
2025-11-19T13:58:59.384386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435699166098417:2112] [node 4] ICS10 traffic start
2025-11-19T13:58:59.384396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:59.385322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435702332487450:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.385344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:59.385349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435702332487450:2109] [node 5] ICP22 created new session: [4:7574435702332487478:2109]
2025-11-19T13:58:59.385354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435702332487478:2109] [node 5] ICS09 handshake done sender: [4:7574435702332487458:2115] self: [4:7574435702332487457:0] peer: [5:7574435699166098415:0] socket: 186 qp: -1
2025-11-19T13:58:59.385359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435702332487478:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:59.385364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:59.404357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:58:59.404450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435700813185896:2048] [node 1] ICS01 socket: 183 reason# EndOfStream
2025-11-19T13:58:59.404460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435700813185804:2048] [node 1] ICP30 unregister session Session# [2:7574435700813185896:2048] VirtualId# [2:7574435700813185886:0]
2025-11-19T13:58:59.404468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435700813185896:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:59.404504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:59.404524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:59.407363Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:58:59.407449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435701718797486:2048] [node 1] ICS01 socket: 206 reason# EndOfStream
2025-11-19T13:58:59.407455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435701718797400:2048] [node 1] ICP30 unregister session Session# [3:7574435701718797486:2048] VirtualId# [3:7574435701718797483:0]
2025-11-19T13:58:59.407461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435701718797486:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:59.407507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:59.407526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:59.408475Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:58:59.408567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435699956755178:2048] [node 1] ICS01 socket: 180 reason# EndOfStream
2025-11-19T13:58:59.408577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435699956755082:2048] [node 1] ICP30 unregister session Session# [6:7574435699956755178:2048] VirtualId# [6:7574435699956755167:0]
2025-11-19T13:58:59.408583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435699956755178:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:59.408671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:59.408700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:59.409124Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:58:59.409196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435702332487473:2048] [node 1] ICS01 socket: 227 reason# EndOfStream
2025-11-19T13:58:59.409202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435702332487376:2048] [node 1] ICP30 unregister session Session# [4:7574435702332487473:2048] VirtualId# [4:7574435702332487466:0]
2025-11-19T13:58:59.409208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435702332487473:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:59.409246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:59.409271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:59.409301Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:58:59.409359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435699166098411:2048] [node 1] ICS01 socket: 178 reason# EndOfStream
2025-11-19T13:58:59.409364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435699166098317:2048] [node 1] ICP30 unregister session Session# [5:7574435699166098411:2048] VirtualId# [5:7574435699166098405:0]
2025-11-19T13:58:59.409369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435699166098411:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:59.409486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:58:59.409507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:58:59.420006Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:58:59.420115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435701718797501:2049] [node 2] ICS01 socket: 231 reason# EndOfStream
2025-11-19T13:58:59.420124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435701718797401:2049] [node 2] ICP30 unregister session Session# [3:7574435701718797501:2049] VirtualId# [3:7574435701718797492:0]
2025-11-19T13:58:59.420132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435701718797501:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:59.420204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:59.420230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:59.426518Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:58:59.426607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435699956755176:2049] [node 2] ICS01 socket: 202 reason# EndOfStream
2025-11-19T13:58:59.426617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435699956755083:2049] [node 2] ICP30 unregister session Session# [6:7574435699956755176:2049] VirtualId# [6:7574435699956755170:0]
2025-11-19T13:58:59.426623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435699956755176:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:59.426671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:59.426693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:59.429888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:58:59.429968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435702332487475:2049] [node 2] ICS01 socket: 233 reason# EndOfStream
2025-11-19T13:58:59.429975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435702332487377:2049] [node 2] ICP30 unregister session Session# [4:7574435702332487475:2049] VirtualId# [4:7574435702332487467:0]
2025-11-19T13:58:59.429982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435702332487475:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:59.430030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:59.430054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:59.430089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:58:59.430151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435699166098413:2049] [node 2] ICS01 socket: 194 reason# EndOfStream
2025-11-19T13:58:59.430156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435699166098318:2049] [node 2] ICP30 unregister session Session# [5:7574435699166098413:2049] VirtualId# [5:7574435699166098399:0]
2025-11-19T13:58:59.430160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435699166098413:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:59.430186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:58:59.430205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:58:59.447980Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:58:59.448075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435699956755183:2050] [node 3] ICS01 socket: 203 reason# EndOfStream
2025-11-19T13:58:59.448084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435699956755084:2050] [node 3] ICP30 unregister session Session# [6:7574435699956755183:2050] VirtualId# [6:7574435699956755180:0]
2025-11-19T13:58:59.448091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435699956755183:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:59.448140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:59.448165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:59.449435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:58:59.449508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435699166098409:2050] [node 3] ICS01 socket: 228 reason# EndOfStream
2025-11-19T13:58:59.449514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435699166098319:2050] [node 3] ICP30 unregister session Session# [5:7574435699166098409:2050] VirtualId# [5:7574435699166098403:0]
2025-11-19T13:58:59.449519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435699166098409:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:59.449567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:58:59.449590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:58:59.479029Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:58:59.479119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435699956755168:2093] [node 4] ICS01 socket: 188 reason# EndOfStream
2025-11-19T13:58:59.479128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435699956755134:2093] [node 4] ICP30 unregister session Session# [6:7574435699956755168:2093] VirtualId# [6:7574435699956755155:0]
2025-11-19T13:58:59.479135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435699956755168:2093] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:58:59.479185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:58:59.479207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:58:59.584817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435702192023383:2048] [node 2] ICP01 ready to work
2025-11-19T13:58:59.584829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435702192023384:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:59.639989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435702029506363:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:59.640003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435702029506364:2049] [node 3] ICP01 ready to work
2025-11-19T13:58:59.684166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435702763594777:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:59.684179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435702763594778:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:59.736362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435698854085809:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:59.736380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435698854085810:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:59.736385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435698854085811:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:59.811927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435701179412497:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:59.811940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435701179412498:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:59.811945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435701179412499:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:59.863884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435701216864051:2048] [node 1] ICP01 ready to work
2025-11-19T13:58:59.863897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435701216864052:2049] [node 2] ICP01 ready to work
2025-11-19T13:58:59.863902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435701216864053:2050] [node 3] ICP01 ready to work
2025-11-19T13:58:59.891719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435698854085841:2076] [node 6] ICP01 ready to work
2025-11-19T13:58:59.891736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435702029506420:2103] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.892087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435702763594836:2104] [node 6] ICP01 ready to work
2025-11-19T13:58:59.892219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435702192023445:2107] [node 6] ICP01 ready to work
2025-11-19T13:58:59.896492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435702763594837:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19263
2025-11-19T13:58:59.896752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435701216864081:2072] [node 4] ICP01 ready to work
2025-11-19T13:58:59.898365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435701179412552:2096] [node 6] ICP01 ready to work
2025-11-19T13:58:59.899258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435702029506428:2108] [node 6] ICP01 ready to work
2025-11-19T13:58:59.899658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435702763594842:2107] [node 4] ICP01 ready to work
2025-11-19T13:58:59.899739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435702763594843:2108] [node 5] ICP01 ready to work
2025-11-19T13:58:59.900133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435701179412538:2084] [node 4] ICP01 ready to work
2025-11-19T13:58:59.907853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435702029506420:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16029956ns) {Configure:104(16029073ns) {ProcessConfigured:135(15642086ns) {StartInitialHandshake:157(15641520ns) {DropHandshakes:469(475ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(1805ns) {SwitchToState:178(893ns)}}}}}
2025-11-19T13:58:59.911874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435702763594836:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19703201ns) {Configure:104(19702263ns) {ProcessConfigured:135(18297623ns) {StartInitialHandshake:157(18296867ns) {DropHandshakes:469(999ns) {DropIncomingHandshake:441(212ns) DropOutgoingHandshake:456(203ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:476(1775ns) {SwitchToState:178(1174ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.915432Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435701216864052:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23580254ns) {Configure:104(23579636ns) {ProcessConfigured:135(23347378ns) {StartInitialHandshake:157(23347149ns) {DropHandshakes:469(478ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:476(903ns) {SwitchToState:178(106ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.914803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435702192023453:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:59.915233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435702192023451:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16399
2025-11-19T13:58:59.915248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435702192023452:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16399
2025-11-19T13:58:59.915718Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435702192023383:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23572404ns) {Configure:104(23571350ns) {ProcessConfigured:135(21401997ns) {StartInitialHandshake:157(21401503ns) {DropHandshakes:469(466ns) {DropIncomingHandshake:441(173ns) DropOutgoingHandshake:456(35ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:476(467ns) {SwitchToState:178(333ns)}}}}}
2025-11-19T13:58:59.915880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435702192023384:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(23625353ns) {Configure:104(23624979ns) {ProcessConfigured:135(21456063ns) {StartInitialHandshake:157(21455425ns) {DropHandshakes:469(189ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(18ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(945ns) {SwitchToState:178(401ns)}}}}}
2025-11-19T13:58:59.913466Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435702763594778:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(21160934ns) {Configure:104(21159762ns) {ProcessConfigured:135(19787737ns) {StartInitialHandshake:157(19786973ns) {DropHandshakes:469(277ns) {DropIncomingHandshake:441(101ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(838ns) {SwitchToState:178(386ns)}}}}}
2025-11-19T13:58:59.915526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435701216864109:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46889
2025-11-19T13:58:59.919321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435702192023446:2108] [node 4] ICP01 ready to work
2025-11-19T13:58:59.923955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435701216864081:2072] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(27140665ns) {Configure:104(27139828ns) {ProcessConfigured:135(26830645ns) {StartInitialHandshake:157(26830286ns) {DropHandshakes:469(527ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(88ns) PrepareNewSessionHandshake:476(1020ns) {SwitchToState:178(130ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.925046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435702029506364:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(27490251ns) {Configure:104(27489357ns) {ProcessConfigured:135(27210737ns) {StartInitialHandshake:157(27210189ns) {DropHandshakes:469(488ns) {DropIncomingHandshake:441(106ns) DropOutgoingHandshake:456(43ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:476(1265ns) {SwitchToState:178(62ns)}}}}}
2025-11-19T13:58:59.925202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435702029506436:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19476
2025-11-19T13:58:59.925216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435702029506437:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19476
2025-11-19T13:58:59.925870Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435702763594842:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(26084588ns) {Configure:104(26084300ns) {ProcessConfigured:135(25787496ns) {StartInitialHandshake:157(25787304ns) {DropHandshakes:469(371ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:476(999ns) {SwitchToState:178(221ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.925691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435702029506428:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26386575ns) {Configure:104(26385918ns) {ProcessConfigured:135(26124928ns) {StartInitialHandshake:157(26124574ns) {DropHandshakes:469(508ns) {DropIncomingHandshake:441(97ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(87ns) PrepareNewSessionHandshake:476(749ns) {SwitchToState:178(480ns)}}}}}
2025-11-19T13:58:59.926107Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435701216864053:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(26236712ns) {Configure:104(26235581ns) {ProcessConfigured:135(25936163ns) {StartInitialHandshake:157(25935702ns) {DropHandshakes:469(512ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(43ns)} GenerateSessionVirtualId:988(250ns) PrepareNewSessionHandshake:476(657ns) {SwitchToState:178(186ns)}}}}}
2025-11-19T13:58:59.926554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435701216864128:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46889
2025-11-19T13:58:59.926568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435701216864129:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.926770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435702029506441:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19476
2025-11-19T13:58:59.928051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435702192023445:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(28183150ns) {Configure:104(28181583ns) {ProcessConfigured:135(27870978ns) {StartInitialHandshake:157(27869990ns) {DropHandshakes:469(411ns) {DropIncomingHandshake:441(45ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(255ns) PrepareNewSessionHandshake:476(1266ns) {SwitchToState:178(968ns)}}}}}
2025-11-19T13:58:59.925394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435701179412552:2096] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26969394ns) {Configure:104(26968925ns) {ProcessConfigured:135(26724985ns) {StartInitialHandshake:157(26724696ns) {DropHandshakes:469(834ns) {DropIncomingHandshake:441(443ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(64ns) PrepareNewSessionHandshake:476(856ns) {SwitchToState:178(53ns)}}}}}
2025-11-19T13:58:59.925496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435701179412570:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60259
2025-11-19T13:58:59.926266Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435701179412499:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(30004326ns) {Configure:104(26369153ns) {ProcessConfigured:135(26074376ns) {StartInitialHandshake:157(26073864ns) {DropHandshakes:469(545ns) {DropIncomingHandshake:441(129ns) DropOutgoingHandshake:456(106ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:476(867ns) {SwitchToState:178(25ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.931851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435702029506363:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(31905115ns) {Configure:104(31904313ns) {ProcessConfigured:135(31576967ns) {StartInitialHandshake:157(31576471ns) {DropHandshakes:469(471ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(40ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(1227ns) {SwitchToState:178(614ns)}}}}}
2025-11-19T13:58:59.932085Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435698854085810:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(32054580ns) {Configure:104(32053936ns) {ProcessConfigured:135(31764771ns) {StartInitialHandshake:157(31764364ns) {DropHandshakes:469(436ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(785ns) {SwitchToState:178(169ns)}}}}}
2025-11-19T13:58:59.933017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435701216864078:2069] [node 5] ICP01 ready to work
2025-11-19T13:58:59.932121Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435701179412538:2084] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(31921308ns) {Configure:104(31920662ns) {ProcessConfigured:135(31690580ns) {StartInitialHandshake:157(31690228ns) {DropHandshakes:469(449ns) {DropIncomingHandshake:441(14ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:476(718ns) {SwitchToState:178(40ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.936117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435698854085862:2086] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.940005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435698854085841:2076] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(48198271ns) {Configure:104(48197290ns) {ProcessConfigured:135(47937749ns) {StartInitialHandshake:157(47937036ns) {DropHandshakes:469(678ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(42ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:476(1513ns) {SwitchToState:178(778ns)}}}}}
2025-11-19T13:58:59.940053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435698854085883:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58154
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.941154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435702029506443:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19476
2025-11-19T13:58:59.941250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435701216864131:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46889
2025-11-19T13:58:59.944030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435702192023446:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24639933ns) {Configure:104(24639079ns) {ProcessConfigured:135(24322891ns) {StartInitialHandshake:157(24322502ns) {DropHandshakes:469(478ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(38ns)} GenerateSessionVirtualId:988(267ns) PrepareNewSessionHandshake:476(1334ns) {SwitchToState:178(920ns)}}}}}
2025-11-19T13:58:59.944137Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435698854085809:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(51759873ns) {Configure:104(51759659ns) {ProcessConfigured:135(51464788ns) {StartInitialHandshake:157(51464478ns) {DropHandshakes:469(515ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(807ns) {SwitchToState:178(101ns)}}}}}
2025-11-19T13:58:59.944640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435698854085881:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58154
2025-11-19T13:58:59.944787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435698854085882:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58154
2025-11-19T13:58:59.947788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435702029506434:2109] [node 5] ICP01 ready to work
2025-11-19T13:58:59.940752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435702763594843:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(40635833ns) {Configure:104(24399216ns) {ProcessConfigured:135(24082889ns) {StartInitialHandshake:157(24080982ns) {DropHandshakes:469(549ns) {DropIncomingHandshake:441(98ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(1185ns) {SwitchToState:178(741ns)}}}}}
2025-11-19T13:58:59.941042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435702763594856:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19263
2025-11-19T13:58:59.941054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435702763594857:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19263
2025-11-19T13:58:59.941064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435702763594858:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19263
2025-11-19T13:58:59.943904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435702763594777:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(26925849ns) {Configure:104(26924530ns) {ProcessConfigured:135(26639385ns) {StartInitialHandshake:157(26638840ns) {DropHandshakes:469(417ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:476(1369ns) {SwitchToState:178(1176ns)}}}}}
2025-11-19T13:58:59.951856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435701216864078:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(18756176ns) {Configure:104(18755247ns) {ProcessConfigured:135(18415560ns) {StartInitialHandshake:157(18415006ns) {DropHandshakes:469(883ns) {DropIncomingHandshake:441(188ns) DropOutgoingHandshake:456(152ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(1639ns) {SwitchToState:178(1423ns)}}}}}
2025-11-19T13:58:59.955446Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435701179412498:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(59429756ns) {Configure:104(59429309ns) {ProcessConfigured:135(59128847ns) {StartInitialHandshake:157(59128102ns) {DropHandshakes:469(407ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:476(1372ns) {SwitchToState:178(734ns)}}}}}
2025-11-19T13:58:59.958891Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435698854085881:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.959003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435698854085810:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.960902Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435702029506443:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.965058Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435702763594857:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.965109Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435702763594837:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.965194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435702763594778:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.965473Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435702763594858:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.962734Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435701216864129:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.962768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435701216864052:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.964086Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435701216864131:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.964114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435701216864081:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.966170Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435698854085884:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.966181Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435698854085884:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.966857Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435702029506439:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.966868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435702029506439:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.967320Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435702029506435:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.967323Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435702029506435:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.967981Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435698854085884:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:59.968082Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435702029506439:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:59.968022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435702763594863:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19263
2025-11-19T13:58:59.968097Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435702029506435:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.968156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435702029506364:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.968179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:58:59.968186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435702029506364:2049] [node 3] ICP22 created new session: [2:7574435702029506449:2049]
2025-11-19T13:58:59.968364Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435702192023451:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.968193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435702029506449:2049] [node 3] ICS09 handshake done sender: [2:7574435702029506435:2111] self: [2:7574435702029506426:0] peer: [3:7574435702763594860:0] socket: 197 qp: -1
2025-11-19T13:58:59.968200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435702029506449:2049] [node 3] ICS10 traffic start
2025-11-19T13:58:59.968208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:58:59.968334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435702029506428:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.968344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:59.968350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435702029506428:2108] [node 6] ICP22 created new session: [2:7574435702029506452:2108]
2025-11-19T13:58:59.968355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435702029506452:2108] [node 6] ICS09 handshake done sender: [2:7574435702029506439:2114] self: [2:7574435702029506429:0] peer: [6:7574435701216864133:0] socket: 200 qp: -1
2025-11-19T13:58:59.968361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435702029506452:2108] [node 6] ICS10 traffic start
2025-11-19T13:58:59.968377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.968602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435701179412576:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60259
2025-11-19T13:58:59.968734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435701216864135:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46889
2025-11-19T13:58:59.968793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435702192023464:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16399
2025-11-19T13:58:59.968838Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435698854085882:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.969002Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435702763594857:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:59.969132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435702763594778:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.969146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:59.969151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435702763594778:2049] [node 2] ICP22 created new session: [3:7574435702763594864:2049]
2025-11-19T13:58:59.969157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435702763594864:2049] [node 2] ICS09 handshake done sender: [3:7574435702763594857:2116] self: [3:7574435702763594860:0] peer: [2:7574435702029506426:0] socket: 212 qp: -1
2025-11-19T13:58:59.969172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435702763594864:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:59.969187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.971033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435701216864136:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46889
2025-11-19T13:58:59.971192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435698854085841:2076] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.971221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:59.971226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435698854085841:2076] [node 6] ICP22 created new session: [4:7574435698854085893:2076]
2025-11-19T13:58:59.971231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435698854085893:2076] [node 6] ICS09 handshake done sender: [4:7574435698854085884:2115] self: [4:7574435698854085844:0] peer: [6:7574435701216864134:0] socket: 211 qp: -1
2025-11-19T13:58:59.971237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435698854085893:2076] [node 6] ICS10 traffic start
2025-11-19T13:58:59.971252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:59.971615Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435701216864128:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.971635Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435701216864129:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:59.973083Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435702192023464:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.973480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435701179412577:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60259
2025-11-19T13:58:59.973656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435701216864078:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.973671Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435701216864131:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:59.973783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435701216864052:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.973794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:59.973799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435701216864052:2049] [node 2] ICP22 created new session: [6:7574435701216864138:2049]
2025-11-19T13:58:59.973804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435701216864138:2049] [node 2] ICS09 handshake done sender: [6:7574435701216864129:2116] self: [6:7574435701216864133:0] peer: [2:7574435702029506429:0] socket: 205 qp: -1
2025-11-19T13:58:59.973809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435701216864138:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:59.973823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:59.974057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435701179412569:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.974067Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435701179412569:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.974293Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435701179412576:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.974318Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435701179412577:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.974339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435701179412498:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.974349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435701179412538:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.974534Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435702029506445:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.974543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435702029506445:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.975041Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435701179412569:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.975179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435701179412552:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.975191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:59.975196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435701179412552:2096] [node 6] ICP22 created new session: [5:7574435701179412582:2096]
2025-11-19T13:58:59.975201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435701179412582:2096] [node 6] ICS09 handshake done sender: [5:7574435701179412569:2111] self: [5:7574435701179412555:0] peer: [6:7574435701216864137:0] socket: 199 qp: -1
2025-11-19T13:58:59.975205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435701179412582:2096] [node 6] ICS10 traffic start
2025-11-19T13:58:59.975211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:59.975331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435701179412581:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60259
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.975490Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435698854085887:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.975497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435698854085887:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.976221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435698854085896:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58154
2025-11-19T13:58:59.976392Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435702029506445:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:59.976551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435701216864081:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.976565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:59.976570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435701216864081:2072] [node 4] ICP22 created new session: [6:7574435701216864141:2072]
2025-11-19T13:58:59.976576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435701216864141:2072] [node 4] ICS09 handshake done sender: [6:7574435701216864131:2117] self: [6:7574435701216864134:0] peer: [4:7574435698854085844:0] socket: 215 qp: -1
2025-11-19T13:58:59.976580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435701216864141:2072] [node 4] ICS10 traffic start
2025-11-19T13:58:59.976588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:59.977368Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435698854085887:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:59.977526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435702029506434:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.977539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:59.977544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435702029506434:2109] [node 5] ICP22 created new session: [2:7574435702029506456:2109]
2025-11-19T13:58:59.977548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435702029506456:2109] [node 5] ICS09 handshake done sender: [2:7574435702029506445:2118] self: [2:7574435702029506444:0] peer: [5:7574435701179412578:0] socket: 221 qp: -1
2025-11-19T13:58:59.977553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435702029506456:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:59.977558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:59.978785Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435701179412576:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.979318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435702192023467:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16399
2025-11-19T13:58:59.979017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435698854085862:2086] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.979028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:59.979034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435698854085862:2086] [node 5] ICP22 created new session: [4:7574435698854085898:2086]
2025-11-19T13:58:59.979039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435698854085898:2086] [node 5] ICS09 handshake done sender: [4:7574435698854085887:2116] self: [4:7574435698854085886:0] peer: [5:7574435701179412579:0] socket: 226 qp: -1
2025-11-19T13:58:59.979042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435698854085898:2086] [node 5] ICS10 traffic start
2025-11-19T13:58:59.979048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.979500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435701216864140:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.979937Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435701216864128:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:59.979820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435701179412585:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60259
2025-11-19T13:58:59.980109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435701216864078:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.980122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:59.980127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435701216864078:2069] [node 5] ICP22 created new session: [6:7574435701216864144:2069]
2025-11-19T13:58:59.980133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435701216864144:2069] [node 5] ICS09 handshake done sender: [6:7574435701216864128:2115] self: [6:7574435701216864137:0] peer: [5:7574435701179412555:0] socket: 204 qp: -1
2025-11-19T13:58:59.980137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435701216864144:2069] [node 5] ICS10 traffic start
2025-11-19T13:58:59.980143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.980520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435702763594866:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19263
2025-11-19T13:58:59.980405Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435701179412577:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:59.980901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435701179412538:2084] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.980918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:59.980924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435701179412538:2084] [node 4] ICP22 created new session: [5:7574435701179412586:2084]
2025-11-19T13:58:59.980929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435701179412586:2084] [node 4] ICS09 handshake done sender: [5:7574435701179412577:2117] self: [5:7574435701179412579:0] peer: [4:7574435698854085886:0] socket: 227 qp: -1
2025-11-19T13:58:59.980935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435701179412586:2084] [node 4] ICS10 traffic start
2025-11-19T13:58:59.980944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:59.981174Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435698854085896:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.981260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435698854085862:2086] [node 5] ICP18 handshake (actor [4:7574435698854085896:2118]) is too old
2025-11-19T13:58:59.981728Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435702192023467:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.984245Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435701179412570:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.984296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435701179412497:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.985002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435701179412589:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60259
2025-11-19T13:58:59.985218Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435701179412570:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:58:59.985269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435701179412497:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.985283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:58:59.985289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435701179412497:2048] [node 1] ICP22 created new session: [5:7574435701179412590:2048]
2025-11-19T13:58:59.985295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435701179412590:2048] [node 1] ICS09 handshake done sender: [5:7574435701179412570:2112] self: [5:7574435701179412588:0] peer: [1:7574435702192023454:0] socket: 198 qp: -1
2025-11-19T13:58:59.985300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435701179412590:2048] [node 1] ICS10 traffic start
2025-11-19T13:58:59.985314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:58:59.984446Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435702192023455:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.984457Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435702192023455:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.985123Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435702192023455:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:58:59.986364Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435702029506432:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.986380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435702029506432:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.985472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435702192023453:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.985492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:58:59.985497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435702192023453:2109] [node 5] ICP22 created new session: [1:7574435702192023470:2109]
2025-11-19T13:58:59.985503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435702192023470:2109] [node 5] ICS09 handshake done sender: [1:7574435702192023455:2112] self: [1:7574435702192023454:0] peer: [5:7574435701179412588:0] socket: 188 qp: -1
2025-11-19T13:58:59.985508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435702192023470:2109] [node 5] ICS10 traffic start
2025-11-19T13:58:59.985524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:58:59.986964Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435698854085883:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.986995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435698854085811:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:58:59.987328Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435702192023452:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.987445Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435702763594866:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.990168Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435701216864109:2098] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.989936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435701179412498:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.989955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:59.989963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435701179412498:2049] [node 2] ICP22 created new session: [5:7574435701179412592:2049]
2025-11-19T13:58:59.989969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435701179412592:2049] [node 2] ICS09 handshake done sender: [5:7574435701179412576:2116] self: [5:7574435701179412578:0] peer: [2:7574435702029506444:0] socket: 223 qp: -1
2025-11-19T13:58:59.989976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435701179412592:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:59.989984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.993776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435698854085902:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58154
2025-11-19T13:58:59.993979Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435702029506432:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:58:59.994098Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435698854085881:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:58:59.994183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435698854085810:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:58:59.994200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:58:59.994207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435698854085810:2049] [node 2] ICP22 created new session: [4:7574435698854085903:2049]
2025-11-19T13:58:59.994214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435698854085903:2049] [node 2] ICS09 handshake done sender: [4:7574435698854085881:2112] self: [4:7574435698854085890:0] peer: [2:7574435702029506423:0] socket: 207 qp: -1
2025-11-19T13:58:59.994220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435698854085903:2049] [node 2] ICS10 traffic start
2025-11-19T13:58:59.994227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:58:59.994465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435702029506420:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.994484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:58:59.994489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435702029506420:2103] [node 4] ICP22 created new session: [2:7574435702029506459:2103]
2025-11-19T13:58:59.994494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435702029506459:2103] [node 4] ICS09 handshake done sender: [2:7574435702029506432:2110] self: [2:7574435702029506423:0] peer: [4:7574435698854085890:0] socket: 185 qp: -1
2025-11-19T13:58:59.994499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435702029506459:2103] [node 4] ICS10 traffic start
2025-11-19T13:58:59.994505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:58:59.994984Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435702763594853:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.995001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435702763594853:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:58:59.995201Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435702763594847:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.995207Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435702763594847:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:58:59.995967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435701216864147:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46889
2025-11-19T13:58:59.996022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435701216864148:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46889
2025-11-19T13:58:59.996976Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435702763594847:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:58:59.997104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435702763594836:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:58:59.997120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:58:59.997126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435702763594836:2104] [node 6] ICP22 created new session: [3:7574435702763594869:2104]
2025-11-19T13:58:59.997132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435702763594869:2104] [node 6] ICS09 handshake done sender: [3:7574435702763594847:2111] self: [3:7574435702763594838:0] peer: [6:7574435701216864146:0] socket: 186 qp: -1
2025-11-19T13:58:59.997137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435702763594869:2104] [node 6] ICS10 traffic start
2025-11-19T13:58:59.997145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:58:59.997379Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435701216864147:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:58:59.997409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435701216864051:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.012284Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435701179412498:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(22301321ns) {HandleHandshakeStatus:329(22300399ns) {DropIncomingHandshake:441(218ns) DropOutgoingHandshake:456(2204ns) SwitchToState:178(144ns) ProcessPendingSessionEvents:495(22208032ns) {ForwardSessionEventToSession:612(22190473ns) ForwardSessionEventToSession:612(3432ns) ForwardSessionEventToSession:612(599ns) ForwardSessionEventToSession:612(2271ns) ForwardSessionEventToSession:612(788ns) ForwardSessionEventToSession:612(614ns) ForwardSessionEventToSession:612(664ns) ForwardSessionEventToSession:612(709ns) ForwardSessionEventToSession:612(836ns) ForwardSessionEventToSession:612(952ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.013918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435705474379890:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60259
2025-11-19T13:59:00.013776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435703149053201:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58154
2025-11-19T13:59:00.018126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435701216864053:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.018172Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435701216864053:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(27944556ns) {IncomingHandshake:261(27944052ns) {DropIncomingHandshake:441(169ns) PrepareNewSessionHandshake:476(370ns) {SwitchToState:178(78ns)} GenerateSessionVirtualId:988(207ns) IssueIncomingHandshakeReply:193(27939684ns) {SwitchToState:178(743ns)}}}
2025-11-19T13:59:00.018208Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435701216864109:2098] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:00.018346Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435702192023462:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.018365Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435702192023462:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.018962Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435702763594853:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:00.019520Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435698854085883:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:00.020535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435701216864053:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.020578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:00.020586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435701216864053:2050] [node 3] ICP22 created new session: [6:7574435705511831446:2050]
2025-11-19T13:59:00.020593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435705511831446:2050] [node 3] ICS09 handshake done sender: [6:7574435701216864109:2098] self: [6:7574435701216864146:0] peer: [3:7574435702763594838:0] socket: 189 qp: -1
2025-11-19T13:59:00.020601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435705511831446:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:00.020611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:00.021446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435702763594842:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.021467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:00.021474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435702763594842:2107] [node 4] ICP22 created new session: [3:7574435707058562169:2107]
2025-11-19T13:59:00.021480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435707058562169:2107] [node 4] ICS09 handshake done sender: [3:7574435702763594853:2113] self: [3:7574435702763594844:0] peer: [4:7574435698854085901:0] socket: 202 qp: -1
2025-11-19T13:59:00.021486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435707058562169:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:00.021505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:00.022391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435698854085811:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.022406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:00.022411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435698854085811:2050] [node 3] ICP22 created new session: [4:7574435703149053202:2050]
2025-11-19T13:59:00.022416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435703149053202:2050] [node 3] ICS09 handshake done sender: [4:7574435698854085883:2114] self: [4:7574435698854085901:0] peer: [3:7574435702763594844:0] socket: 209 qp: -1
2025-11-19T13:59:00.022421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435703149053202:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:00.022427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:00.024317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435702763594836:2104] [node 6] ICP35 event processing took too much time PendingConnection:278(27181605ns) {HandleHandshakeStatus:329(27180526ns) {DropIncomingHandshake:441(2860ns) DropOutgoingHandshake:456(150ns) SwitchToState:178(158ns) ProcessPendingSessionEvents:495(5983852ns) {ForwardSessionEventToSession:612(5971514ns) ForwardSessionEventToSession:612(1429ns) ForwardSessionEventToSession:612(792ns) ForwardSessionEventToSession:612(655ns) ForwardSessionEventToSession:612(663ns) ForwardSessionEventToSession:612(1202ns) ForwardSessionEventToSession:612(980ns) ForwardSessionEventToSession:612(558ns) ForwardSessionEventToSession:612(556ns) ForwardSessionEventToSession:612(877ns)}}}
2025-11-19T13:59:00.026756Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435705474379890:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.026811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435701179412499:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.027929Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435702192023462:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:00.031212Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435702029506437:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.031266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435702029506363:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.031074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435702192023445:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.031101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:00.031108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435702192023445:2107] [node 6] ICP22 created new session: [1:7574435706486990771:2107]
2025-11-19T13:59:00.031114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435706486990771:2107] [node 6] ICS09 handshake done sender: [1:7574435702192023462:2115] self: [1:7574435702192023449:0] peer: [6:7574435701216864149:0] socket: 240 qp: -1
2025-11-19T13:59:00.031120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435706486990771:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:00.031128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.037553Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435702192023457:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.037565Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435702192023457:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.038245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435706486990773:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16399
2025-11-19T13:59:00.039039Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435702192023457:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.039205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435703149053204:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58154
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.039488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435706324473758:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19476
2025-11-19T13:59:00.039394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435702192023383:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.039419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:00.039426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435702192023383:2048] [node 2] ICP22 created new session: [1:7574435706486990776:2048]
2025-11-19T13:59:00.039431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435706486990776:2048] [node 2] ICS09 handshake done sender: [1:7574435702192023457:2113] self: [1:7574435702192023447:0] peer: [2:7574435706324473757:0] socket: 191 qp: -1
2025-11-19T13:59:00.039446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435706486990776:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:00.039455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:00.040141Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435702763594855:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.040157Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435702763594855:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.040295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435702029506428:2108] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:00.040396Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435702029506437:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:00.040468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435702029506363:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.040485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:00.040492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435702029506363:2048] [node 1] ICP22 created new session: [2:7574435706324473759:2048]
2025-11-19T13:59:00.040498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435706324473759:2048] [node 1] ICS09 handshake done sender: [2:7574435702029506437:2113] self: [2:7574435706324473757:0] peer: [1:7574435702192023447:0] socket: 196 qp: -1
2025-11-19T13:59:00.040504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435706324473759:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:00.040510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:00.046387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435701216864051:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3904) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435701216864147:2122] held: no
2025-11-19T13:59:00.047061Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435702763594856:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.047115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435702763594777:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.048670Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435702029506364:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:00.049059Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435706486990773:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.049809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435705511831448:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46889
2025-11-19T13:59:00.050079Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435701216864147:2122] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:00.050193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435701216864051:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.050208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:00.050216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435701216864051:2048] [node 1] ICP22 created new session: [6:7574435705511831449:2048]
2025-11-19T13:59:00.050222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435705511831449:2048] [node 1] ICS09 handshake done sender: [6:7574435701216864147:2122] self: [6:7574435701216864149:0] peer: [1:7574435702192023449:0] socket: 241 qp: -1
2025-11-19T13:59:00.050228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435705511831449:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:00.050237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:00.050603Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435703149053204:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.050648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435698854085809:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.050847Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435702192023463:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.050854Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435702192023463:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.051606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435703149053206:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58154
2025-11-19T13:59:00.051870Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435702192023463:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:00.052056Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435703149053204:2121] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:00.052012Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435702763594855:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.052176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435698854085809:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.052194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:00.052199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435698854085809:2048] [node 1] ICP22 created new session: [4:7574435703149053207:2048]
2025-11-19T13:59:00.052205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435703149053207:2048] [node 1] ICS09 handshake done sender: [4:7574435703149053204:2121] self: [4:7574435703149053205:0] peer: [1:7574435702192023461:0] socket: 232 qp: -1
2025-11-19T13:59:00.052211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435703149053207:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:00.052221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:00.053008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435705474379892:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60259
2025-11-19T13:59:00.053194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435702192023446:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.053212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:00.053222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435702192023446:2108] [node 4] ICP22 created new session: [1:7574435706486990780:2108]
2025-11-19T13:59:00.053229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435706486990780:2108] [node 4] ICS09 handshake done sender: [1:7574435702192023463:2116] self: [1:7574435702192023461:0] peer: [4:7574435703149053205:0] socket: 183 qp: -1
2025-11-19T13:59:00.053234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435706486990780:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:00.053240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:00.053237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435702763594843:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.053262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:00.053267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435702763594843:2108] [node 5] ICP22 created new session: [3:7574435707058562174:2108]
2025-11-19T13:59:00.053273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435707058562174:2108] [node 5] ICS09 handshake done sender: [3:7574435702763594855:2114] self: [3:7574435702763594850:0] peer: [5:7574435705474379891:0] socket: 246 qp: -1
2025-11-19T13:59:00.053280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435707058562174:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:00.053288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:00.053820Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435705474379890:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:00.053968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435701179412499:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.053988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:00.053993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435701179412499:2050] [node 3] ICP22 created new session: [5:7574435705474379893:2050]
2025-11-19T13:59:00.053999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435705474379893:2050] [node 3] ICS09 handshake done sender: [5:7574435705474379890:2121] self: [5:7574435705474379891:0] peer: [3:7574435702763594850:0] socket: 248 qp: -1
2025-11-19T13:59:00.054006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435705474379893:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:00.054019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.082867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435707058562176:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19263
2025-11-19T13:59:00.081922Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435702192023459:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.081945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435702192023459:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.083662Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435702192023459:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:00.083805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435702192023384:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.083836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:00.083843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435702192023384:2049] [node 3] ICP22 created new session: [1:7574435706486990783:2049]
2025-11-19T13:59:00.083850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435706486990783:2049] [node 3] ICS09 handshake done sender: [1:7574435702192023459:2114] self: [1:7574435702192023448:0] peer: [3:7574435707058562172:0] socket: 192 qp: -1
2025-11-19T13:59:00.083857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435706486990783:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:00.083869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:00.089280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435702763594777:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3904) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435702763594856:2115] held: no
2025-11-19T13:59:00.089717Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435702763594856:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:00.089838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435702763594777:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.089856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:00.089864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435702763594777:2048] [node 1] ICP22 created new session: [3:7574435707058562177:2048]
2025-11-19T13:59:00.089871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435707058562177:2048] [node 1] ICS09 handshake done sender: [3:7574435702763594856:2115] self: [3:7574435707058562172:0] peer: [1:7574435702192023448:0] socket: 193 qp: -1
2025-11-19T13:59:00.089880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435707058562177:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:00.089906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:00.096347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:00.096420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435707058562177:2048] [node 1] ICS01 socket: 193 reason# EndOfStream
2025-11-19T13:59:00.096428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435702763594777:2048] [node 1] ICP30 unregister session Session# [3:7574435707058562177:2048] VirtualId# [3:7574435707058562172:0]
2025-11-19T13:59:00.096434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435707058562177:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.096479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:00.096130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:00.096234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574435706324473759:2048] [node 1] ICS07 socket disconnect 196 reason# EndOfStream
2025-11-19T13:59:00.096241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435706324473759:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.096296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:00.096319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435706324473759:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:00.096499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:00.098043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435702029506363:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3904) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:00.098055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435706324473759:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:00.101326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:00.101414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435701179412590:2048] [node 1] ICS01 socket: 198 reason# EndOfStream
2025-11-19T13:59:00.101421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435701179412497:2048] [node 1] ICP30 unregister session Session# [5:7574435701179412590:2048] VirtualId# [5:7574435701179412588:0]
2025-11-19T13:59:00.101429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435701179412590:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.101478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:00.101501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:00.110228Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:00.110333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435705511831449:2048] [node 1] ICS01 socket: 241 reason# EndOfStream
2025-11-19T13:59:00.110343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435701216864051:2048] [node 1] ICP30 unregister session Session# [6:7574435705511831449:2048] VirtualId# [6:7574435701216864149:0]
2025-11-19T13:59:00.110351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435705511831449:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.110413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:00.110437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:00.110476Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:00.110535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435703149053207:2048] [node 1] ICS01 socket: 232 reason# EndOfStream
2025-11-19T13:59:00.110540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435698854085809:2048] [node 1] ICP30 unregister session Session# [4:7574435703149053207:2048] VirtualId# [4:7574435703149053205:0]
2025-11-19T13:59:00.110544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435703149053207:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.110570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:00.110591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:00.124336Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:00.124432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435702763594864:2049] [node 2] ICS01 socket: 212 reason# EndOfStream
2025-11-19T13:59:00.124442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435702763594778:2049] [node 2] ICP30 unregister session Session# [3:7574435702763594864:2049] VirtualId# [3:7574435702763594860:0]
2025-11-19T13:59:00.124451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435702763594864:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.124483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:00.124503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:00.127559Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:00.127649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435701179412592:2049] [node 2] ICS07 socket disconnect 223 reason# EndOfStream
2025-11-19T13:59:00.127655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435701179412592:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.127718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:00.127739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435701179412592:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:00.129251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435701179412498:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5196) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:00.129266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435701179412592:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:00.129271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435701179412498:2049] [node 2] ICP30 unregister session Session# [5:7574435701179412592:2049] VirtualId# [5:7574435701179412578:0]
2025-11-19T13:59:00.129284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:00.129289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435701179412498:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:00.129292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:00.128453Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:00.128546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435698854085903:2049] [node 2] ICS01 socket: 207 reason# EndOfStream
2025-11-19T13:59:00.128550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435698854085810:2049] [node 2] ICP30 unregister session Session# [4:7574435698854085903:2049] VirtualId# [4:7574435698854085890:0]
2025-11-19T13:59:00.128556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435698854085903:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.128583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:00.128603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:00.128245Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:00.128323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435701216864138:2049] [node 2] ICS01 socket: 205 reason# EndOfStream
2025-11-19T13:59:00.128330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435701216864052:2049] [node 2] ICP30 unregister session Session# [6:7574435701216864138:2049] VirtualId# [6:7574435701216864133:0]
2025-11-19T13:59:00.128338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435701216864138:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.128392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:00.128417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:00.140716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:00.142215Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:00.142308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435705511831446:2050] [node 3] ICS01 socket: 189 reason# EndOfStream
2025-11-19T13:59:00.142316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435701216864053:2050] [node 3] ICP30 unregister session Session# [6:7574435705511831446:2050] VirtualId# [6:7574435701216864146:0]
2025-11-19T13:59:00.142322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435705511831446:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.142366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:00.142390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:00.140839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435705474379893:2050] [node 3] ICS01 socket: 248 reason# EndOfStream
2025-11-19T13:59:00.140854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435701179412499:2050] [node 3] ICP30 unregister session Session# [5:7574435705474379893:2050] VirtualId# [5:7574435705474379891:0]
2025-11-19T13:59:00.140862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435705474379893:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.140923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:00.140948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:00.141362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435701179412498:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:00.166645Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:00.166739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435701216864141:2072] [node 4] ICS01 socket: 215 reason# EndOfStream
2025-11-19T13:59:00.166749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435701216864081:2072] [node 4] ICP30 unregister session Session# [6:7574435701216864141:2072] VirtualId# [6:7574435701216864134:0]
2025-11-19T13:59:00.166756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435701216864141:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.166821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:00.166845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:00.307913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435704919133530:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:00.307934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435704919133531:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:00.380181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435706121613955:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:00.380197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435706121613956:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:00.472173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435703888627466:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:00.472192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435703888627467:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:00.540005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435704823898171:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:00.540025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435704823898172:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:00.540030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435704823898173:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:00.577550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435706467163811:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:00.577564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435706467163812:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:00.577578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435706467163813:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:00.624337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435706664434551:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:00.624352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435706664434552:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:00.624356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435706664434553:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:00.630103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435706664434566:2060] [node 5] ICP01 ready to work
2025-11-19T13:59:00.630426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435703888627518:2098] [node 6] ICP01 ready to work
2025-11-19T13:59:00.631248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435704919133588:2104] [node 6] ICP01 ready to work
2025-11-19T13:59:00.630308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435706467163844:2079] [node 6] ICP01 ready to work
2025-11-19T13:59:00.637029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435703888627528:2101] [node 4] ICP01 ready to work
2025-11-19T13:59:00.637126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435703888627529:2103] [node 5] ICP01 ready to work
2025-11-19T13:59:00.636948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435706121614010:2096] [node 6] ICP01 ready to work
2025-11-19T13:59:00.638069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435704823898205:2070] [node 6] ICP01 ready to work
2025-11-19T13:59:00.638156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435704823898206:2074] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.651202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435706121614025:2104] [node 5] ICP01 ready to work
2025-11-19T13:59:00.651288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435706121614026:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:00.655244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435703888627541:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55650
2025-11-19T13:59:00.655925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435704823898172:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17725086ns) {Configure:104(17724714ns) {ProcessConfigured:135(17339845ns) {StartInitialHandshake:157(17339565ns) {DropHandshakes:469(541ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(25ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:476(585ns) {SwitchToState:178(188ns)}}}}}
2025-11-19T13:59:00.660287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435706664434551:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(29671225ns) {Configure:104(29670669ns) {ProcessConfigured:135(29446914ns) {StartInitialHandshake:157(29446405ns) {DropHandshakes:469(298ns) {DropIncomingHandshake:441(45ns) DropOutgoingHandshake:456(43ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:476(972ns) {SwitchToState:178(559ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.667894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435706467163813:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(36996620ns) {Configure:104(36995960ns) {ProcessConfigured:135(36755395ns) {StartInitialHandshake:157(36754896ns) {DropHandshakes:469(441ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:476(1585ns) {SwitchToState:178(1015ns)}}}}}
2025-11-19T13:59:00.668082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435706467163879:2085] [node 4] ICP01 ready to work
2025-11-19T13:59:00.669156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435706467163880:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54600
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.676386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435706664434622:2072] [node 4] ICP01 ready to work
2025-11-19T13:59:00.677308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435706664434619:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29452
2025-11-19T13:59:00.677386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435706664434620:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29452
2025-11-19T13:59:00.677397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435706664434621:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29452
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.681758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435703888627546:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55650
2025-11-19T13:59:00.684008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435704823898246:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6468
2025-11-19T13:59:00.684080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435704823898247:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6468
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.685559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435704919133588:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(54246755ns) {Configure:104(54245300ns) {ProcessConfigured:135(53810733ns) {StartInitialHandshake:157(53809356ns) {DropHandshakes:469(434ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(123ns) PrepareNewSessionHandshake:476(1758ns) {SwitchToState:178(1449ns)}}}}}
2025-11-19T13:59:00.685708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435704919133596:2105] [node 4] ICP01 ready to work
2025-11-19T13:59:00.685790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435704919133597:2106] [node 5] ICP01 ready to work
2025-11-19T13:59:00.686942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435704919133598:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36701
2025-11-19T13:59:00.686958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435704919133599:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36701
2025-11-19T13:59:00.686969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435704919133600:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36701
2025-11-19T13:59:00.689682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435706121614034:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24666
2025-11-19T13:59:00.689703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435706121614035:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24666
2025-11-19T13:59:00.695981Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435704919133598:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.696624Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435706664434619:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.700208Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435703888627541:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.700253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435703888627467:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.703955Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435703888627536:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.703968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435703888627536:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.704842Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435703888627536:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:00.704995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435703888627528:2101] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.705014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:00.705022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435703888627528:2101] [node 4] ICP22 created new session: [3:7574435703888627549:2101]
2025-11-19T13:59:00.705028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435703888627549:2101] [node 4] ICS09 handshake done sender: [3:7574435703888627536:2111] self: [3:7574435703888627531:0] peer: [4:7574435704823898252:0] socket: 191 qp: -1
2025-11-19T13:59:00.705036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435703888627549:2101] [node 4] ICS10 traffic start
2025-11-19T13:59:00.705045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:00.699985Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435704823898246:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.703099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435704823898251:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6468
2025-11-19T13:59:00.703120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435704823898173:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.705806Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435706121614022:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.705821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435706121614022:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.707209Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435706121614022:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:00.707324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435706121613956:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.707340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:00.707347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435706121613956:2049] [node 3] ICP22 created new session: [2:7574435706121614037:2049]
2025-11-19T13:59:00.707354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435706121614037:2049] [node 3] ICS09 handshake done sender: [2:7574435706121614022:2111] self: [2:7574435706121614021:0] peer: [3:7574435703888627547:0] socket: 188 qp: -1
2025-11-19T13:59:00.707360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435706121614037:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:00.707369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:00.706623Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435706467163880:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.706676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435706467163890:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54600
2025-11-19T13:59:00.706705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435706467163891:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54600
2025-11-19T13:59:00.706724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435706467163813:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.709781Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435706664434620:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.709905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435706664434553:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.709935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435706664434632:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29452
2025-11-19T13:59:00.709974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435706664434633:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29452
2025-11-19T13:59:00.710056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435706664434552:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.710285Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435706121614034:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.710191Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435703888627538:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.710203Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435703888627538:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.710759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435703888627552:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55650
2025-11-19T13:59:00.711089Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435703888627520:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.711096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435703888627520:2099] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.711692Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435703888627541:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:00.711945Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435703888627538:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.712226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435703888627467:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.712248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:00.712256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435703888627467:2049] [node 2] ICP22 created new session: [3:7574435703888627555:2049]
2025-11-19T13:59:00.712262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435703888627555:2049] [node 2] ICS09 handshake done sender: [3:7574435703888627541:2114] self: [3:7574435703888627547:0] peer: [2:7574435706121614021:0] socket: 190 qp: -1
2025-11-19T13:59:00.712271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435703888627555:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:00.712278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:00.712473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435703888627529:2103] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.712479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:00.712484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435703888627529:2103] [node 5] ICP22 created new session: [3:7574435703888627557:2103]
2025-11-19T13:59:00.712491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435703888627557:2103] [node 5] ICS09 handshake done sender: [3:7574435703888627538:2112] self: [3:7574435703888627537:0] peer: [5:7574435706467163892:0] socket: 192 qp: -1
2025-11-19T13:59:00.712495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435703888627557:2103] [node 5] ICS10 traffic start
2025-11-19T13:59:00.712504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:00.712096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435706664434636:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29452
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.719775Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435706121614020:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.719792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435706121614020:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.722123Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435706121614020:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:00.722236Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435706121614035:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.722289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435706121614010:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.722300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:00.722306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435706121614010:2096] [node 6] ICP22 created new session: [2:7574435706121614042:2096]
2025-11-19T13:59:00.722312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435706121614042:2096] [node 6] ICS09 handshake done sender: [2:7574435706121614020:2110] self: [2:7574435706121614011:0] peer: [6:7574435706664434635:0] socket: 187 qp: -1
2025-11-19T13:59:00.722316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435706121614042:2096] [node 6] ICS10 traffic start
2025-11-19T13:59:00.722322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:00.722981Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435703888627520:2099] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:00.720844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435704823898253:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6468
2025-11-19T13:59:00.721611Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435704823898246:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:00.721664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435704823898173:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.721677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:00.721682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435704823898173:2050] [node 3] ICP22 created new session: [4:7574435704823898254:2050]
2025-11-19T13:59:00.721688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435704823898254:2050] [node 3] ICS09 handshake done sender: [4:7574435704823898246:2115] self: [4:7574435704823898252:0] peer: [3:7574435703888627531:0] socket: 193 qp: -1
2025-11-19T13:59:00.721695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435704823898254:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:00.721702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:00.723209Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435703888627546:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.723316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435703888627518:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.723325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:00.723330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435703888627518:2098] [node 6] ICP22 created new session: [3:7574435703888627559:2098]
2025-11-19T13:59:00.723335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435703888627559:2098] [node 6] ICS09 handshake done sender: [3:7574435703888627520:2099] self: [3:7574435703888627519:0] peer: [6:7574435706664434634:0] socket: 186 qp: -1
2025-11-19T13:59:00.723340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435703888627559:2098] [node 6] ICS10 traffic start
2025-11-19T13:59:00.723345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:00.721268Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435706467163890:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.721351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435706467163893:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54600
2025-11-19T13:59:00.721384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435706467163894:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54600
2025-11-19T13:59:00.724903Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435706467163893:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.724559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435703888627529:2103] [node 5] ICP18 handshake (actor [3:7574435703888627546:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.728610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435704919133612:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36701
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.728854Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435704823898247:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.728888Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435704823898251:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.728932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435704823898257:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6468
2025-11-19T13:59:00.728969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435704823898171:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.729138Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435704823898257:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.729152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435704823898172:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.729886Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435706664434632:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.729912Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435706664434621:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.729950Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435706121614030:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.729962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435706121614030:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.731356Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435704919133612:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.731394Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435704919133600:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.731421Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435704919133602:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.731424Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435704919133602:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.732407Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435706467163880:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:00.732585Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435706664434633:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.732624Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435706664434552:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23136) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435706664434620:2113] held: no
2025-11-19T13:59:00.732665Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435706664434619:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.732753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435706467163813:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25305) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435706467163880:2112] held: no
2025-11-19T13:59:00.732773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435706467163813:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.732783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:00.732791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435706467163813:2050] [node 3] ICP22 created new session: [5:7574435706467163897:2050]
2025-11-19T13:59:00.732796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435706467163897:2050] [node 3] ICS09 handshake done sender: [5:7574435706467163880:2112] self: [5:7574435706467163892:0] peer: [3:7574435703888627537:0] socket: 195 qp: -1
2025-11-19T13:59:00.732803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435706467163897:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:00.732810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:00.732937Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435706467163891:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.732960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435706467163812:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.733006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435706467163896:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54600
2025-11-19T13:59:00.733125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435706467163811:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.733295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435706664434637:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29452
2025-11-19T13:59:00.733388Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435704919133602:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:00.733518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435706664434622:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.733542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435706664434566:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.733554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435706664434551:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.733560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435706664434553:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.733569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:00.733572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435706664434553:2050] [node 3] ICP22 created new session: [6:7574435706664434641:2050]
2025-11-19T13:59:00.733577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435706664434641:2050] [node 3] ICS09 handshake done sender: [6:7574435706664434619:2112] self: [6:7574435706664434634:0] peer: [3:7574435703888627519:0] socket: 194 qp: -1
2025-11-19T13:59:00.733581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435706664434641:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:00.733587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:00.733869Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435706664434620:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:00.734055Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435704919133599:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.734134Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435704919133604:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.734137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435704919133604:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.734570Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435704919133595:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.734573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435704919133595:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.735580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435704919133596:2105] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.735591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:00.735598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435704919133596:2105] [node 4] ICP22 created new session: [1:7574435704919133618:2105]
2025-11-19T13:59:00.735603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435704919133618:2105] [node 4] ICS09 handshake done sender: [1:7574435704919133602:2114] self: [1:7574435704919133601:0] peer: [4:7574435704823898259:0] socket: 213 qp: -1
2025-11-19T13:59:00.735608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435704919133618:2105] [node 4] ICS10 traffic start
2025-11-19T13:59:00.735614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:00.736002Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435706121614030:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:00.736938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435704823898240:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.736947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435704823898240:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.736139Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435706121614028:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.736147Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435706121614028:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.736606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435706121614026:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.736616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:00.736621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435706121614026:2109] [node 4] ICP22 created new session: [2:7574435706121614047:2109]
2025-11-19T13:59:00.736625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435706121614047:2109] [node 4] ICS09 handshake done sender: [2:7574435706121614030:2114] self: [2:7574435706121614029:0] peer: [4:7574435704823898260:0] socket: 215 qp: -1
2025-11-19T13:59:00.736630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435706121614047:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:00.736638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.737455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435704823898261:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6468
2025-11-19T13:59:00.737691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435704823898262:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6468
2025-11-19T13:59:00.737811Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435704823898257:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:00.737854Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435704823898251:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:00.737942Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435704823898240:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:00.738006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435704823898172:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.738016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:00.738021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435704823898172:2049] [node 2] ICP22 created new session: [4:7574435704823898264:2049]
2025-11-19T13:59:00.738026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435704823898264:2049] [node 2] ICS09 handshake done sender: [4:7574435704823898257:2119] self: [4:7574435704823898260:0] peer: [2:7574435706121614029:0] socket: 238 qp: -1
2025-11-19T13:59:00.738031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435704823898264:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:00.738035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:00.738138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435704823898171:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.738148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:00.738152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435704823898171:2048] [node 1] ICP22 created new session: [4:7574435704823898266:2048]
2025-11-19T13:59:00.738156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435704823898266:2048] [node 1] ICS09 handshake done sender: [4:7574435704823898251:2117] self: [4:7574435704823898259:0] peer: [1:7574435704919133601:0] socket: 219 qp: -1
2025-11-19T13:59:00.738159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435704823898266:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:00.738164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:00.738239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435704823898205:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.738243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:00.738246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435704823898205:2070] [node 6] ICP22 created new session: [4:7574435704823898268:2070]
2025-11-19T13:59:00.738250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435704823898268:2070] [node 6] ICS09 handshake done sender: [4:7574435704823898240:2111] self: [4:7574435704823898239:0] peer: [6:7574435706664434638:0] socket: 208 qp: -1
2025-11-19T13:59:00.738253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435704823898268:2070] [node 6] ICS10 traffic start
2025-11-19T13:59:00.738257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:00.739060Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435706467163847:2081] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.739068Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435706467163847:2081] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.739513Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435706467163896:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.740699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435706467163879:2085] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.740994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435706664434552:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.741012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:00.741017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435706664434552:2049] [node 2] ICP22 created new session: [6:7574435706664434646:2049]
2025-11-19T13:59:00.741022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435706664434646:2049] [node 2] ICS09 handshake done sender: [6:7574435706664434620:2113] self: [6:7574435706664434635:0] peer: [2:7574435706121614011:0] socket: 199 qp: -1
2025-11-19T13:59:00.741027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435706664434646:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:00.741033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.741923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435704919133617:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36701
2025-11-19T13:59:00.742093Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435706467163847:2081] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:00.742314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435706467163844:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.742330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:00.742335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435706467163844:2079] [node 6] ICP22 created new session: [5:7574435706467163906:2079]
2025-11-19T13:59:00.742340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435706467163906:2079] [node 6] ICS09 handshake done sender: [5:7574435706467163847:2081] self: [5:7574435706467163845:0] peer: [6:7574435706664434639:0] socket: 196 qp: -1
2025-11-19T13:59:00.742345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435706467163906:2079] [node 6] ICS10 traffic start
2025-11-19T13:59:00.742350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.742711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435706467163904:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54600
2025-11-19T13:59:00.742727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435706467163905:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54600
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.755941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435706664434644:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29452
2025-11-19T13:59:00.756110Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435706121614028:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:00.757011Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435704919133604:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:00.757495Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435706467163893:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.757686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435706121614025:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.757701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:00.757710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435706121614025:2104] [node 5] ICP22 created new session: [2:7574435706121614052:2104]
2025-11-19T13:59:00.757716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435706121614052:2104] [node 5] ICS09 handshake done sender: [2:7574435706121614028:2113] self: [2:7574435706121614027:0] peer: [5:7574435706467163899:0] socket: 225 qp: -1
2025-11-19T13:59:00.757723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435706121614052:2104] [node 5] ICS10 traffic start
2025-11-19T13:59:00.757730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:00.757929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435706121614050:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24666
2025-11-19T13:59:00.757972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435706121614051:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24666
2025-11-19T13:59:00.758228Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435704919133595:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.758436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435706664434645:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29452
2025-11-19T13:59:00.758549Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435706467163891:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:00.758699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435706467163812:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.758715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:00.758722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435706467163812:2049] [node 2] ICP22 created new session: [5:7574435706467163909:2049]
2025-11-19T13:59:00.758727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435706467163909:2049] [node 2] ICS09 handshake done sender: [5:7574435706467163893:2118] self: [5:7574435706467163899:0] peer: [2:7574435706121614027:0] socket: 228 qp: -1
2025-11-19T13:59:00.758732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435706467163909:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:00.758740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:00.758893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435706467163811:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.758901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:00.758905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435706467163811:2048] [node 1] ICP22 created new session: [5:7574435706467163911:2048]
2025-11-19T13:59:00.758909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435706467163911:2048] [node 1] ICS09 handshake done sender: [5:7574435706467163891:2117] self: [5:7574435706467163901:0] peer: [1:7574435704919133603:0] socket: 221 qp: -1
2025-11-19T13:59:00.758914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435706467163911:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:00.758919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:00.759985Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435704919133617:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.760030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435704919133597:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.760041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:00.760047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435704919133597:2106] [node 5] ICP22 created new session: [1:7574435704919133624:2106]
2025-11-19T13:59:00.760051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435704919133624:2106] [node 5] ICS09 handshake done sender: [1:7574435704919133604:2115] self: [1:7574435704919133603:0] peer: [5:7574435706467163901:0] socket: 214 qp: -1
2025-11-19T13:59:00.760057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435704919133624:2106] [node 5] ICS10 traffic start
2025-11-19T13:59:00.760063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:00.760209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435704919133588:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.760220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:00.760225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435704919133588:2104] [node 6] ICP22 created new session: [1:7574435704919133626:2104]
2025-11-19T13:59:00.760229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435704919133626:2104] [node 6] ICS09 handshake done sender: [1:7574435704919133595:2110] self: [1:7574435704919133589:0] peer: [6:7574435706664434640:0] socket: 212 qp: -1
2025-11-19T13:59:00.760234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435704919133626:2104] [node 6] ICS10 traffic start
2025-11-19T13:59:00.760240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:00.761461Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435704823898244:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.761474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435704823898244:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.762441Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435706121614050:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.762561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435706121614025:2104] [node 5] ICP18 handshake (actor [2:7574435706121614050:2117]) is too old
2025-11-19T13:59:00.762899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435704919133597:2106] [node 5] ICP18 handshake (actor [1:7574435704919133617:2119]) is too old
2025-11-19T13:59:00.763300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435706664434566:2060] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18645) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435706664434621:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.765178Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435704823898244:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:00.765925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435704823898206:2074] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.765945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:00.765950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435704823898206:2074] [node 5] ICP22 created new session: [4:7574435704823898272:2074]
2025-11-19T13:59:00.765956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435704823898272:2074] [node 5] ICS09 handshake done sender: [4:7574435704823898244:2113] self: [4:7574435704823898243:0] peer: [5:7574435706467163903:0] socket: 234 qp: -1
2025-11-19T13:59:00.765961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435704823898272:2074] [node 5] ICS10 traffic start
2025-11-19T13:59:00.765970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.766578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435704823898271:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6468
2025-11-19T13:59:00.764996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435703888627561:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55650
2025-11-19T13:59:00.768586Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435706664434632:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:00.769437Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435706664434621:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.769547Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435706664434551:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23951) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435706664434633:2119] held: no
2025-11-19T13:59:00.769600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435706664434649:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29452
2025-11-19T13:59:00.769743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435706664434622:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.769761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:00.769768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435706664434622:2072] [node 4] ICP22 created new session: [6:7574435706664434650:2072]
2025-11-19T13:59:00.769774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435706664434650:2072] [node 4] ICS09 handshake done sender: [6:7574435706664434632:2118] self: [6:7574435706664434638:0] peer: [4:7574435704823898239:0] socket: 217 qp: -1
2025-11-19T13:59:00.769780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435706664434650:2072] [node 4] ICS10 traffic start
2025-11-19T13:59:00.769790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:00.769961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435706664434566:2060] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.769970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:00.769974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435706664434566:2060] [node 5] ICP22 created new session: [6:7574435706664434652:2060]
2025-11-19T13:59:00.769978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435706664434652:2060] [node 5] ICS09 handshake done sender: [6:7574435706664434621:2114] self: [6:7574435706664434639:0] peer: [5:7574435706467163845:0] socket: 200 qp: -1
2025-11-19T13:59:00.769983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435706664434652:2060] [node 5] ICS10 traffic start
2025-11-19T13:59:00.769988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.770503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435706467163879:2085] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17433) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435706467163896:2120] held: no
2025-11-19T13:59:00.770613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435706467163913:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54600
2025-11-19T13:59:00.770766Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435706664434633:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:00.771069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435706664434551:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.771081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:00.771086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435706664434551:2048] [node 1] ICP22 created new session: [6:7574435706664434654:2048]
2025-11-19T13:59:00.771090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435706664434654:2048] [node 1] ICS09 handshake done sender: [6:7574435706664434633:2119] self: [6:7574435706664434640:0] peer: [1:7574435704919133589:0] socket: 218 qp: -1
2025-11-19T13:59:00.771094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435706664434654:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:00.771101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:00.771415Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435706467163896:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:00.771484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435706467163879:2085] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.771495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:00.771499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435706467163879:2085] [node 4] ICP22 created new session: [5:7574435706467163914:2085]
2025-11-19T13:59:00.771503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435706467163914:2085] [node 4] ICS09 handshake done sender: [5:7574435706467163896:2120] self: [5:7574435706467163903:0] peer: [4:7574435704823898243:0] socket: 237 qp: -1
2025-11-19T13:59:00.771508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435706467163914:2085] [node 4] ICS10 traffic start
2025-11-19T13:59:00.771514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:00.772281Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435703888627561:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.772314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435703888627466:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.772398Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435706121614051:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.772421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435706121613955:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:00.772849Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435704919133608:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.772860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435704919133608:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:00.773377Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435704919133606:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:00.773379Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435704919133606:2116] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.774288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435706121614055:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24666
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:00.774388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435703888627563:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55650
2025-11-19T13:59:00.774512Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435704919133608:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:00.774673Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435703888627561:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:00.774729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435703888627466:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.774742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:00.774747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435703888627466:2048] [node 1] ICP22 created new session: [3:7574435703888627564:2048]
2025-11-19T13:59:00.774752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435703888627564:2048] [node 1] ICS09 handshake done sender: [3:7574435703888627561:2118] self: [3:7574435703888627562:0] peer: [1:7574435704919133607:0] socket: 239 qp: -1
2025-11-19T13:59:00.774757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435703888627564:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:00.774764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:00.775714Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435704919133606:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:00.775906Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435706121614051:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:00.775982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435706121613955:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:00.775995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:00.776000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435706121613955:2048] [node 1] ICP22 created new session: [2:7574435706121614056:2048]
2025-11-19T13:59:00.776005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435706121614056:2048] [node 1] ICS09 handshake done sender: [2:7574435706121614051:2118] self: [2:7574435706121614054:0] peer: [1:7574435704919133605:0] socket: 250 qp: -1
2025-11-19T13:59:00.776010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435706121614056:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:00.776018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:00.776954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435704919133531:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.776973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:00.776979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435704919133531:2049] [node 3] ICP22 created new session: [1:7574435704919133631:2049]
2025-11-19T13:59:00.776984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435704919133631:2049] [node 3] ICS09 handshake done sender: [1:7574435704919133608:2117] self: [1:7574435704919133607:0] peer: [3:7574435703888627562:0] socket: 210 qp: -1
2025-11-19T13:59:00.776989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435704919133631:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:00.776996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:00.777122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435704919133530:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:00.777129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:00.777132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435704919133530:2048] [node 2] ICP22 created new session: [1:7574435704919133633:2048]
2025-11-19T13:59:00.777136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435704919133633:2048] [node 2] ICS09 handshake done sender: [1:7574435704919133606:2116] self: [1:7574435704919133605:0] peer: [2:7574435706121614054:0] socket: 248 qp: -1
2025-11-19T13:59:00.777139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435704919133633:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:00.777144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:00.787037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:00.787129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435706121614056:2048] [node 1] ICS01 socket: 250 reason# EndOfStream
2025-11-19T13:59:00.787139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435706121613955:2048] [node 1] ICP30 unregister session Session# [2:7574435706121614056:2048] VirtualId# [2:7574435706121614054:0]
2025-11-19T13:59:00.787146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435706121614056:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.787186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:00.787205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:00.796149Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:00.796243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435703888627564:2048] [node 1] ICS01 socket: 239 reason# EndOfStream
2025-11-19T13:59:00.796252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435703888627466:2048] [node 1] ICP30 unregister session Session# [3:7574435703888627564:2048] VirtualId# [3:7574435703888627562:0]
2025-11-19T13:59:00.796260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435703888627564:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.796273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:00.796329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:00.804713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:00.804807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435706664434654:2048] [node 1] ICS01 socket: 218 reason# EndOfStream
2025-11-19T13:59:00.804817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435706664434551:2048] [node 1] ICP30 unregister session Session# [6:7574435706664434654:2048] VirtualId# [6:7574435706664434640:0]
2025-11-19T13:59:00.804825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435706664434654:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.804882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:00.804906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:00.804945Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:00.805003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435704823898266:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-11-19T13:59:00.805009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435704823898171:2048] [node 1] ICP30 unregister session Session# [4:7574435704823898266:2048] VirtualId# [4:7574435704823898259:0]
2025-11-19T13:59:00.805013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435704823898266:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.805039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:00.805061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:00.805090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:00.805147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435706467163911:2048] [node 1] ICS01 socket: 221 reason# EndOfStream
2025-11-19T13:59:00.805152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435706467163811:2048] [node 1] ICP30 unregister session Session# [5:7574435706467163911:2048] VirtualId# [5:7574435706467163901:0]
2025-11-19T13:59:00.805155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435706467163911:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.805177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:00.805197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:00.808826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:00.808903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435703888627555:2049] [node 2] ICS01 socket: 190 reason# EndOfStream
2025-11-19T13:59:00.808910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435703888627467:2049] [node 2] ICP30 unregister session Session# [3:7574435703888627555:2049] VirtualId# [3:7574435703888627547:0]
2025-11-19T13:59:00.808915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435703888627555:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.808945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:00.808963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:00.814102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:00.814195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435706664434646:2049] [node 2] ICS01 socket: 199 reason# EndOfStream
2025-11-19T13:59:00.814204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435706664434552:2049] [node 2] ICP30 unregister session Session# [6:7574435706664434646:2049] VirtualId# [6:7574435706664434635:0]
2025-11-19T13:59:00.814211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435706664434646:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.814278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:00.814303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:00.814370Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:00.814432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435706467163909:2049] [node 2] ICS01 socket: 228 reason# EndOfStream
2025-11-19T13:59:00.814437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435706467163812:2049] [node 2] ICP30 unregister session Session# [5:7574435706467163909:2049] VirtualId# [5:7574435706467163899:0]
2025-11-19T13:59:00.814441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435706467163909:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.814468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:00.814489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:00.814516Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:00.814576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435704823898264:2049] [node 2] ICS01 socket: 238 reason# EndOfStream
2025-11-19T13:59:00.814580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435704823898172:2049] [node 2] ICP30 unregister session Session# [4:7574435704823898264:2049] VirtualId# [4:7574435704823898260:0]
2025-11-19T13:59:00.814584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435704823898264:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.814607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:00.814626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:00.820964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:00.821060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435706467163897:2050] [node 3] ICS01 socket: 195 reason# EndOfStream
2025-11-19T13:59:00.821068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435706467163813:2050] [node 3] ICP30 unregister session Session# [5:7574435706467163897:2050] VirtualId# [5:7574435706467163892:0]
2025-11-19T13:59:00.821075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435706467163897:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.821129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:00.821152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:00.822419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:00.822496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435706664434641:2050] [node 3] ICS01 socket: 194 reason# EndOfStream
2025-11-19T13:59:00.822504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435706664434553:2050] [node 3] ICP30 unregister session Session# [6:7574435706664434641:2050] VirtualId# [6:7574435706664434634:0]
2025-11-19T13:59:00.822510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435706664434641:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.822567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:00.822591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:00.836170Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:00.836261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435706664434650:2072] [node 4] ICS01 socket: 217 reason# EndOfStream
2025-11-19T13:59:00.836272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435706664434622:2072] [node 4] ICP30 unregister session Session# [6:7574435706664434650:2072] VirtualId# [6:7574435706664434638:0]
2025-11-19T13:59:00.836279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435706664434650:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:00.836335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:00.836360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:01.048091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435707152244185:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:01.048108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435707152244186:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:01.068389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435709606512714:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:01.068402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435709606512715:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:01.128309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435710530829542:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:01.128324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435710530829543:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:01.179902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435709963469548:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:01.179914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435709963469549:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:01.179918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435709963469550:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:01.240195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435709958623673:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:01.240215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435709958623674:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:01.240220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435709958623675:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:01.280296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435708126453298:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:01.280319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435708126453299:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:01.280324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435708126453300:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:01.304688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435708126453321:2066] [node 5] ICP01 ready to work
2025-11-19T13:59:01.304771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435708126453322:2068] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.308126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435707152244247:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:01.308228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435707152244248:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:01.305670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435709958623719:2090] [node 4] ICP01 ready to work
2025-11-19T13:59:01.306202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435709958623724:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34660
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.332202Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435709606512714:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24338450ns) {Configure:104(24337538ns) {ProcessConfigured:135(23031467ns) {StartInitialHandshake:157(23030360ns) {DropHandshakes:469(880ns) {DropIncomingHandshake:441(125ns) DropOutgoingHandshake:456(47ns)} GenerateSessionVirtualId:988(297ns) PrepareNewSessionHandshake:476(1431ns) {SwitchToState:178(1064ns)}}}}}
2025-11-19T13:59:01.332379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435709606512780:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:01.332526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435709606512782:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:01.332598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435709606512783:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:01.333450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435709606512781:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:812
2025-11-19T13:59:01.333743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435709606512715:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(25644700ns) {Configure:104(25642879ns) {ProcessConfigured:135(24406921ns) {StartInitialHandshake:157(24406178ns) {DropHandshakes:469(264ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(13ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:476(514ns) {SwitchToState:178(383ns)}}}}}
2025-11-19T13:59:01.334722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435709958623744:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34660
2025-11-19T13:59:01.336970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435707152244254:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:01.337275Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435707152244247:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25402054ns) {Configure:104(25400593ns) {ProcessConfigured:135(25029776ns) {StartInitialHandshake:157(25028829ns) {DropHandshakes:469(765ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(340ns) PrepareNewSessionHandshake:476(1007ns) {SwitchToState:178(594ns)}}}}}
2025-11-19T13:59:01.340088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435710530829606:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:01.340201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435710530829607:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:01.347950Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709606512781:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.348393Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435709958623744:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.348448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435709958623674:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.348764Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435709958623724:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.355953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435710530829609:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:01.357100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435710530829605:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49766
2025-11-19T13:59:01.357515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435707152244185:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19910345ns) {Configure:104(19909902ns) {ProcessConfigured:135(14987267ns) {StartInitialHandshake:157(14986931ns) {DropHandshakes:469(865ns) {DropIncomingHandshake:441(376ns) DropOutgoingHandshake:456(140ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(908ns) {SwitchToState:178(413ns)}}}}}
2025-11-19T13:59:01.357558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435707152244251:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2131
2025-11-19T13:59:01.357609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435707152244252:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2131
2025-11-19T13:59:01.357643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435707152244253:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2131
2025-11-19T13:59:01.357972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435707152244186:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(20642858ns) {Configure:104(20642656ns) {ProcessConfigured:135(14636664ns) {StartInitialHandshake:157(14635475ns) {DropHandshakes:469(741ns) {DropIncomingHandshake:441(40ns) DropOutgoingHandshake:456(556ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(402ns) {SwitchToState:178(32ns)}}}}}
2025-11-19T13:59:01.359978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435709958623750:2096] [node 6] ICP01 ready to work
2025-11-19T13:59:01.360000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435710530829543:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19735309ns) {Configure:104(19734479ns) {ProcessConfigured:135(19314021ns) {StartInitialHandshake:157(19313298ns) {DropHandshakes:469(773ns) {DropIncomingHandshake:441(130ns) DropOutgoingHandshake:456(115ns)} GenerateSessionVirtualId:988(180ns) PrepareNewSessionHandshake:476(557ns) {SwitchToState:178(372ns)}}}}}
2025-11-19T13:59:01.360304Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709606512787:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.360321Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435709606512787:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.360516Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435710530829605:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.360767Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707152244251:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.360815Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707152244252:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.360836Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707152244253:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.363842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435709963469548:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(47842388ns) {Configure:104(47840671ns) {ProcessConfigured:135(47449178ns) {StartInitialHandshake:157(47448075ns) {DropHandshakes:469(906ns) {DropIncomingHandshake:441(81ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:476(1263ns) {SwitchToState:178(575ns)}}}}}
2025-11-19T13:59:01.364356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435709963469617:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.375828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435709958623753:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34660
2025-11-19T13:59:01.376155Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435709606512787:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:01.376466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435709606512782:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.376484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:01.376491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435709606512782:2108] [node 5] ICP22 created new session: [2:7574435709606512796:2108]
2025-11-19T13:59:01.376497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435709606512796:2108] [node 5] ICS09 handshake done sender: [2:7574435709606512787:2113] self: [2:7574435709606512786:0] peer: [5:7574435709958623751:0] socket: 204 qp: -1
2025-11-19T13:59:01.376505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435709606512796:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:01.376515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:01.380205Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435709958623753:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.380233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435709958623675:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.376085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435707152244267:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2131
2025-11-19T13:59:01.380123Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707152244267:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.379882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435709963469619:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:01.382123Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435709963469550:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17649336ns) {Configure:104(17648215ns) {ProcessConfigured:135(17232442ns) {StartInitialHandshake:157(17231625ns) {DropHandshakes:469(672ns) {DropIncomingHandshake:441(124ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(99ns) PrepareNewSessionHandshake:476(869ns) {SwitchToState:178(404ns)}}}}}
2025-11-19T13:59:01.382746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435709963469613:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5183
2025-11-19T13:59:01.382804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435709963469614:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5183
2025-11-19T13:59:01.382836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435709963469615:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5183
2025-11-19T13:59:01.382868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435709963469616:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5183
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.383372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435710530829623:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49766
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.384098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435709958623757:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34660
2025-11-19T13:59:01.384152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435709958623758:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34660
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.384567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435707152244272:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2131
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.387590Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435710530829615:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.387603Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435710530829615:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.387700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435709958623759:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34660
2025-11-19T13:59:01.387824Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435709958623757:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.387846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435709958623673:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.383481Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435709963469614:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.383504Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435709963469615:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.384566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435709963469550:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.387406Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435709963469616:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.387429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435709963469548:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.388041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435709963469549:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.388608Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435710530829611:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.388614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435710530829611:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.389302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435708126453376:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1574
2025-11-19T13:59:01.389401Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709606512785:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.389411Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435709606512785:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.390216Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435709606512785:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:01.390388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435709606512780:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.390404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:01.390410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435709606512780:2107] [node 4] ICP22 created new session: [2:7574435709606512800:2107]
2025-11-19T13:59:01.390415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435709606512800:2107] [node 4] ICS09 handshake done sender: [2:7574435709606512785:2112] self: [2:7574435709606512784:0] peer: [4:7574435709963469632:0] socket: 203 qp: -1
2025-11-19T13:59:01.390421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435709606512800:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:01.390430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.390853Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435709963469613:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.390988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435709963469633:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5183
2025-11-19T13:59:01.390998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435709963469634:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5183
2025-11-19T13:59:01.391102Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435710530829615:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:01.391238Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435710530829611:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:01.391399Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435710530829623:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.391409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435710530829607:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.391421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:01.391426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435710530829607:2109] [node 5] ICP22 created new session: [3:7574435710530829627:2109]
2025-11-19T13:59:01.391430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435710530829627:2109] [node 5] ICS09 handshake done sender: [3:7574435710530829615:2113] self: [3:7574435710530829614:0] peer: [5:7574435709958623756:0] socket: 211 qp: -1
2025-11-19T13:59:01.391434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435710530829627:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:01.391441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:01.391521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435710530829606:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.391527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:01.391531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435710530829606:2108] [node 4] ICP22 created new session: [3:7574435710530829629:2108]
2025-11-19T13:59:01.391534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435710530829629:2108] [node 4] ICS09 handshake done sender: [3:7574435710530829611:2111] self: [3:7574435710530829610:0] peer: [4:7574435709963469629:0] socket: 209 qp: -1
2025-11-19T13:59:01.391538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435710530829629:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:01.391542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:01.392024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707152244257:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.392038Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435707152244257:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.390111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435708126453377:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1574
2025-11-19T13:59:01.390720Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435708126453377:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.390740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435708126453299:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.392010Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709606512789:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.392018Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435709606512789:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.393800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435708126453379:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1574
2025-11-19T13:59:01.393978Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435709963469614:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:01.394066Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435709606512789:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:01.394223Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435708126453377:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:01.394315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435708126453299:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.394326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:01.394331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435708126453299:2049] [node 2] ICP22 created new session: [6:7574435708126453380:2049]
2025-11-19T13:59:01.394337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435708126453380:2049] [node 2] ICS09 handshake done sender: [6:7574435708126453377:2116] self: [6:7574435708126453378:0] peer: [2:7574435709606512788:0] socket: 231 qp: -1
2025-11-19T13:59:01.394342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435708126453380:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:01.394348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.396295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435710530829606:2108] [node 4] ICP18 handshake (actor [3:7574435710530829623:2116]) is too old
2025-11-19T13:59:01.396864Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435708126453376:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.396248Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707152244259:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.396890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435708126453298:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.396261Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435707152244259:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.397046Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435709958623759:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.397899Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435709958623675:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26306) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435709958623753:2116] held: no
2025-11-19T13:59:01.397964Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435709958623744:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:01.398047Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707152244255:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.398054Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435707152244255:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.398628Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435707152244257:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.398781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435709958623719:2090] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.398808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435709958623761:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34660
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.398951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435709958623762:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34660
2025-11-19T13:59:01.398961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435709958623674:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.398971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:01.398976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435709958623674:2049] [node 2] ICP22 created new session: [5:7574435709958623764:2049]
2025-11-19T13:59:01.398981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435709958623764:2049] [node 2] ICS09 handshake done sender: [5:7574435709958623744:2113] self: [5:7574435709958623751:0] peer: [2:7574435709606512786:0] socket: 205 qp: -1
2025-11-19T13:59:01.398986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435709958623764:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:01.398994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:01.399268Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435709958623753:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:01.399376Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435709963469615:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:01.399605Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435709963469623:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.399611Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435709963469623:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.400056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435709963469549:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.400068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:01.400074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435709963469549:2049] [node 2] ICP22 created new session: [4:7574435709963469638:2049]
2025-11-19T13:59:01.400078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435709963469638:2049] [node 2] ICS09 handshake done sender: [4:7574435709963469614:2112] self: [4:7574435709963469632:0] peer: [2:7574435709606512784:0] socket: 219 qp: -1
2025-11-19T13:59:01.400085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435709963469638:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:01.400092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:01.400227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435709963469550:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.400233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:01.400239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435709963469550:2050] [node 3] ICP22 created new session: [4:7574435709963469640:2050]
2025-11-19T13:59:01.400242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435709963469640:2050] [node 3] ICS09 handshake done sender: [4:7574435709963469615:2113] self: [4:7574435709963469629:0] peer: [3:7574435710530829610:0] socket: 220 qp: -1
2025-11-19T13:59:01.400246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435709963469640:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:01.400253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:01.401324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435709958623675:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.401337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:01.401342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435709958623675:2050] [node 3] ICP22 created new session: [5:7574435709958623767:2050]
2025-11-19T13:59:01.401346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435709958623767:2050] [node 3] ICS09 handshake done sender: [5:7574435709958623753:2116] self: [5:7574435709958623756:0] peer: [3:7574435710530829614:0] socket: 215 qp: -1
2025-11-19T13:59:01.401350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435709958623767:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:01.401355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:01.402747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435710530829626:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49766
2025-11-19T13:59:01.403235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435709606512783:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.403260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:01.403266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435709606512783:2109] [node 6] ICP22 created new session: [2:7574435709606512804:2109]
2025-11-19T13:59:01.403272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435709606512804:2109] [node 6] ICS09 handshake done sender: [2:7574435709606512789:2114] self: [2:7574435709606512788:0] peer: [6:7574435708126453378:0] socket: 223 qp: -1
2025-11-19T13:59:01.403278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435709606512804:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:01.403284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:01.403453Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435707152244259:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:01.403583Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707152244272:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.403593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435707152244248:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.403601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:01.403606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435707152244248:2108] [node 4] ICP22 created new session: [1:7574435707152244276:2108]
2025-11-19T13:59:01.403610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435707152244276:2108] [node 4] ICS09 handshake done sender: [1:7574435707152244257:2114] self: [1:7574435707152244256:0] peer: [4:7574435709963469630:0] socket: 213 qp: -1
2025-11-19T13:59:01.403614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435707152244276:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:01.403622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:01.403710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435707152244254:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.403727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:01.403731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435707152244254:2109] [node 5] ICP22 created new session: [1:7574435707152244278:2109]
2025-11-19T13:59:01.403734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435707152244278:2109] [node 5] ICS09 handshake done sender: [1:7574435707152244259:2115] self: [1:7574435707152244258:0] peer: [5:7574435709958623760:0] socket: 214 qp: -1
2025-11-19T13:59:01.403737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435707152244278:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:01.403741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:01.404086Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435709958623757:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:01.404156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435709958623673:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.404167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:01.404172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435709958623673:2048] [node 1] ICP22 created new session: [5:7574435709958623769:2048]
2025-11-19T13:59:01.404177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435709958623769:2048] [node 1] ICS09 handshake done sender: [5:7574435709958623757:2118] self: [5:7574435709958623760:0] peer: [1:7574435707152244258:0] socket: 218 qp: -1
2025-11-19T13:59:01.404182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435709958623769:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:01.404187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.405151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435709606512806:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:812
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.403852Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435710530829626:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.406019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435710530829543:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.406112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435709958623771:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34660
2025-11-19T13:59:01.406193Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709606512793:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.406201Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435709606512793:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.407223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435707152244248:2108] [node 4] ICP18 handshake (actor [1:7574435707152244272:2119]) is too old
2025-11-19T13:59:01.407273Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435707152244255:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:01.407458Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435709606512793:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:01.407558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435709606512715:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.407571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:01.407576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435709606512715:2049] [node 3] ICP22 created new session: [2:7574435709606512808:2049]
2025-11-19T13:59:01.407581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435709606512808:2049] [node 3] ICS09 handshake done sender: [2:7574435709606512793:2115] self: [2:7574435709606512776:0] peer: [3:7574435710530829633:0] socket: 235 qp: -1
2025-11-19T13:59:01.407586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435709606512808:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:01.407593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:01.405997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435709963469636:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5183
2025-11-19T13:59:01.406018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435709963469637:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5183
2025-11-19T13:59:01.408248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435707152244247:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.408264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:01.408269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435707152244247:2107] [node 6] ICP22 created new session: [1:7574435707152244282:2107]
2025-11-19T13:59:01.408274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435707152244282:2107] [node 6] ICS09 handshake done sender: [1:7574435707152244255:2113] self: [1:7574435707152244250:0] peer: [6:7574435708126453383:0] socket: 212 qp: -1
2025-11-19T13:59:01.408278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435707152244282:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:01.408283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.408606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435709606512810:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:812
2025-11-19T13:59:01.410884Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709606512810:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.410908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435709606512714:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.411025Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707152244262:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.411032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435707152244262:2116] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.411486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435709606512812:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:812
2025-11-19T13:59:01.411585Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435707152244262:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:01.411732Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435709606512810:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:01.411855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435709606512714:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.411868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:01.411872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435709606512714:2048] [node 1] ICP22 created new session: [2:7574435709606512813:2048]
2025-11-19T13:59:01.411877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435709606512813:2048] [node 1] ICS09 handshake done sender: [2:7574435709606512810:2117] self: [2:7574435709606512811:0] peer: [1:7574435707152244260:0] socket: 251 qp: -1
2025-11-19T13:59:01.411882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435709606512813:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:01.411887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:01.412765Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709606512806:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.412800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435709606512715:2049] [node 3] ICP18 handshake (actor [2:7574435709606512806:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.416048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435708126453298:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21256) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435708126453376:2115] held: no
2025-11-19T13:59:01.417393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435709963469548:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21256) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435709963469616:2114] held: no
2025-11-19T13:59:01.417475Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435709963469623:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.418409Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435709958623759:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:01.418487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435709958623719:2090] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.418500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:01.418506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435709958623719:2090] [node 4] ICP22 created new session: [5:7574435709958623772:2090]
2025-11-19T13:59:01.418512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435709958623772:2090] [node 4] ICS09 handshake done sender: [5:7574435709958623759:2120] self: [5:7574435709958623763:0] peer: [4:7574435709963469622:0] socket: 229 qp: -1
2025-11-19T13:59:01.418517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435709958623772:2090] [node 4] ICS10 traffic start
2025-11-19T13:59:01.418524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.419484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435707152244185:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.419509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:01.419513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435707152244185:2048] [node 2] ICP22 created new session: [1:7574435707152244285:2048]
2025-11-19T13:59:01.419517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435707152244285:2048] [node 2] ICS09 handshake done sender: [1:7574435707152244262:2116] self: [1:7574435707152244260:0] peer: [2:7574435709606512811:0] socket: 208 qp: -1
2025-11-19T13:59:01.419522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435707152244285:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:01.419528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:01.418135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435710530829543:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435710530829626:2117] held: no
2025-11-19T13:59:01.418201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435710530829634:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49766
2025-11-19T13:59:01.419370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435710530829635:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49766
2025-11-19T13:59:01.419414Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435710530829626:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.420085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435708126453384:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1574
2025-11-19T13:59:01.420152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435708126453385:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1574
2025-11-19T13:59:01.420188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435708126453386:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1574
2025-11-19T13:59:01.420611Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435708126453384:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.420641Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435708126453376:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:01.421465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435708126453298:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.421478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:01.421483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435708126453298:2048] [node 1] ICP22 created new session: [6:7574435708126453388:2048]
2025-11-19T13:59:01.421488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435708126453388:2048] [node 1] ICS09 handshake done sender: [6:7574435708126453376:2115] self: [6:7574435708126453383:0] peer: [1:7574435707152244250:0] socket: 216 qp: -1
2025-11-19T13:59:01.421493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435708126453388:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:01.421499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:01.421575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435708126453321:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.421799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435710530829543:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.421814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:01.421819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435710530829543:2049] [node 2] ICP22 created new session: [3:7574435710530829636:2049]
2025-11-19T13:59:01.421823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435710530829636:2049] [node 2] ICS09 handshake done sender: [3:7574435710530829626:2117] self: [3:7574435710530829633:0] peer: [2:7574435709606512776:0] socket: 238 qp: -1
2025-11-19T13:59:01.421828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435710530829636:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:01.421834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:01.421955Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435710530829635:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.422053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435710530829542:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.422067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435710530829638:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49766
2025-11-19T13:59:01.422884Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435708126453385:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.423914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435708126453300:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.422262Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707152244266:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.422271Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435707152244266:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.423928Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435707152244266:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:01.424629Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435709963469616:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:01.424062Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435710530829617:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.424064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435710530829640:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49766
2025-11-19T13:59:01.424069Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435710530829617:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.425237Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435710530829617:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:01.425106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435707152244186:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.425125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:01.425131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435707152244186:2049] [node 3] ICP22 created new session: [1:7574435707152244288:2049]
2025-11-19T13:59:01.425136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435707152244288:2049] [node 3] ICS09 handshake done sender: [1:7574435707152244266:2117] self: [1:7574435707152244261:0] peer: [3:7574435710530829639:0] socket: 250 qp: -1
2025-11-19T13:59:01.425141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435707152244288:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:01.425148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:01.427843Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435709958623754:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.427857Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435709958623754:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.429055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435709963469617:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.429074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:01.429080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435709963469617:2109] [node 5] ICP22 created new session: [4:7574435709963469644:2109]
2025-11-19T13:59:01.429085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435709963469644:2109] [node 5] ICS09 handshake done sender: [4:7574435709963469623:2116] self: [4:7574435709963469622:0] peer: [5:7574435709958623763:0] socket: 226 qp: -1
2025-11-19T13:59:01.429089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435709963469644:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:01.429097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:01.429230Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435709963469636:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.430728Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435710530829638:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.430740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435710530829609:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.430749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:01.430755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435710530829609:2107] [node 6] ICP22 created new session: [3:7574435710530829642:2107]
2025-11-19T13:59:01.430760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435710530829642:2107] [node 6] ICS09 handshake done sender: [3:7574435710530829617:2114] self: [3:7574435710530829616:0] peer: [6:7574435708126453391:0] socket: 246 qp: -1
2025-11-19T13:59:01.430765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435710530829642:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:01.430771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:01.431078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435709963469548:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.431089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:01.431093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435709963469548:2048] [node 1] ICP22 created new session: [4:7574435709963469647:2048]
2025-11-19T13:59:01.431097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435709963469647:2048] [node 1] ICS09 handshake done sender: [4:7574435709963469616:2114] self: [4:7574435709963469630:0] peer: [1:7574435707152244256:0] socket: 221 qp: -1
2025-11-19T13:59:01.431101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435709963469647:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:01.431106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:01.432098Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435710530829635:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:01.432166Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435710530829542:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21256) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435710530829635:2119] held: no
2025-11-19T13:59:01.432185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435710530829542:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.432193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:01.432197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435710530829542:2048] [node 1] ICP22 created new session: [3:7574435710530829644:2048]
2025-11-19T13:59:01.432202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435710530829644:2048] [node 1] ICS09 handshake done sender: [3:7574435710530829635:2119] self: [3:7574435710530829639:0] peer: [1:7574435707152244261:0] socket: 202 qp: -1
2025-11-19T13:59:01.432206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435710530829644:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:01.432213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:01.432313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435710530829609:2107] [node 6] ICP18 handshake (actor [3:7574435710530829638:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.432647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435708126453392:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1574
2025-11-19T13:59:01.435225Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435708126453300:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26306) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435708126453385:2119] held: no
2025-11-19T13:59:01.435252Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435708126453385:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:01.435329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435708126453300:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.435341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:01.435346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435708126453300:2050] [node 3] ICP22 created new session: [6:7574435708126453393:2050]
2025-11-19T13:59:01.435350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435708126453393:2050] [node 3] ICS09 handshake done sender: [6:7574435708126453385:2119] self: [6:7574435708126453391:0] peer: [3:7574435710530829616:0] socket: 252 qp: -1
2025-11-19T13:59:01.435356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435708126453393:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:01.435361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.472060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435708126453395:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1574
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.472599Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435708126453395:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.472647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435708126453396:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1574
2025-11-19T13:59:01.472666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435708126453322:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.472787Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435709963469626:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.472799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435709963469626:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.473643Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435709958623754:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:01.473767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435709958623750:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.473786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:01.473793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435709958623750:2096] [node 6] ICP22 created new session: [5:7574435709958623775:2096]
2025-11-19T13:59:01.473798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435709958623775:2096] [node 6] ICS09 handshake done sender: [5:7574435709958623754:2117] self: [5:7574435709958623752:0] peer: [6:7574435708126453390:0] socket: 195 qp: -1
2025-11-19T13:59:01.473804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435709958623775:2096] [node 6] ICS10 traffic start
2025-11-19T13:59:01.473812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.474350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435708126453398:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1574
2025-11-19T13:59:01.474503Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435708126453384:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:01.474654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435708126453321:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.474665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:01.474670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435708126453321:2066] [node 5] ICP22 created new session: [6:7574435708126453399:2066]
2025-11-19T13:59:01.474675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435708126453399:2066] [node 5] ICS09 handshake done sender: [6:7574435708126453384:2118] self: [6:7574435708126453390:0] peer: [5:7574435709958623752:0] socket: 201 qp: -1
2025-11-19T13:59:01.474680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435708126453399:2066] [node 5] ICS10 traffic start
2025-11-19T13:59:01.474687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:01.475875Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435709963469626:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:01.476113Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435708126453395:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:01.476186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435708126453322:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.476199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:01.476205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435708126453322:2068] [node 4] ICP22 created new session: [6:7574435708126453401:2068]
2025-11-19T13:59:01.476210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435708126453401:2068] [node 4] ICS09 handshake done sender: [6:7574435708126453395:2122] self: [6:7574435708126453397:0] peer: [4:7574435709963469625:0] socket: 193 qp: -1
2025-11-19T13:59:01.476216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435708126453401:2068] [node 4] ICS10 traffic start
2025-11-19T13:59:01.476221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:01.477257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435709963469619:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.477275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:01.477281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435709963469619:2108] [node 6] ICP22 created new session: [4:7574435709963469650:2108]
2025-11-19T13:59:01.477286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435709963469650:2108] [node 6] ICS09 handshake done sender: [4:7574435709963469626:2117] self: [4:7574435709963469625:0] peer: [6:7574435708126453397:0] socket: 217 qp: -1
2025-11-19T13:59:01.477291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435709963469650:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:01.477298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:01.481041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:01.481119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435709606512813:2048] [node 1] ICS01 socket: 251 reason# EndOfStream
2025-11-19T13:59:01.481127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435709606512714:2048] [node 1] ICP30 unregister session Session# [2:7574435709606512813:2048] VirtualId# [2:7574435709606512811:0]
2025-11-19T13:59:01.481135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435709606512813:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:01.481166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:01.481182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:01.481706Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:01.481779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435710530829644:2048] [node 1] ICS01 socket: 202 reason# EndOfStream
2025-11-19T13:59:01.481784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435710530829542:2048] [node 1] ICP30 unregister session Session# [3:7574435710530829644:2048] VirtualId# [3:7574435710530829639:0]
2025-11-19T13:59:01.481789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435710530829644:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:01.481812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:01.481828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:01.486538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:01.486627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435709963469647:2048] [node 1] ICS01 socket: 221 reason# EndOfStream
2025-11-19T13:59:01.486635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435709963469548:2048] [node 1] ICP30 unregister session Session# [4:7574435709963469647:2048] VirtualId# [4:7574435709963469630:0]
2025-11-19T13:59:01.486642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435709963469647:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:01.486707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:01.486731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:01.486767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:01.486833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435709958623769:2048] [node 1] ICS01 socket: 218 reason# EndOfStream
2025-11-19T13:59:01.486839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435709958623673:2048] [node 1] ICP30 unregister session Session# [5:7574435709958623769:2048] VirtualId# [5:7574435709958623760:0]
2025-11-19T13:59:01.486843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435709958623769:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:01.486870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:01.486891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:01.487099Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:01.487166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435708126453388:2048] [node 1] ICS01 socket: 216 reason# EndOfStream
2025-11-19T13:59:01.487172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435708126453298:2048] [node 1] ICP30 unregister session Session# [6:7574435708126453388:2048] VirtualId# [6:7574435708126453383:0]
2025-11-19T13:59:01.487177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435708126453388:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:01.487205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:01.487226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:01.490344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:01.490417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435710530829636:2049] [node 2] ICS01 socket: 238 reason# EndOfStream
2025-11-19T13:59:01.490424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435710530829543:2049] [node 2] ICP30 unregister session Session# [3:7574435710530829636:2049] VirtualId# [3:7574435710530829633:0]
2025-11-19T13:59:01.490430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435710530829636:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:01.490470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:01.490491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:01.493555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:01.493651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435709963469638:2049] [node 2] ICS01 socket: 219 reason# EndOfStream
2025-11-19T13:59:01.493658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435709963469549:2049] [node 2] ICP30 unregister session Session# [4:7574435709963469638:2049] VirtualId# [4:7574435709963469632:0]
2025-11-19T13:59:01.493665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435709963469638:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:01.493715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:01.493739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:01.498903Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:01.498997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435708126453380:2049] [node 2] ICS01 socket: 231 reason# EndOfStream
2025-11-19T13:59:01.499004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435708126453299:2049] [node 2] ICP30 unregister session Session# [6:7574435708126453380:2049] VirtualId# [6:7574435708126453378:0]
2025-11-19T13:59:01.499011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435708126453380:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:01.499076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:01.499102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:01.499142Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:01.499198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435709958623764:2049] [node 2] ICS01 socket: 205 reason# EndOfStream
2025-11-19T13:59:01.499203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435709958623674:2049] [node 2] ICP30 unregister session Session# [5:7574435709958623764:2049] VirtualId# [5:7574435709958623751:0]
2025-11-19T13:59:01.499207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435709958623764:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:01.499237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:01.499259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:01.505677Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:01.505767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435709958623767:2050] [node 3] ICS01 socket: 215 reason# EndOfStream
2025-11-19T13:59:01.505776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435709958623675:2050] [node 3] ICP30 unregister session Session# [5:7574435709958623767:2050] VirtualId# [5:7574435709958623756:0]
2025-11-19T13:59:01.505785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435709958623767:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:01.505850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:01.505876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:01.506593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:01.506668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435708126453393:2050] [node 3] ICS01 socket: 252 reason# EndOfStream
2025-11-19T13:59:01.506675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435708126453300:2050] [node 3] ICP30 unregister session Session# [6:7574435708126453393:2050] VirtualId# [6:7574435708126453391:0]
2025-11-19T13:59:01.506681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435708126453393:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:01.506722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:01.506745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:01.517656Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:01.517728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435708126453401:2068] [node 4] ICS01 socket: 193 reason# EndOfStream
2025-11-19T13:59:01.517734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435708126453322:2068] [node 4] ICP30 unregister session Session# [6:7574435708126453401:2068] VirtualId# [6:7574435708126453397:0]
2025-11-19T13:59:01.517739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435708126453401:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:01.517781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:01.517805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:01.608857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435707938457100:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:01.608868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435707938457101:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:01.652234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435709826016447:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:01.652246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435709826016448:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:01.700256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435707403148130:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:01.700269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435707403148131:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:01.752039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435709251027698:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:01.752051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435709251027699:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:01.752055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435709251027700:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:01.828154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435708191937849:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:01.828168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435708191937850:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:01.828172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435708191937851:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:01.884394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435707224736029:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:01.884405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435707224736030:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:01.884409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435707224736031:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:01.910658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435708191937901:2098] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.911777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435707224736040:2058] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15179
2025-11-19T13:59:01.912112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435707224736047:2062] [node 4] ICP01 ready to work
2025-11-19T13:59:01.912270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435708191937908:2101] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.912824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435709251027759:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42093
2025-11-19T13:59:01.920186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435709251027762:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:01.920281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435709251027763:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:01.924433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435707224736090:2067] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.928990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435708191937927:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
2025-11-19T13:59:01.929125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435707403148196:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4205
2025-11-19T13:59:01.929468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435707403148199:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:01.929535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435707403148200:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:01.931643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435709826016513:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:01.926699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435707938457165:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:01.926842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435707938457169:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:01.926917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435707938457167:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:01.932166Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435707224736040:2058] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.932208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435707224736090:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.937521Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435707224736031:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(25435545ns) {Configure:104(25434585ns) {ProcessConfigured:135(22801186ns) {StartInitialHandshake:157(22800721ns) {DropHandshakes:469(275ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(18ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:476(954ns) {SwitchToState:178(530ns)}}}}}
2025-11-19T13:59:01.935629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435709826016512:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:01.935795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435709826016515:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.940690Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435707938457100:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(27712173ns) {Configure:104(27711098ns) {ProcessConfigured:135(26817177ns) {StartInitialHandshake:157(26816580ns) {DropHandshakes:469(550ns) {DropIncomingHandshake:441(96ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:476(558ns) {SwitchToState:178(119ns)}}}}}
2025-11-19T13:59:01.942191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435707938457171:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64588
2025-11-19T13:59:01.942209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435707938457172:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64588
2025-11-19T13:59:01.942272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435707938457173:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64588
2025-11-19T13:59:01.944188Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707938457173:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.958234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435709251027763:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(37502488ns) {Configure:104(30068638ns) {ProcessConfigured:135(29753476ns) {StartInitialHandshake:157(29752388ns) {DropHandshakes:469(392ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(28ns)} GenerateSessionVirtualId:988(352ns) PrepareNewSessionHandshake:476(1588ns) {SwitchToState:178(882ns)}}}}}
2025-11-19T13:59:01.964062Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435708191937903:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.964085Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435708191937903:2099] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.965216Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435708191937927:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.965361Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435709251027759:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.970092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435707403148208:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4205
2025-11-19T13:59:01.969259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435709826016523:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11873
2025-11-19T13:59:01.969334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435709826016524:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11873
2025-11-19T13:59:01.969345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435709826016525:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11873
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.970349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435709251027776:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42093
2025-11-19T13:59:01.970396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435709251027777:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42093
2025-11-19T13:59:01.970429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435709251027778:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42093
2025-11-19T13:59:01.970645Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435709251027776:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.972486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435707224736109:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15179
2025-11-19T13:59:01.972507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435707224736110:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15179
2025-11-19T13:59:01.974174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435707403148207:2114] [node 5] ICP01 ready to work
2025-11-19T13:59:01.970373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435709826016513:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(34518762ns) {Configure:104(34517185ns) {ProcessConfigured:135(34245614ns) {StartInitialHandshake:157(34245201ns) {DropHandshakes:469(458ns) {DropIncomingHandshake:441(43ns) DropOutgoingHandshake:456(113ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:476(765ns) {SwitchToState:178(54ns)}}}}}
2025-11-19T13:59:01.974842Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709826016523:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.976725Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435709251027777:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.976754Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435709251027778:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.979984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435709251027762:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(59656137ns) {Configure:104(59655177ns) {ProcessConfigured:135(59304216ns) {StartInitialHandshake:157(59303695ns) {DropHandshakes:469(546ns) {DropIncomingHandshake:441(144ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:476(1108ns) {SwitchToState:178(583ns)}}}}}
2025-11-19T13:59:01.980009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435709251027699:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.982600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435709251027784:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42093
2025-11-19T13:59:01.980192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435707938457167:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(40028354ns) {Configure:104(40027513ns) {ProcessConfigured:135(39716227ns) {StartInitialHandshake:157(39715841ns) {DropHandshakes:469(586ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(252ns) PrepareNewSessionHandshake:476(1026ns) {SwitchToState:178(637ns)}}}}}
2025-11-19T13:59:01.981567Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707938457172:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.981726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435707938457181:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64588
2025-11-19T13:59:01.983831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435707938457165:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(56861559ns) {Configure:104(56860681ns) {ProcessConfigured:135(56555200ns) {StartInitialHandshake:157(56554431ns) {DropHandshakes:469(470ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(855ns) {SwitchToState:178(66ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.983086Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435707403148202:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.983098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435707403148202:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.983702Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435707403148196:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.983943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435707403148130:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.983962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435707403148210:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4205
2025-11-19T13:59:01.981880Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709826016520:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.981890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435709826016520:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.982692Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435709826016520:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:01.982773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435709826016515:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.982784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:01.982789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435709826016515:2110] [node 4] ICP22 created new session: [2:7574435709826016531:2110]
2025-11-19T13:59:01.982794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435709826016531:2110] [node 4] ICS09 handshake done sender: [2:7574435709826016520:2112] self: [2:7574435709826016519:0] peer: [4:7574435709251027782:0] socket: 227 qp: -1
2025-11-19T13:59:01.982800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435709826016531:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:01.982809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:01.984212Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709826016524:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.984370Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707938457171:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.986013Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435707403148202:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:01.986358Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435707224736109:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.986396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435707224736029:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.988131Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435707403148210:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.988158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435707403148131:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.988251Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709826016522:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.988260Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435709826016522:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.989200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435707403148214:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4205
2025-11-19T13:59:01.989290Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435709826016522:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:01.989459Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435707403148210:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:01.990524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435707403148131:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.990543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:01.990549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435707403148131:2049] [node 2] ICP22 created new session: [3:7574435707403148215:2049]
2025-11-19T13:59:01.990555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435707403148215:2049] [node 2] ICS09 handshake done sender: [3:7574435707403148210:2116] self: [3:7574435707403148213:0] peer: [2:7574435709826016521:0] socket: 237 qp: -1
2025-11-19T13:59:01.990562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435707403148215:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:01.990569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:01.991523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435709826016448:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.991541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:01.991545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435709826016448:2049] [node 3] ICP22 created new session: [2:7574435709826016536:2049]
2025-11-19T13:59:01.991550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435709826016536:2049] [node 3] ICS09 handshake done sender: [2:7574435709826016522:2113] self: [2:7574435709826016521:0] peer: [3:7574435707403148213:0] socket: 236 qp: -1
2025-11-19T13:59:01.991555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435709826016536:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:01.991562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.992294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435708191937929:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
2025-11-19T13:59:01.993359Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435708191937929:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.993398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435708191937850:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.993643Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709826016518:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.993652Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435709826016518:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.993987Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707938457160:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.994006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435707938457160:2106] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.994109Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707938457177:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:01.994115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435707938457177:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:01.994049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435709251027700:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:01.994080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435709251027700:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(17289906ns) {IncomingHandshake:261(17289516ns) {DropIncomingHandshake:441(40ns) PrepareNewSessionHandshake:476(472ns) {SwitchToState:178(282ns)} GenerateSessionVirtualId:988(214ns) IssueIncomingHandshakeReply:193(17285491ns) {SwitchToState:178(330ns)}}}
2025-11-19T13:59:01.994251Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435709251027778:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:01.994616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435707224736112:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15179
2025-11-19T13:59:01.994671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435707224736113:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15179
2025-11-19T13:59:01.994682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435707224736114:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15179
2025-11-19T13:59:01.994853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435709251027699:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:01.994870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:01.994875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435709251027699:2049] [node 2] ICP22 created new session: [4:7574435709251027785:2049]
2025-11-19T13:59:01.994880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435709251027785:2049] [node 2] ICS09 handshake done sender: [4:7574435709251027778:2117] self: [4:7574435709251027782:0] peer: [2:7574435709826016519:0] socket: 230 qp: -1
2025-11-19T13:59:01.994885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435709251027785:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:01.994902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:01.992754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435707403148207:2114] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(18506379ns) {Configure:104(18505354ns) {ProcessConfigured:135(18180362ns) {StartInitialHandshake:157(18179742ns) {DropHandshakes:469(805ns) {DropIncomingHandshake:441(174ns) DropOutgoingHandshake:456(123ns)} GenerateSessionVirtualId:988(776ns) PrepareNewSessionHandshake:476(700ns) {SwitchToState:178(71ns)}}}}}
2025-11-19T13:59:01.994632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435707403148199:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:01.994642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:01.994649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435707403148199:2110] [node 4] ICP22 created new session: [3:7574435707403148219:2110]
2025-11-19T13:59:01.994669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435707403148219:2110] [node 4] ICS09 handshake done sender: [3:7574435707403148202:2112] self: [3:7574435707403148201:0] peer: [4:7574435709251027780:0] socket: 214 qp: -1
2025-11-19T13:59:01.994675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435707403148219:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:01.994683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:02.000233Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709826016525:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.000288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435709826016447:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.001395Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435708191937903:2099] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:02.001846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435708191937901:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.001866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:02.001872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435708191937901:2098] [node 6] ICP22 created new session: [5:7574435712486905230:2098]
2025-11-19T13:59:02.001878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435712486905230:2098] [node 6] ICS09 handshake done sender: [5:7574435708191937903:2099] self: [5:7574435708191937902:0] peer: [6:7574435707224736106:0] socket: 198 qp: -1
2025-11-19T13:59:02.001883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435712486905230:2098] [node 6] ICS10 traffic start
2025-11-19T13:59:02.001891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.002188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435712486905228:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
2025-11-19T13:59:02.002201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435712486905229:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
2025-11-19T13:59:02.003099Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435707224736112:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.003575Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435709826016518:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:02.003706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435709826016512:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.003718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:02.003726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435709826016512:2108] [node 5] ICP22 created new session: [2:7574435709826016541:2108]
2025-11-19T13:59:02.003731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435709826016541:2108] [node 5] ICS09 handshake done sender: [2:7574435709826016518:2111] self: [2:7574435709826016517:0] peer: [5:7574435708191937930:0] socket: 226 qp: -1
2025-11-19T13:59:02.003737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435709826016541:2108] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.003885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435707403148222:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4205
2025-11-19T13:59:02.003941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435713545995084:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42093
2025-11-19T13:59:02.003893Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435707224736113:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.003926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435707224736030:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.003937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435707224736031:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.004842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435714120983839:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11873
2025-11-19T13:59:02.007165Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435709251027777:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:02.007326Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435707938457177:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:02.007424Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435707938457160:2106] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:02.007679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.007930Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435707403148208:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.007961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435707403148199:2110] [node 4] ICP18 handshake (actor [3:7574435707403148208:2115]) is too old
2025-11-19T13:59:02.004756Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435707224736040:2058] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:02.007887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435707224736090:2067] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15051) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435707224736040:2058] held: no
2025-11-19T13:59:02.007904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435707224736090:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.007912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:02.007917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435707224736090:2067] [node 5] ICP22 created new session: [6:7574435711519703414:2067]
2025-11-19T13:59:02.007922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435711519703414:2067] [node 5] ICS09 handshake done sender: [6:7574435707224736040:2058] self: [6:7574435707224736106:0] peer: [5:7574435708191937902:0] socket: 199 qp: -1
2025-11-19T13:59:02.007927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435711519703414:2067] [node 5] ICS10 traffic start
2025-11-19T13:59:02.007935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:02.008080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435711519703413:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15179
2025-11-19T13:59:02.008862Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435707224736110:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.008886Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435707224736109:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:02.008938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435707224736029:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.008958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:02.008965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435707224736029:2048] [node 1] ICP22 created new session: [6:7574435711519703416:2048]
2025-11-19T13:59:02.008970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435711519703416:2048] [node 1] ICS09 handshake done sender: [6:7574435707224736109:2116] self: [6:7574435707224736111:0] peer: [1:7574435707938457176:0] socket: 220 qp: -1
2025-11-19T13:59:02.008975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435711519703416:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:02.008981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:02.009079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435707224736047:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.009004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435707938457101:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.009021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:02.009025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435707938457101:2049] [node 3] ICP22 created new session: [1:7574435712233424484:2049]
2025-11-19T13:59:02.009030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435712233424484:2049] [node 3] ICS09 handshake done sender: [1:7574435707938457160:2106] self: [1:7574435707938457159:0] peer: [3:7574435707403148212:0] socket: 202 qp: -1
2025-11-19T13:59:02.009034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435712233424484:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:02.009052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:02.009223Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435709251027700:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27920) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435709251027777:2116] held: no
2025-11-19T13:59:02.009491Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435707403148196:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:02.009383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435707938457169:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.009393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:02.009397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435707938457169:2109] [node 6] ICP22 created new session: [1:7574435712233424486:2109]
2025-11-19T13:59:02.009402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435712233424486:2109] [node 6] ICS09 handshake done sender: [1:7574435707938457177:2115] self: [1:7574435707938457176:0] peer: [6:7574435707224736111:0] socket: 219 qp: -1
2025-11-19T13:59:02.009405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435712233424486:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:02.009410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:02.009547Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707938457181:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.009586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435707403148130:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.009600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:02.009607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435707403148130:2048] [node 1] ICP22 created new session: [3:7574435711698115519:2048]
2025-11-19T13:59:02.009613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435711698115519:2048] [node 1] ICS09 handshake done sender: [3:7574435707403148196:2109] self: [3:7574435707403148212:0] peer: [1:7574435707938457159:0] socket: 203 qp: -1
2025-11-19T13:59:02.009618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435711698115519:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:02.009624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:02.010703Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435709251027766:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.010711Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435709251027766:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.011332Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435712486905229:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.011420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435708191937851:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.011591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435712486905234:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.011765Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435709826016528:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.011777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435709826016528:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.011832Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435707403148205:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.011848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435707403148205:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.012009Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435708191937929:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:02.014809Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707938457175:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.014826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435707938457175:2114] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.016413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435711698115521:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4205
2025-11-19T13:59:02.017378Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435711698115521:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.020914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435709826016512:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(17171382ns) {HandleHandshakeStatus:329(17170723ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(128ns) SwitchToState:178(71ns) ProcessPendingSessionEvents:495(13145507ns) {ForwardSessionEventToSession:612(13132186ns) ForwardSessionEventToSession:612(1766ns) ForwardSessionEventToSession:612(592ns) ForwardSessionEventToSession:612(1169ns) ForwardSessionEventToSession:612(532ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(665ns) ForwardSessionEventToSession:612(580ns) ForwardSessionEventToSession:612(753ns) ForwardSessionEventToSession:612(817ns)}}}
2025-11-19T13:59:02.020106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435709251027700:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.020129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:02.020138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435709251027700:2050] [node 3] ICP22 created new session: [4:7574435713545995085:2050]
2025-11-19T13:59:02.020145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435713545995085:2050] [node 3] ICS09 handshake done sender: [4:7574435709251027777:2116] self: [4:7574435709251027780:0] peer: [3:7574435707403148201:0] socket: 229 qp: -1
2025-11-19T13:59:02.020152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435713545995085:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:02.020186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:02.021162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435713545995087:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42093
2025-11-19T13:59:02.023644Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435709251027766:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:02.023705Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435707403148217:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.023719Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435707403148217:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.023465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435712486905235:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
2025-11-19T13:59:02.023525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435708191937850:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.023539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:02.023548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435708191937850:2049] [node 2] ICP22 created new session: [5:7574435712486905236:2049]
2025-11-19T13:59:02.023555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435712486905236:2049] [node 2] ICS09 handshake done sender: [5:7574435708191937929:2116] self: [5:7574435708191937930:0] peer: [2:7574435709826016517:0] socket: 232 qp: -1
2025-11-19T13:59:02.023561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435712486905236:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:02.023569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:02.027662Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435707403148217:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.027866Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435707938457175:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: 2025-11-19T13:59:02.027865Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435707403148205:2113] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-19T13:59:02.027953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435711519703422:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15179
2025-11-19T13:59:02.028005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435709251027763:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.028029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:02.028037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435709251027763:2110] [node 6] ICP22 created new session: [4:7574435713545995089:2110]
2025-11-19T13:59:02.028045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435713545995089:2110] [node 6] ICS09 handshake done sender: [4:7574435709251027766:2111] self: [4:7574435709251027765:0] peer: [6:7574435711519703418:0] socket: 221 qp: -1
2025-11-19T13:59:02.028050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435713545995089:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:02.028058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.028207Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435707224736113:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:02.027871Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435709826016528:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:02.028028Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435714120983839:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.028118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435709826016512:2108] [node 5] ICP18 handshake (actor [2:7574435714120983839:2118]) is too old
2025-11-19T13:59:02.028134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435709826016513:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.028149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:02.028155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435709826016513:2107] [node 6] ICP22 created new session: [2:7574435714120983842:2107]
2025-11-19T13:59:02.028160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435714120983842:2107] [node 6] ICS09 handshake done sender: [2:7574435709826016528:2117] self: [2:7574435709826016516:0] peer: [6:7574435711519703411:0] socket: 241 qp: -1
2025-11-19T13:59:02.028166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435714120983842:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:02.028173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:02.028234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435707403148207:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.028248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:02.028254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435707403148207:2114] [node 5] ICP22 created new session: [3:7574435711698115525:2114]
2025-11-19T13:59:02.028260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435711698115525:2114] [node 5] ICS09 handshake done sender: [3:7574435707403148217:2118] self: [3:7574435707403148209:0] peer: [5:7574435712486905233:0] socket: 247 qp: -1
2025-11-19T13:59:02.028265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435711698115525:2114] [node 5] ICS10 traffic start
2025-11-19T13:59:02.028272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:02.028372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435707938457100:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.028390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:02.028395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435707938457100:2048] [node 2] ICP22 created new session: [1:7574435712233424492:2048]
2025-11-19T13:59:02.028400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435712233424492:2048] [node 2] ICS09 handshake done sender: [1:7574435707938457175:2114] self: [1:7574435707938457162:0] peer: [2:7574435709826016539:0] socket: 218 qp: -1
2025-11-19T13:59:02.028405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435712233424492:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:02.028410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:02.028534Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435712486905235:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.028617Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435713545995087:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.028542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435707224736031:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.028556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:02.028561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435707224736031:2050] [node 3] ICP22 created new session: [6:7574435711519703423:2050]
2025-11-19T13:59:02.028566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435711519703423:2050] [node 3] ICS09 handshake done sender: [6:7574435707224736113:2119] self: [6:7574435711519703412:0] peer: [3:7574435707403148204:0] socket: 244 qp: -1
2025-11-19T13:59:02.028570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435711519703423:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:02.028597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:02.028663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435711519703421:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15179
2025-11-19T13:59:02.028689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435711519703420:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15179
2025-11-19T13:59:02.028708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435714120983841:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11873
2025-11-19T13:59:02.029028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435707403148200:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.029041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:02.029048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435707403148200:2111] [node 6] ICP22 created new session: [3:7574435711698115527:2111]
2025-11-19T13:59:02.029053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435711698115527:2111] [node 6] ICS09 handshake done sender: [3:7574435707403148205:2113] self: [3:7574435707403148204:0] peer: [6:7574435711519703412:0] socket: 242 qp: -1
2025-11-19T13:59:02.029058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435711698115527:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:02.029063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:02.030413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435709826016447:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19708) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435709826016525:2116] held: no
2025-11-19T13:59:02.031576Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435709826016525:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:02.028908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435709251027698:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.032082Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707938457184:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.032095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435707938457184:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.033151Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435712486905228:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.033251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435708191937849:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.033310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435708191937908:2101] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.033462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435712486905241:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
2025-11-19T13:59:02.035435Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435707938457180:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.035443Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435707938457180:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.036211Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435712486905229:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:02.036246Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435707938457184:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:02.036386Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435707938457180:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:02.036473Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435709251027781:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.036481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435709251027781:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.036840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435707938457165:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.036857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:02.036862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435707938457165:2107] [node 4] ICP22 created new session: [1:7574435712233424496:2107]
2025-11-19T13:59:02.036867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435712233424496:2107] [node 4] ICS09 handshake done sender: [1:7574435707938457184:2118] self: [1:7574435707938457170:0] peer: [4:7574435713545995091:0] socket: 228 qp: -1
2025-11-19T13:59:02.036872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435712233424496:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:02.036879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:02.036958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435707938457167:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.036967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:02.036970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435707938457167:2108] [node 5] ICP22 created new session: [1:7574435712233424498:2108]
2025-11-19T13:59:02.036973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435712233424498:2108] [node 5] ICS09 handshake done sender: [1:7574435707938457180:2116] self: [1:7574435707938457174:0] peer: [5:7574435712486905239:0] socket: 223 qp: -1
2025-11-19T13:59:02.036976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435712233424498:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:02.036979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:02.036747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435711698115529:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4205
2025-11-19T13:59:02.037267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435708191937851:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.037280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:02.037285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435708191937851:2050] [node 3] ICP22 created new session: [5:7574435712486905243:2050]
2025-11-19T13:59:02.037289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435712486905243:2050] [node 3] ICS09 handshake done sender: [5:7574435712486905229:2118] self: [5:7574435712486905233:0] peer: [3:7574435707403148209:0] socket: 249 qp: -1
2025-11-19T13:59:02.037293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435712486905243:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:02.037299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.037429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435712486905242:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.038270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435713545995092:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42093
2025-11-19T13:59:02.038539Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435709251027781:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:02.048026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435709251027762:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.048058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:02.048067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435709251027762:2109] [node 5] ICP22 created new session: [4:7574435713545995094:2109]
2025-11-19T13:59:02.048074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435713545995094:2109] [node 5] ICS09 handshake done sender: [4:7574435709251027781:2118] self: [4:7574435709251027764:0] peer: [5:7574435712486905240:0] socket: 246 qp: -1
2025-11-19T13:59:02.048083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435713545995094:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:02.048108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:02.049022Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435709251027698:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19708) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435713545995087:2121] held: no
2025-11-19T13:59:02.049443Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435707224736112:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:02.049492Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435707224736110:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:02.049575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435707224736030:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24878) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435707224736112:2118] held: no
2025-11-19T13:59:02.049601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435707224736047:2062] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28068) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435707224736110:2117] held: no
2025-11-19T13:59:02.049790Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435713545995087:2121] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:02.052736Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435712486905235:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:02.055628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435708191937849:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19708) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435712486905235:2120] held: no
2025-11-19T13:59:02.060013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435707403148207:2114] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:02.074410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435709826016447:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.074438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:02.074449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435709826016447:2048] [node 1] ICP22 created new session: [2:7574435714120983844:2048]
2025-11-19T13:59:02.074455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435714120983844:2048] [node 1] ICS09 handshake done sender: [2:7574435709826016525:2116] self: [2:7574435709826016539:0] peer: [1:7574435707938457162:0] socket: 225 qp: -1
2025-11-19T13:59:02.074464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435714120983844:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:02.074475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:02.075973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435709251027698:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.075994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:02.076000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435709251027698:2048] [node 1] ICP22 created new session: [4:7574435713545995096:2048]
2025-11-19T13:59:02.076007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435713545995096:2048] [node 1] ICS09 handshake done sender: [4:7574435713545995087:2121] self: [4:7574435713545995091:0] peer: [1:7574435707938457170:0] socket: 209 qp: -1
2025-11-19T13:59:02.076011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435713545995096:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:02.076018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:02.076255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435707224736047:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.076270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:02.076275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435707224736047:2062] [node 4] ICP22 created new session: [6:7574435711519703425:2062]
2025-11-19T13:59:02.076280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435711519703425:2062] [node 4] ICS09 handshake done sender: [6:7574435707224736110:2117] self: [6:7574435711519703418:0] peer: [4:7574435709251027765:0] socket: 231 qp: -1
2025-11-19T13:59:02.076285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435711519703425:2062] [node 4] ICS10 traffic start
2025-11-19T13:59:02.076290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:02.076355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435707224736030:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.076360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:02.076364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435707224736030:2049] [node 2] ICP22 created new session: [6:7574435711519703427:2049]
2025-11-19T13:59:02.076368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435711519703427:2049] [node 2] ICS09 handshake done sender: [6:7574435707224736112:2118] self: [6:7574435711519703411:0] peer: [2:7574435709826016516:0] socket: 243 qp: -1
2025-11-19T13:59:02.076372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435711519703427:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:02.076377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.076857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435708191937908:2101] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28068) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435712486905228:2117] held: no
2025-11-19T13:59:02.076880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435708191937849:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.076888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:02.076893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435708191937849:2048] [node 1] ICP22 created new session: [5:7574435712486905246:2048]
2025-11-19T13:59:02.076897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435712486905246:2048] [node 1] ICS09 handshake done sender: [5:7574435712486905235:2120] self: [5:7574435712486905239:0] peer: [1:7574435707938457174:0] socket: 205 qp: -1
2025-11-19T13:59:02.076901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435712486905246:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:02.076907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:02.077022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435712486905245:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52026
2025-11-19T13:59:02.077569Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435712486905228:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:02.077739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435708191937908:2101] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.077745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:02.077751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435708191937908:2101] [node 4] ICP22 created new session: [5:7574435712486905248:2101]
2025-11-19T13:59:02.077755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435712486905248:2101] [node 4] ICS09 handshake done sender: [5:7574435712486905228:2117] self: [5:7574435712486905240:0] peer: [4:7574435709251027764:0] socket: 248 qp: -1
2025-11-19T13:59:02.077759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435712486905248:2101] [node 4] ICS10 traffic start
2025-11-19T13:59:02.077768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:02.093719Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:02.093810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435714120983844:2048] [node 1] ICS01 socket: 225 reason# EndOfStream
2025-11-19T13:59:02.093818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435709826016447:2048] [node 1] ICP30 unregister session Session# [2:7574435714120983844:2048] VirtualId# [2:7574435709826016539:0]
2025-11-19T13:59:02.093825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435714120983844:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.093865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:02.093887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:02.096296Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:02.096375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435711698115519:2048] [node 1] ICS01 socket: 203 reason# EndOfStream
2025-11-19T13:59:02.096382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435707403148130:2048] [node 1] ICP30 unregister session Session# [3:7574435711698115519:2048] VirtualId# [3:7574435707403148212:0]
2025-11-19T13:59:02.096389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435711698115519:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.096420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:02.096439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:02.106302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:02.106394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435713545995096:2048] [node 1] ICS01 socket: 209 reason# EndOfStream
2025-11-19T13:59:02.106404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435709251027698:2048] [node 1] ICP30 unregister session Session# [4:7574435713545995096:2048] VirtualId# [4:7574435713545995091:0]
2025-11-19T13:59:02.106411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435713545995096:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.106478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:02.106502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:02.106541Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:02.106596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435712486905246:2048] [node 1] ICS01 socket: 205 reason# EndOfStream
2025-11-19T13:59:02.106600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435708191937849:2048] [node 1] ICP30 unregister session Session# [5:7574435712486905246:2048] VirtualId# [5:7574435712486905239:0]
2025-11-19T13:59:02.106604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435712486905246:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.106632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:02.106651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:02.106679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:02.106730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435711519703416:2048] [node 1] ICS01 socket: 220 reason# EndOfStream
2025-11-19T13:59:02.106734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435707224736029:2048] [node 1] ICP30 unregister session Session# [6:7574435711519703416:2048] VirtualId# [6:7574435707224736111:0]
2025-11-19T13:59:02.106738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435711519703416:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.106764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:02.106782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:02.110126Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:02.110202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435707403148215:2049] [node 2] ICS01 socket: 237 reason# EndOfStream
2025-11-19T13:59:02.110209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435707403148131:2049] [node 2] ICP30 unregister session Session# [3:7574435707403148215:2049] VirtualId# [3:7574435707403148213:0]
2025-11-19T13:59:02.110216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435707403148215:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.110249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:02.110266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:02.116004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:02.116094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435711519703427:2049] [node 2] ICS01 socket: 243 reason# EndOfStream
2025-11-19T13:59:02.116103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435707224736030:2049] [node 2] ICP30 unregister session Session# [6:7574435711519703427:2049] VirtualId# [6:7574435711519703411:0]
2025-11-19T13:59:02.116111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435711519703427:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.116166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:02.116193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:02.116302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:02.116366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435712486905236:2049] [node 2] ICS07 socket disconnect 232 reason# EndOfStream
2025-11-19T13:59:02.116371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435712486905236:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.116418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:02.116438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435712486905236:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:02.116753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:02.116817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435709251027785:2049] [node 2] ICS01 socket: 230 reason# EndOfStream
2025-11-19T13:59:02.116822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435709251027699:2049] [node 2] ICP30 unregister session Session# [4:7574435709251027785:2049] VirtualId# [4:7574435709251027782:0]
2025-11-19T13:59:02.116826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435709251027785:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.116855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:02.116888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:02.117314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435708191937850:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24878) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:02.117327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435712486905236:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:02.117331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435708191937850:2049] [node 2] ICP30 unregister session Session# [5:7574435712486905236:2049] VirtualId# [5:7574435708191937930:0]
2025-11-19T13:59:02.117344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:02.117349Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435708191937850:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24878) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:02.117352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24878) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:02.127895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435708191937850:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:02.144206Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:02.144301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435711519703423:2050] [node 3] ICS01 socket: 244 reason# EndOfStream
2025-11-19T13:59:02.144310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435707224736031:2050] [node 3] ICP30 unregister session Session# [6:7574435711519703423:2050] VirtualId# [6:7574435711519703412:0]
2025-11-19T13:59:02.144316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435711519703423:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.144372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:02.144395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:02.144438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:02.144505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435712486905243:2050] [node 3] ICS01 socket: 249 reason# EndOfStream
2025-11-19T13:59:02.144510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435708191937851:2050] [node 3] ICP30 unregister session Session# [5:7574435712486905243:2050] VirtualId# [5:7574435712486905233:0]
2025-11-19T13:59:02.144514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435712486905243:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.144539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:02.144559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:02.158843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:02.158944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435711519703425:2062] [node 4] ICS01 socket: 231 reason# EndOfStream
2025-11-19T13:59:02.158954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435707224736047:2062] [node 4] ICP30 unregister session Session# [6:7574435711519703425:2062] VirtualId# [6:7574435711519703418:0]
2025-11-19T13:59:02.158961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435711519703425:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.159029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:02.159054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:02.403880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435714841679667:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:02.403893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435714841679668:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:02.443858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435711977855140:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:02.443875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435711977855141:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:02.423896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435714356685135:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:02.423911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435714356685136:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:02.508953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435713049099854:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:02.508968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435713049099855:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:02.508973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435713049099856:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:02.576018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435713428137432:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:02.576035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435713428137433:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:02.576040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435713428137434:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:02.637706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435714960930148:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:02.637721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435714960930149:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:02.637726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435714960930150:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:02.696229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435714960930161:2059] [node 5] ICP01 ready to work
2025-11-19T13:59:02.698067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435713428137460:2069] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.698717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435713049099911:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1376
2025-11-19T13:59:02.700213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435714841679730:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.704645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435713428137472:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45069
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.707735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435711977855200:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5652
2025-11-19T13:59:02.708190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435711977855201:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5652
2025-11-19T13:59:02.708250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435714960930188:2077] [node 4] ICP01 ready to work
2025-11-19T13:59:02.708241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435713428137456:2061] [node 6] ICP01 ready to work
2025-11-19T13:59:02.708850Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435713049099911:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.709144Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435711977855200:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.709269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435711977855203:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:02.709624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435711977855203:2109] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.713508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435711977855206:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:02.716579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435714960930149:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16639639ns) {Configure:104(16638786ns) {ProcessConfigured:135(16306129ns) {StartInitialHandshake:157(16305325ns) {DropHandshakes:469(1017ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:476(1010ns) {SwitchToState:178(544ns)}}}}}
2025-11-19T13:59:02.716861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435714841679730:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16591989ns) {Configure:104(16591433ns) {ProcessConfigured:135(16223184ns) {StartInitialHandshake:157(16222271ns) {DropHandshakes:469(460ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(77ns)} GenerateSessionVirtualId:988(264ns) PrepareNewSessionHandshake:476(655ns) {SwitchToState:178(262ns)}}}}}
2025-11-19T13:59:02.718134Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435713428137448:2062] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.718152Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435713428137448:2062] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.717516Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435714960930148:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17018911ns) {Configure:104(17018062ns) {ProcessConfigured:135(16784097ns) {StartInitialHandshake:157(16783786ns) {DropHandshakes:469(324ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(582ns) {SwitchToState:178(406ns)}}}}}
2025-11-19T13:59:02.722026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435714356685204:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:02.725008Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435713428137472:2078] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.729831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435713049099916:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.745042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435714960930225:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12081
2025-11-19T13:59:02.745406Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435714960930225:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.745461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435714960930148:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.745623Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435714841679737:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.745634Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435714841679737:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.746557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435714960930227:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12081
2025-11-19T13:59:02.746694Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435713428137448:2062] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:02.746817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435713428137434:2050] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.746837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:02.746845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435713428137434:2050] [node 3] ICP22 created new session: [5:7574435713428137510:2050]
2025-11-19T13:59:02.746852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435713428137510:2050] [node 3] ICS09 handshake done sender: [5:7574435713428137448:2062] self: [5:7574435713428137447:0] peer: [3:7574435711977855204:0] socket: 204 qp: -1
2025-11-19T13:59:02.746860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435713428137510:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:02.746892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:02.747299Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435714841679737:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:02.747522Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435714960930225:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:02.747623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435714960930148:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.747637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:02.747643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435714960930148:2048] [node 1] ICP22 created new session: [6:7574435714960930228:2048]
2025-11-19T13:59:02.747648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435714960930228:2048] [node 1] ICS09 handshake done sender: [6:7574435714960930225:2114] self: [6:7574435714960930226:0] peer: [1:7574435714841679731:0] socket: 218 qp: -1
2025-11-19T13:59:02.747654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435714960930228:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:02.747669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:02.747954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435714841679730:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.747969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:02.747974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435714841679730:2108] [node 6] ICP22 created new session: [1:7574435714841679743:2108]
2025-11-19T13:59:02.747979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435714841679743:2108] [node 6] ICS09 handshake done sender: [1:7574435714841679737:2111] self: [1:7574435714841679731:0] peer: [6:7574435714960930226:0] socket: 216 qp: -1
2025-11-19T13:59:02.747985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435714841679743:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:02.748001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:02.747874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435713049099854:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(27411801ns) {Configure:104(27410872ns) {ProcessConfigured:135(27163076ns) {StartInitialHandshake:157(27162442ns) {DropHandshakes:469(241ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(149ns) PrepareNewSessionHandshake:476(1059ns) {SwitchToState:178(670ns)}}}}}
2025-11-19T13:59:02.752493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435713049099912:2103] [node 6] ICP01 ready to work
2025-11-19T13:59:02.752288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435714841679729:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.759859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435713428137432:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(59043392ns) {Configure:104(59042620ns) {ProcessConfigured:135(58827694ns) {StartInitialHandshake:157(58826806ns) {DropHandshakes:469(183ns) {DropIncomingHandshake:441(13ns) DropOutgoingHandshake:456(42ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:476(1263ns) {SwitchToState:178(578ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.760183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435711977855216:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5652
2025-11-19T13:59:02.760234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435711977855217:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5652
2025-11-19T13:59:02.760262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435711977855218:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5652
2025-11-19T13:59:02.760440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435713049099931:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1376
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.761171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435713049099916:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(30536626ns) {Configure:104(30536351ns) {ProcessConfigured:135(30216432ns) {StartInitialHandshake:157(30201639ns) {DropHandshakes:469(575ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(161ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:476(991ns) {SwitchToState:178(193ns)}} IncomingHandshake:261(5386ns) {DropIncomingHandshake:441(193ns) PrepareNewSessionHandshake:476(133ns) {SwitchToState:178(81ns)} GenerateSessionVirtualId:988(183ns) IssueIncomingHandshakeReply:193(396ns)}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.763866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435713049099934:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1376
2025-11-19T13:59:02.764183Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435711977855216:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.764212Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435711977855217:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.764267Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435711977855200:2107] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.764399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435711977855203:2109] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.764413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:02.764419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435711977855203:2109] [node 5] ICP22 created new session: [3:7574435711977855220:2109]
2025-11-19T13:59:02.764424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435711977855220:2109] [node 5] ICS09 handshake done sender: [3:7574435711977855200:2107] self: [3:7574435711977855204:0] peer: [5:7574435713428137447:0] socket: 205 qp: -1
2025-11-19T13:59:02.764431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435711977855220:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:02.764449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:02.764636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435711977855219:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5652
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.762033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435714960930188:2077] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(53741249ns) {Configure:104(53740598ns) {ProcessConfigured:135(53501181ns) {StartInitialHandshake:157(53500825ns) {DropHandshakes:469(454ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(76ns)} GenerateSessionVirtualId:988(91ns) PrepareNewSessionHandshake:476(621ns) {SwitchToState:178(148ns)}}}}}
2025-11-19T13:59:02.765200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435714960930232:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12081
2025-11-19T13:59:02.761316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435714356685136:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(39200087ns) {Configure:104(39199712ns) {ProcessConfigured:135(38926830ns) {StartInitialHandshake:157(38926403ns) {DropHandshakes:469(456ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(72ns) PrepareNewSessionHandshake:476(635ns) {SwitchToState:178(149ns)}}}}}
2025-11-19T13:59:02.761707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435714356685201:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44030
2025-11-19T13:59:02.761750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435714356685202:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44030
2025-11-19T13:59:02.761778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435714356685203:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44030
2025-11-19T13:59:02.761834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435714356685209:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44030
2025-11-19T13:59:02.761862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435714356685210:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44030
2025-11-19T13:59:02.764856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435714356685201:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.764877Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435714356685209:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.764889Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435714356685203:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.765041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435714356685135:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(53114891ns) {Configure:104(53113662ns) {ProcessConfigured:135(52726794ns) {StartInitialHandshake:157(52726532ns) {DropHandshakes:469(512ns) {DropIncomingHandshake:441(90ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:476(528ns) {SwitchToState:178(215ns)}}}}}
2025-11-19T13:59:02.766240Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435714356685202:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.766269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435714356685135:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.766488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435714841679736:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:02.766982Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435713049099934:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.764936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435713428137456:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(56484780ns) {Configure:104(56484173ns) {ProcessConfigured:135(56243903ns) {StartInitialHandshake:157(56243416ns) {DropHandshakes:469(420ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(47ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:476(918ns) {SwitchToState:178(524ns)}}}}}
2025-11-19T13:59:02.773079Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435713049099912:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20521601ns) {Configure:104(20520913ns) {ProcessConfigured:135(20236500ns) {StartInitialHandshake:157(20235857ns) {DropHandshakes:469(846ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(291ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:476(836ns) {SwitchToState:178(403ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.771900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435711977855140:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(54494749ns) {Configure:104(54494015ns) {ProcessConfigured:135(54226309ns) {StartInitialHandshake:157(54225240ns) {DropHandshakes:469(462ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(267ns) PrepareNewSessionHandshake:476(1221ns) {SwitchToState:178(697ns)}}}}}
2025-11-19T13:59:02.771949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435711977855140:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.775511Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435711977855201:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.773535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435714356685217:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44030
2025-11-19T13:59:02.772225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435714841679729:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19871425ns) {Configure:104(19870378ns) {ProcessConfigured:135(19558519ns) {StartInitialHandshake:157(19557598ns) {DropHandshakes:469(701ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(168ns)} GenerateSessionVirtualId:988(578ns) PrepareNewSessionHandshake:476(540ns) {SwitchToState:178(276ns)}}}}}
2025-11-19T13:59:02.772269Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435714841679733:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.772272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435714841679733:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.781114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435714356685199:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:02.781534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435714356685200:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.787435Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435714960930232:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.787488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435714960930149:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.787902Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435714356685208:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.787914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435714356685208:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.783771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435711977855211:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.789635Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435714356685208:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:02.784898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435713428137516:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45069
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.792493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435711977855229:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5652
2025-11-19T13:59:02.792566Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435713049099931:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.792605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435713049099856:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.792753Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435713428137516:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.792786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435713428137432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.792963Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435711977855212:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.792967Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435711977855212:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.793653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435713049099938:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1376
2025-11-19T13:59:02.793817Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435711977855212:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:02.793983Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435713049099931:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:02.794083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435713049099856:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.794105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:02.794111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435713049099856:2050] [node 3] ICP22 created new session: [4:7574435713049099939:2050]
2025-11-19T13:59:02.794117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435713049099939:2050] [node 3] ICS09 handshake done sender: [4:7574435713049099931:2114] self: [4:7574435713049099937:0] peer: [3:7574435711977855207:0] socket: 226 qp: -1
2025-11-19T13:59:02.794124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435713049099939:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:02.794131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:02.788798Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435714841679733:2109] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:02.788920Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435714841679736:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(22365435ns) {Configure:104(22364690ns) {ProcessConfigured:135(22059928ns) {StartInitialHandshake:157(22059547ns) {DropHandshakes:469(464ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(45ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:476(530ns) {SwitchToState:178(97ns)}}}}}
2025-11-19T13:59:02.788972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435714841679750:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19206
2025-11-19T13:59:02.790658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435714841679752:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19206
2025-11-19T13:59:02.790696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435714841679753:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19206
2025-11-19T13:59:02.790721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435714841679754:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19206
2025-11-19T13:59:02.791642Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435714841679739:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.791649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435714841679739:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.792164Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435714841679750:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.792226Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435714841679752:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.792240Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435714841679754:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.796721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435711977855206:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.796734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:02.796739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435711977855206:2110] [node 4] ICP22 created new session: [3:7574435711977855231:2110]
2025-11-19T13:59:02.796743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435711977855231:2110] [node 4] ICS09 handshake done sender: [3:7574435711977855212:2112] self: [3:7574435711977855207:0] peer: [4:7574435713049099937:0] socket: 224 qp: -1
2025-11-19T13:59:02.796747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435711977855231:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:02.796752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:02.797676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435714356685204:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.797694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:02.797699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435714356685204:2109] [node 6] ICP22 created new session: [2:7574435714356685223:2109]
2025-11-19T13:59:02.797704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435714356685223:2109] [node 6] ICS09 handshake done sender: [2:7574435714356685208:2114] self: [2:7574435714356685207:0] peer: [6:7574435714960930233:0] socket: 238 qp: -1
2025-11-19T13:59:02.797709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435714356685223:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:02.797715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:02.795947Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435714841679739:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:02.796291Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435714841679747:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.796294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435714841679747:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.796598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435714841679667:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.796609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:02.796616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435714841679667:2048] [node 2] ICP22 created new session: [1:7574435714841679759:2048]
2025-11-19T13:59:02.796621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435714841679759:2048] [node 2] ICS09 handshake done sender: [1:7574435714841679733:2109] self: [1:7574435714841679732:0] peer: [2:7574435714356685216:0] socket: 213 qp: -1
2025-11-19T13:59:02.796627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435714841679759:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:02.796641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:02.797206Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435714841679747:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:02.797270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435714841679729:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.797283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:02.797288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435714841679729:2107] [node 5] ICP22 created new session: [1:7574435714841679763:2107]
2025-11-19T13:59:02.797293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435714841679763:2107] [node 5] ICS09 handshake done sender: [1:7574435714841679747:2113] self: [1:7574435714841679745:0] peer: [5:7574435713428137517:0] socket: 246 qp: -1
2025-11-19T13:59:02.797297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435714841679763:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:02.797304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.801168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435714960930234:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12081
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.801280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435713428137518:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45069
2025-11-19T13:59:02.801317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435713428137519:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45069
2025-11-19T13:59:02.801847Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435711977855219:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.801862Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435711977855216:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:02.802211Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435713428137518:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.802244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435713428137460:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.802387Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435713049099932:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.802393Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435713049099932:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.800591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435714841679668:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.800614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:02.800620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435714841679668:2049] [node 3] ICP22 created new session: [1:7574435714841679765:2049]
2025-11-19T13:59:02.800626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435714841679765:2049] [node 3] ICS09 handshake done sender: [1:7574435714841679739:2112] self: [1:7574435714841679738:0] peer: [3:7574435711977855224:0] socket: 217 qp: -1
2025-11-19T13:59:02.800631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435714841679765:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:02.800636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:02.801927Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435714841679753:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.803335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435711977855141:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.803654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435713428137521:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45069
2025-11-19T13:59:02.801630Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435714356685202:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:02.803845Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435713428137516:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:02.803097Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435714960930232:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:02.803506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435714960930235:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12081
2025-11-19T13:59:02.803888Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435713049099932:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:02.804366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435713049099916:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.804381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:02.804387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435713049099916:2107] [node 5] ICP22 created new session: [4:7574435713049099943:2107]
2025-11-19T13:59:02.804392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435713049099943:2107] [node 5] ICS09 handshake done sender: [4:7574435713049099932:2115] self: [4:7574435713049099927:0] peer: [5:7574435713428137520:0] socket: 258 qp: -1
2025-11-19T13:59:02.804397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435713049099943:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:02.804403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:02.804932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435714356685135:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.804948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:02.804953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435714356685135:2048] [node 1] ICP22 created new session: [2:7574435714356685225:2048]
2025-11-19T13:59:02.804958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435714356685225:2048] [node 1] ICS09 handshake done sender: [2:7574435714356685202:2111] self: [2:7574435714356685216:0] peer: [1:7574435714841679732:0] socket: 219 qp: -1
2025-11-19T13:59:02.804962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435714356685225:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:02.804977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:02.805086Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435714356685206:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.805089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435714356685206:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.804867Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435713428137518:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:02.805298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435713428137432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.805307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:02.805314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435713428137432:2048] [node 1] ICP22 created new session: [5:7574435713428137522:2048]
2025-11-19T13:59:02.805320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435713428137522:2048] [node 1] ICS09 handshake done sender: [5:7574435713428137516:2116] self: [5:7574435713428137517:0] peer: [1:7574435714841679745:0] socket: 248 qp: -1
2025-11-19T13:59:02.805325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435713428137522:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:02.805341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.808012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435711977855140:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.808022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:02.808027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435711977855140:2048] [node 1] ICP22 created new session: [3:7574435711977855235:2048]
2025-11-19T13:59:02.808031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435711977855235:2048] [node 1] ICS09 handshake done sender: [3:7574435711977855216:2114] self: [3:7574435711977855224:0] peer: [1:7574435714841679738:0] socket: 221 qp: -1
2025-11-19T13:59:02.808034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435711977855235:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:02.808039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:02.807285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435714960930149:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65195) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435714960930232:2117] held: no
2025-11-19T13:59:02.807338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435714960930236:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12081
2025-11-19T13:59:02.807911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435714960930149:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.807921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:02.807926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435714960930149:2049] [node 2] ICP22 created new session: [6:7574435714960930237:2049]
2025-11-19T13:59:02.807932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435714960930237:2049] [node 2] ICS09 handshake done sender: [6:7574435714960930232:2117] self: [6:7574435714960930233:0] peer: [2:7574435714356685207:0] socket: 241 qp: -1
2025-11-19T13:59:02.809771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435711977855140:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:02.807937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435714960930237:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:02.807955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:02.809928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435714356685200:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(28341716ns) {Configure:104(28340878ns) {ProcessConfigured:135(28126150ns) {StartInitialHandshake:157(28118388ns) {DropHandshakes:469(196ns) {DropIncomingHandshake:441(41ns) DropOutgoingHandshake:456(31ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:476(828ns) {SwitchToState:178(286ns)}} IncomingHandshake:261(3259ns) {DropIncomingHandshake:441(57ns) PrepareNewSessionHandshake:476(127ns) {SwitchToState:178(93ns)} GenerateSessionVirtualId:988(316ns) IssueIncomingHandshakeReply:193(222ns)}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.810298Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435714356685206:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:02.810442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435711977855237:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5652
2025-11-19T13:59:02.811054Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435714356685210:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.811124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435714356685136:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.811131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:02.811136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435714356685136:2049] [node 3] ICP22 created new session: [2:7574435714356685232:2049]
2025-11-19T13:59:02.811140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435714356685232:2049] [node 3] ICS09 handshake done sender: [2:7574435714356685206:2113] self: [2:7574435714356685205:0] peer: [3:7574435711977855234:0] socket: 237 qp: -1
2025-11-19T13:59:02.811143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435714356685232:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:02.811148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:02.810783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435713049099946:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1376
2025-11-19T13:59:02.810797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435713049099947:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1376
2025-11-19T13:59:02.811911Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435711977855219:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:02.813294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435714356685136:2049] [node 3] ICP18 handshake (actor [2:7574435714356685210:2116]) is too old
2025-11-19T13:59:02.810570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435714841679667:2048] [node 2] ICP18 handshake (actor [1:7574435714841679753:2117]) is too old
2025-11-19T13:59:02.813449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435711977855141:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.813461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:02.813465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435711977855141:2049] [node 2] ICP22 created new session: [3:7574435711977855238:2049]
2025-11-19T13:59:02.813470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435711977855238:2049] [node 2] ICS09 handshake done sender: [3:7574435711977855219:2117] self: [3:7574435711977855234:0] peer: [2:7574435714356685205:0] socket: 242 qp: -1
2025-11-19T13:59:02.813474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435711977855238:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:02.813479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:02.814366Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435711977855227:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.814372Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435711977855227:2119] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.815430Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435711977855227:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:02.817113Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435713049099947:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.817140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435713049099855:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.817298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435711977855211:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.817312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:02.817316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435711977855211:2111] [node 6] ICP22 created new session: [3:7574435711977855241:2111]
2025-11-19T13:59:02.817321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435711977855241:2111] [node 6] ICS09 handshake done sender: [3:7574435711977855227:2119] self: [3:7574435711977855226:0] peer: [6:7574435714960930239:0] socket: 233 qp: -1
2025-11-19T13:59:02.817325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435711977855241:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:02.817329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:02.814190Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435714960930235:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.814219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435714960930150:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.815190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435714960930240:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12081
2025-11-19T13:59:02.815572Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435714960930235:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:02.815631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435714960930150:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.815640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:02.815643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435714960930150:2050] [node 3] ICP22 created new session: [6:7574435714960930241:2050]
2025-11-19T13:59:02.815648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435714960930241:2050] [node 3] ICS09 handshake done sender: [6:7574435714960930235:2119] self: [6:7574435714960930239:0] peer: [3:7574435711977855226:0] socket: 264 qp: -1
2025-11-19T13:59:02.815652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435714960930241:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:02.815658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:02.817856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435713428137460:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.817868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:02.817874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435713428137460:2069] [node 4] ICP22 created new session: [5:7574435713428137524:2069]
2025-11-19T13:59:02.817878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435713428137524:2069] [node 4] ICS09 handshake done sender: [5:7574435713428137518:2117] self: [5:7574435713428137520:0] peer: [4:7574435713049099927:0] socket: 259 qp: -1
2025-11-19T13:59:02.817883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435713428137524:2069] [node 4] ICS10 traffic start
2025-11-19T13:59:02.817888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:02.818206Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435713049099946:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.818225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435713049099854:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.818408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435713428137460:2069] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:02.818480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435713428137526:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45069
2025-11-19T13:59:02.818652Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435714841679751:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.818661Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435714841679751:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.818896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435714356685204:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(21193840ns) {HandleHandshakeStatus:329(21193365ns) {DropIncomingHandshake:441(3721ns) DropOutgoingHandshake:456(130ns) SwitchToState:178(235ns) ProcessPendingSessionEvents:495(18248370ns) {ForwardSessionEventToSession:612(18238639ns) ForwardSessionEventToSession:612(1140ns) ForwardSessionEventToSession:612(1097ns) ForwardSessionEventToSession:612(560ns) ForwardSessionEventToSession:612(551ns) ForwardSessionEventToSession:612(732ns) ForwardSessionEventToSession:612(526ns) ForwardSessionEventToSession:612(508ns) ForwardSessionEventToSession:612(630ns) ForwardSessionEventToSession:612(1239ns)}}}
2025-11-19T13:59:02.819653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435714841679668:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(22722412ns) {HandleHandshakeStatus:329(22721898ns) {DropIncomingHandshake:441(3386ns) DropOutgoingHandshake:456(138ns) SwitchToState:178(224ns) ProcessPendingSessionEvents:495(10279378ns) {ForwardSessionEventToSession:612(10270704ns) ForwardSessionEventToSession:612(1537ns) ForwardSessionEventToSession:612(776ns) ForwardSessionEventToSession:612(521ns) ForwardSessionEventToSession:612(524ns) ForwardSessionEventToSession:612(774ns) ForwardSessionEventToSession:612(651ns) ForwardSessionEventToSession:612(604ns) ForwardSessionEventToSession:612(450ns) ForwardSessionEventToSession:612(882ns)}}}
2025-11-19T13:59:02.823882Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435713428137432:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(18557052ns) {HandleHandshakeStatus:329(18556456ns) {DropIncomingHandshake:441(259ns) DropOutgoingHandshake:456(1012ns) SwitchToState:178(223ns) ProcessPendingSessionEvents:495(14466251ns) {ForwardSessionEventToSession:612(14456165ns) ForwardSessionEventToSession:612(1581ns) ForwardSessionEventToSession:612(1100ns) ForwardSessionEventToSession:612(611ns) ForwardSessionEventToSession:612(609ns) ForwardSessionEventToSession:612(1004ns) ForwardSessionEventToSession:612(542ns) ForwardSessionEventToSession:612(569ns) ForwardSessionEventToSession:612(500ns) ForwardSessionEventToSession:612(996ns)}}}
2025-11-19T13:59:02.826864Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435713428137526:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.826894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435713428137433:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.827036Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435714356685221:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.827043Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435714356685221:2119] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.827596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435713428137529:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45069
2025-11-19T13:59:02.827725Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435714356685221:2119] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:02.827893Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435713428137526:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:02.827925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435713428137433:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.827937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:02.827942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435713428137433:2049] [node 2] ICP22 created new session: [5:7574435713428137530:2049]
2025-11-19T13:59:02.827950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435713428137530:2049] [node 2] ICS09 handshake done sender: [5:7574435713428137526:2120] self: [5:7574435713428137528:0] peer: [2:7574435714356685218:0] socket: 214 qp: -1
2025-11-19T13:59:02.827956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435713428137530:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:02.827962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:02.828090Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435714356685229:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.828110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435714356685229:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.828119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435714356685199:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.828129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:02.828133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435714356685199:2108] [node 5] ICP22 created new session: [2:7574435714356685235:2108]
2025-11-19T13:59:02.828138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435714356685235:2108] [node 5] ICS09 handshake done sender: [2:7574435714356685221:2119] self: [2:7574435714356685218:0] peer: [5:7574435713428137528:0] socket: 249 qp: -1
2025-11-19T13:59:02.828142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435714356685235:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:02.828147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:02.828923Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435714960930236:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.829113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435714960930188:2077] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.829135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435714960930243:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12081
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.833202Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435714356685229:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:02.833298Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435713049099935:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.833308Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435713049099935:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:02.833925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435713049099950:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1376
2025-11-19T13:59:02.834466Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435713049099935:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:02.834570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435714356685200:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.834589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:02.834594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435714356685200:2107] [node 4] ICP22 created new session: [2:7574435714356685238:2107]
2025-11-19T13:59:02.834600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435714356685238:2107] [node 4] ICS09 handshake done sender: [2:7574435714356685229:2120] self: [2:7574435714356685219:0] peer: [4:7574435713049099948:0] socket: 266 qp: -1
2025-11-19T13:59:02.834605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435714356685238:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:02.834612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:02.834815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435713049099912:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.834830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:02.834835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435713049099912:2103] [node 6] ICP22 created new session: [4:7574435713049099952:2103]
2025-11-19T13:59:02.834839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435713049099952:2103] [node 6] ICS09 handshake done sender: [4:7574435713049099935:2117] self: [4:7574435713049099930:0] peer: [6:7574435714960930244:0] socket: 220 qp: -1
2025-11-19T13:59:02.834844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435713049099952:2103] [node 6] ICS10 traffic start
2025-11-19T13:59:02.834849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:02.835016Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435713049099947:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:02.835069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435713049099855:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.835080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:02.835085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435713049099855:2049] [node 2] ICP22 created new session: [4:7574435713049099954:2049]
2025-11-19T13:59:02.835089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435713049099954:2049] [node 2] ICS09 handshake done sender: [4:7574435713049099947:2120] self: [4:7574435713049099948:0] peer: [2:7574435714356685219:0] socket: 268 qp: -1
2025-11-19T13:59:02.835093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435713049099954:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:02.835098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:02.840274Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435714960930243:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.840355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435714960930161:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:02.841700Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435713428137514:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:02.841712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435713428137514:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.843103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435713049099956:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1376
2025-11-19T13:59:02.844221Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435713428137514:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:02.844400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435714960930246:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12081
2025-11-19T13:59:02.844478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435714960930247:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12081
2025-11-19T13:59:02.844615Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435714960930243:2122] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:02.845852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435713428137456:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.845872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:02.845879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435713428137456:2061] [node 6] ICP22 created new session: [5:7574435713428137533:2061]
2025-11-19T13:59:02.845885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435713428137533:2061] [node 6] ICS09 handshake done sender: [5:7574435713428137514:2115] self: [5:7574435713428137481:0] peer: [6:7574435714960930245:0] socket: 207 qp: -1
2025-11-19T13:59:02.845891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435713428137533:2061] [node 6] ICS10 traffic start
2025-11-19T13:59:02.845898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:02.846290Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435714960930188:2077] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24581) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435714960930236:2120] held: no
2025-11-19T13:59:02.847093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435714960930161:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.847105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:02.847110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435714960930161:2059] [node 5] ICP22 created new session: [6:7574435714960930248:2059]
2025-11-19T13:59:02.847116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435714960930248:2059] [node 5] ICS09 handshake done sender: [6:7574435714960930243:2122] self: [6:7574435714960930245:0] peer: [5:7574435713428137481:0] socket: 232 qp: -1
2025-11-19T13:59:02.847121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435714960930248:2059] [node 5] ICS10 traffic start
2025-11-19T13:59:02.847129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:02.847191Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435714960930161:2059] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:02.847419Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435714841679751:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:02.847594Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435713049099946:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:02.847681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435713049099854:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.847693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:02.847699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435713049099854:2048] [node 1] ICP22 created new session: [4:7574435713049099957:2048]
2025-11-19T13:59:02.847703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435713049099957:2048] [node 1] ICS09 handshake done sender: [4:7574435713049099946:2119] self: [4:7574435713049099949:0] peer: [1:7574435714841679746:0] socket: 261 qp: -1
2025-11-19T13:59:02.847708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435713049099957:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:02.847715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:02.848742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435714841679736:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:02.848762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:02.848768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435714841679736:2110] [node 4] ICP22 created new session: [1:7574435714841679768:2110]
2025-11-19T13:59:02.848773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435714841679768:2110] [node 4] ICS09 handshake done sender: [1:7574435714841679751:2115] self: [1:7574435714841679746:0] peer: [4:7574435713049099949:0] socket: 260 qp: -1
2025-11-19T13:59:02.848777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435714841679768:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:02.848783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:02.859890Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435714960930236:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:02.860274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435714960930188:2077] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:02.860293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:02.860301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435714960930188:2077] [node 4] ICP22 created new session: [6:7574435714960930250:2077]
2025-11-19T13:59:02.860308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435714960930250:2077] [node 4] ICS09 handshake done sender: [6:7574435714960930236:2120] self: [6:7574435714960930244:0] peer: [4:7574435713049099930:0] socket: 209 qp: -1
2025-11-19T13:59:02.860315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435714960930250:2077] [node 4] ICS10 traffic start
2025-11-19T13:59:02.860322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:02.882153Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:02.882264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435714356685225:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-11-19T13:59:02.882275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435714356685135:2048] [node 1] ICP30 unregister session Session# [2:7574435714356685225:2048] VirtualId# [2:7574435714356685216:0]
2025-11-19T13:59:02.882283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435714356685225:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.882338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:02.882362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:02.883799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:02.883871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435711977855235:2048] [node 1] ICS01 socket: 221 reason# EndOfStream
2025-11-19T13:59:02.883877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435711977855140:2048] [node 1] ICP30 unregister session Session# [3:7574435711977855235:2048] VirtualId# [3:7574435711977855224:0]
2025-11-19T13:59:02.883892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435711977855235:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.883939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:02.883960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:02.893470Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:02.893569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435714960930228:2048] [node 1] ICS01 socket: 218 reason# EndOfStream
2025-11-19T13:59:02.893579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435714960930148:2048] [node 1] ICP30 unregister session Session# [6:7574435714960930228:2048] VirtualId# [6:7574435714960930226:0]
2025-11-19T13:59:02.893586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435714960930228:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.893636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:02.893661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:02.893699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:02.893756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435713049099957:2048] [node 1] ICS01 socket: 261 reason# EndOfStream
2025-11-19T13:59:02.893761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435713049099854:2048] [node 1] ICP30 unregister session Session# [4:7574435713049099957:2048] VirtualId# [4:7574435713049099949:0]
2025-11-19T13:59:02.893764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435713049099957:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.893788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:02.893805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:02.893915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:02.893972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435713428137522:2048] [node 1] ICS07 socket disconnect 248 reason# EndOfStream
2025-11-19T13:59:02.893976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435713428137522:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.893998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:02.894015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435713428137522:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:02.894759Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435713428137432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1611) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:02.894771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435713428137522:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:02.894775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435713428137432:2048] [node 1] ICP30 unregister session Session# [5:7574435713428137522:2048] VirtualId# [5:7574435713428137517:0]
2025-11-19T13:59:02.894789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:02.894794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435713428137432:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1611) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:02.894797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1611) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:02.898708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:02.898786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435711977855238:2049] [node 2] ICS01 socket: 242 reason# EndOfStream
2025-11-19T13:59:02.898793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435711977855141:2049] [node 2] ICP30 unregister session Session# [3:7574435711977855238:2049] VirtualId# [3:7574435711977855234:0]
2025-11-19T13:59:02.898800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435711977855238:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.898839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:02.898859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:02.903875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435713428137432:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:02.915994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:02.916096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435713049099954:2049] [node 2] ICS01 socket: 268 reason# EndOfStream
2025-11-19T13:59:02.916107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435713049099855:2049] [node 2] ICP30 unregister session Session# [4:7574435713049099954:2049] VirtualId# [4:7574435713049099948:0]
2025-11-19T13:59:02.916115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435713049099954:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.916149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:02.916166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:02.916952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:02.917027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435714960930237:2049] [node 2] ICS01 socket: 241 reason# EndOfStream
2025-11-19T13:59:02.917032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435714960930149:2049] [node 2] ICP30 unregister session Session# [6:7574435714960930237:2049] VirtualId# [6:7574435714960930233:0]
2025-11-19T13:59:02.917036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435714960930237:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.917062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:02.917080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:02.919849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:02.919926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435713428137530:2049] [node 2] ICS01 socket: 214 reason# EndOfStream
2025-11-19T13:59:02.919933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435713428137433:2049] [node 2] ICP30 unregister session Session# [5:7574435713428137530:2049] VirtualId# [5:7574435713428137528:0]
2025-11-19T13:59:02.919939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435713428137530:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.919994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:02.920019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:02.935003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:02.935104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435714960930241:2050] [node 3] ICS01 socket: 264 reason# EndOfStream
2025-11-19T13:59:02.935115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435714960930150:2050] [node 3] ICP30 unregister session Session# [6:7574435714960930241:2050] VirtualId# [6:7574435714960930239:0]
2025-11-19T13:59:02.935123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435714960930241:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.935165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:02.935188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:02.935381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:02.935459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435713428137510:2050] [node 3] ICS01 socket: 204 reason# EndOfStream
2025-11-19T13:59:02.935466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435713428137434:2050] [node 3] ICP30 unregister session Session# [5:7574435713428137510:2050] VirtualId# [5:7574435713428137447:0]
2025-11-19T13:59:02.935472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435713428137510:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.936679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:02.936722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:02.969054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:02.969133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435714960930250:2077] [node 4] ICS01 socket: 209 reason# EndOfStream
2025-11-19T13:59:02.969141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435714960930188:2077] [node 4] ICP30 unregister session Session# [6:7574435714960930250:2077] VirtualId# [6:7574435714960930244:0]
2025-11-19T13:59:02.969148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435714960930250:2077] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:02.969190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:02.969213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:03.272008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435715996190027:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:03.272022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435715996190028:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:03.352344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435717121859846:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:03.352363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435717121859847:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:03.412029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435719094578959:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:03.412051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435719094578960:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:03.463862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435718497418546:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:03.463877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435718497418547:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:03.463882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435718497418548:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:03.507834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435719746525512:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:03.507846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435719746525513:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:03.507851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435719746525514:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:03.571983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435715754511975:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:03.572000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435715754511976:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:03.572006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435715754511977:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:03.592233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435715754511996:2065] [node 5] ICP01 ready to work
2025-11-19T13:59:03.592291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435719094579006:2092] [node 4] ICP01 ready to work
2025-11-19T13:59:03.592124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435718497418586:2086] [node 5] ICP01 ready to work
2025-11-19T13:59:03.592302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435717121859893:2093] [node 6] ICP01 ready to work
2025-11-19T13:59:03.593136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435715996190079:2098] [node 6] ICP01 ready to work
2025-11-19T13:59:03.598120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435717121859902:2096] [node 4] ICP01 ready to work
2025-11-19T13:59:03.604714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435719094579018:2097] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.608557Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435715754511975:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16541206ns) {Configure:104(16540013ns) {ProcessConfigured:135(13570319ns) {StartInitialHandshake:157(13569637ns) {DropHandshakes:469(408ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(128ns) PrepareNewSessionHandshake:476(692ns) {SwitchToState:178(202ns)}}}}}
2025-11-19T13:59:03.608730Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435715996190027:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16481610ns) {Configure:104(16481084ns) {ProcessConfigured:135(13583175ns) {StartInitialHandshake:157(13582424ns) {DropHandshakes:469(283ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(40ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:476(812ns) {SwitchToState:178(237ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.609180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435715996190028:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(16811798ns) {Configure:104(16810015ns) {ProcessConfigured:135(13844676ns) {StartInitialHandshake:157(13844318ns) {DropHandshakes:469(260ns) {DropIncomingHandshake:441(21ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(631ns) {SwitchToState:178(292ns)}}}}}
2025-11-19T13:59:03.609202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435715996190095:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50041
2025-11-19T13:59:03.609216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435715996190096:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50041
2025-11-19T13:59:03.610749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435715996190094:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50041
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.620024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435717121859922:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17160
2025-11-19T13:59:03.620049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435717121859923:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17160
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.620889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435715754512041:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T13:59:03.620909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435715754512042:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T13:59:03.622377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435719746525576:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21027
2025-11-19T13:59:03.622559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435718497418546:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18744906ns) {Configure:104(18743422ns) {ProcessConfigured:135(18390422ns) {StartInitialHandshake:157(18389682ns) {DropHandshakes:469(949ns) {DropIncomingHandshake:441(227ns) DropOutgoingHandshake:456(170ns)} GenerateSessionVirtualId:988(282ns) PrepareNewSessionHandshake:476(1631ns) {SwitchToState:178(648ns)}}}}}
2025-11-19T13:59:03.623189Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435718497418548:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19247629ns) {Configure:104(19247126ns) {ProcessConfigured:135(18981465ns) {StartInitialHandshake:157(18981291ns) {DropHandshakes:469(769ns) {DropIncomingHandshake:441(144ns) DropOutgoingHandshake:456(202ns)} GenerateSessionVirtualId:988(175ns) PrepareNewSessionHandshake:476(655ns) {SwitchToState:178(328ns)}}}}}
2025-11-19T13:59:03.624043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435715996190082:2100] [node 5] ICP01 ready to work
2025-11-19T13:59:03.628057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435715754512042:2105] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.628108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435715754511977:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:03.628136Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435717121859922:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.635826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435719746525575:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21027
2025-11-19T13:59:03.639875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435717121859847:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(47140915ns) {Configure:104(47139956ns) {ProcessConfigured:135(46913150ns) {StartInitialHandshake:157(46912399ns) {DropHandshakes:469(529ns) {DropIncomingHandshake:441(94ns) DropOutgoingHandshake:456(31ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:476(1335ns) {SwitchToState:178(745ns)}}}}}
2025-11-19T13:59:03.640417Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435715754511976:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(48456634ns) {Configure:104(48454662ns) {ProcessConfigured:135(45810603ns) {StartInitialHandshake:157(45810126ns) {DropHandshakes:469(438ns) {DropIncomingHandshake:441(130ns) DropOutgoingHandshake:456(49ns)} GenerateSessionVirtualId:988(126ns) PrepareNewSessionHandshake:476(944ns) {SwitchToState:178(285ns)}}}}}
2025-11-19T13:59:03.640544Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435719094578960:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(30102387ns) {Configure:104(30101735ns) {ProcessConfigured:135(29813468ns) {StartInitialHandshake:157(29813241ns) {DropHandshakes:469(488ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(522ns) PrepareNewSessionHandshake:476(791ns) {SwitchToState:178(601ns)}}}}}
2025-11-19T13:59:03.640693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435715996190079:2098] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(47515147ns) {Configure:104(47514839ns) {ProcessConfigured:135(45448179ns) {StartInitialHandshake:157(45447957ns) {DropHandshakes:469(141ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(45ns) PrepareNewSessionHandshake:476(968ns) {SwitchToState:178(180ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.641591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435715754512052:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T13:59:03.651575Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435715996190094:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.654011Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435719746525575:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.654097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435719746525564:2073] [node 4] ICP01 ready to work
2025-11-19T13:59:03.655879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435715754512002:2063] [node 4] ICP01 ready to work
2025-11-19T13:59:03.655878Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435719094579020:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.655904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435719094579020:2102] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:03.655942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435719094579007:2090] [node 5] ICP01 ready to work
2025-11-19T13:59:03.653144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435715996190102:2103] [node 4] ICP01 ready to work
2025-11-19T13:59:03.654268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435717121859921:2106] [node 5] ICP01 ready to work
2025-11-19T13:59:03.663986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435715996190082:2100] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(39862635ns) {Configure:104(39862062ns) {ProcessConfigured:135(39573884ns) {StartInitialHandshake:157(39573468ns) {DropHandshakes:469(1013ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(429ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:476(1312ns) {SwitchToState:178(803ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.667976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435717121859926:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17160
2025-11-19T13:59:03.671373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435719746525550:2064] [node 6] ICP01 ready to work
2025-11-19T13:59:03.670952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435718497418605:2090] [node 6] ICP01 ready to work
2025-11-19T13:59:03.672123Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435717121859926:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.673783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435715996190102:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(20558666ns) {Configure:104(20557486ns) {ProcessConfigured:135(20226079ns) {StartInitialHandshake:157(20225270ns) {DropHandshakes:469(1034ns) {DropIncomingHandshake:441(121ns) DropOutgoingHandshake:456(150ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:476(1354ns) {SwitchToState:178(772ns)}}}}}
2025-11-19T13:59:03.676537Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435715754512052:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.676565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435715754511975:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:03.676381Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435715996190096:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.677125Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435718497418588:2087] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.677137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435718497418588:2087] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.676734Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435715996190104:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.676740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435715996190104:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:03.678136Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435718497418588:2087] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:03.676326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435719746525564:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:03.676378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435719746525564:2073] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(22209099ns) {Configure:104(22208857ns) {ProcessConfigured:135(21937738ns) {StartInitialHandshake:157(21891298ns) {DropHandshakes:469(393ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(329ns) PrepareNewSessionHandshake:476(1622ns) {SwitchToState:178(791ns)}} IncomingHandshake:261(35710ns) {DropIncomingHandshake:441(340ns) PrepareNewSessionHandshake:476(200ns) {SwitchToState:178(130ns)} GenerateSessionVirtualId:988(678ns) IssueIncomingHandshakeReply:193(28703ns) {SwitchToState:178(37ns)}}}}}
2025-11-19T13:59:03.676995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435719746525512:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(72776624ns) {Configure:104(72776000ns) {ProcessConfigured:135(72539496ns) {StartInitialHandshake:157(72539223ns) {DropHandshakes:469(212ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(15ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:476(592ns) {SwitchToState:178(244ns)}}}}}
2025-11-19T13:59:03.678056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435719746525592:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21027
2025-11-19T13:59:03.678358Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435719746525575:2103] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:03.678463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435719746525564:2073] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.678482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:03.678488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435719746525564:2073] [node 4] ICP22 created new session: [5:7574435719746525593:2073]
2025-11-19T13:59:03.678494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435719746525593:2073] [node 4] ICS09 handshake done sender: [5:7574435719746525575:2103] self: [5:7574435719746525589:0] peer: [4:7574435718497418587:0] socket: 211 qp: -1
2025-11-19T13:59:03.678501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435719746525593:2073] [node 4] ICS10 traffic start
2025-11-19T13:59:03.678510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:03.680468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435718497418586:2086] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.680485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:03.680491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435718497418586:2086] [node 5] ICP22 created new session: [4:7574435718497418624:2086]
2025-11-19T13:59:03.680496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435718497418624:2086] [node 5] ICS09 handshake done sender: [4:7574435718497418588:2087] self: [4:7574435718497418587:0] peer: [5:7574435719746525589:0] socket: 210 qp: -1
2025-11-19T13:59:03.680501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435718497418624:2086] [node 5] ICS10 traffic start
2025-11-19T13:59:03.680508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.680797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435719094579007:2090] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(24803385ns) {Configure:104(24802969ns) {ProcessConfigured:135(24531013ns) {StartInitialHandshake:157(24530687ns) {DropHandshakes:469(401ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(187ns) PrepareNewSessionHandshake:476(642ns) {SwitchToState:178(229ns)}}}}}
2025-11-19T13:59:03.680857Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435719094579020:2102] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.681468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435719094579018:2097] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.681483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:03.681489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435719094579018:2097] [node 6] ICP22 created new session: [3:7574435719094579040:2097]
2025-11-19T13:59:03.681494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435719094579040:2097] [node 6] ICS09 handshake done sender: [3:7574435719094579020:2102] self: [3:7574435719094579019:0] peer: [6:7574435715754512047:0] socket: 220 qp: -1
2025-11-19T13:59:03.681501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435719094579040:2097] [node 6] ICS10 traffic start
2025-11-19T13:59:03.681515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:03.681013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435718497418627:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54835
2025-11-19T13:59:03.681061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435718497418628:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54835
2025-11-19T13:59:03.681088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435718497418629:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54835
2025-11-19T13:59:03.682353Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435718497418628:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.682383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435718497418548:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:03.680651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435715754512002:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24690114ns) {Configure:104(24689318ns) {ProcessConfigured:135(24402626ns) {StartInitialHandshake:157(24402015ns) {DropHandshakes:469(565ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:476(583ns) {SwitchToState:178(51ns)}}}}}
2025-11-19T13:59:03.681346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435715754512058:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T13:59:03.683293Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435715754512042:2105] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:03.683376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435715754511977:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.683387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:03.683393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435715754511977:2050] [node 3] ICP22 created new session: [6:7574435715754512059:2050]
2025-11-19T13:59:03.683398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435715754512059:2050] [node 3] ICS09 handshake done sender: [6:7574435715754512042:2105] self: [6:7574435715754512047:0] peer: [3:7574435719094579019:0] socket: 233 qp: -1
2025-11-19T13:59:03.683404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435715754512059:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:03.683419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:03.684013Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435715996190095:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.684000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435719094579037:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53111
2025-11-19T13:59:03.684080Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435718497418627:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.684117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435718497418547:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:03.688197Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435719094579037:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.687928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435717121859921:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(33558073ns) {Configure:104(33557570ns) {ProcessConfigured:135(33264883ns) {StartInitialHandshake:157(33252719ns) {DropHandshakes:469(553ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(112ns)} GenerateSessionVirtualId:988(308ns) PrepareNewSessionHandshake:476(635ns) {SwitchToState:178(331ns)}} IncomingHandshake:261(4194ns) {DropIncomingHandshake:441(136ns) PrepareNewSessionHandshake:476(141ns) {SwitchToState:178(109ns)} GenerateSessionVirtualId:988(164ns) IssueIncomingHandshakeReply:193(156ns)}}}}
2025-11-19T13:59:03.691193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435718497418605:2090] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20155509ns) {Configure:104(20154555ns) {ProcessConfigured:135(19836778ns) {StartInitialHandshake:157(19835830ns) {DropHandshakes:469(2114ns) {DropIncomingHandshake:441(281ns) DropOutgoingHandshake:456(147ns)} GenerateSessionVirtualId:988(291ns) PrepareNewSessionHandshake:476(621ns) {SwitchToState:178(87ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.691354Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435719746525550:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19913684ns) {Configure:104(19913152ns) {ProcessConfigured:135(19647158ns) {StartInitialHandshake:157(19646924ns) {DropHandshakes:469(518ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(244ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(579ns) {SwitchToState:178(38ns)}}}}}
2025-11-19T13:59:03.692082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435718497418586:2086] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:03.691915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435715996190112:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50041
2025-11-19T13:59:03.691976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435715996190113:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50041
2025-11-19T13:59:03.692740Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435715996190112:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.693264Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435715996190104:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.693408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435715754512061:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T13:59:03.693566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435715996190079:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.693586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:03.693591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435715996190079:2098] [node 6] ICP22 created new session: [1:7574435715996190116:2098]
2025-11-19T13:59:03.693598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435715996190116:2098] [node 6] ICS09 handshake done sender: [1:7574435715996190104:2115] self: [1:7574435715996190084:0] peer: [6:7574435715754512056:0] socket: 239 qp: -1
2025-11-19T13:59:03.693604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435715996190116:2098] [node 6] ICS10 traffic start
2025-11-19T13:59:03.693633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.694346Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435715754512052:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:03.695784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435715754511975:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.695803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:03.695809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435715754511975:2048] [node 1] ICP22 created new session: [6:7574435715754512062:2048]
2025-11-19T13:59:03.695814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435715754512062:2048] [node 1] ICS09 handshake done sender: [6:7574435715754512052:2114] self: [6:7574435715754512056:0] peer: [1:7574435715996190084:0] socket: 240 qp: -1
2025-11-19T13:59:03.695819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435715754512062:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:03.695826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:03.696335Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435715754512041:2104] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.696364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435715754511976:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:03.696449Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435717121859923:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.694116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435719746525597:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21027
2025-11-19T13:59:03.695653Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435719746525576:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.696225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435719746525597:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.696256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435719746525512:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.702100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435718497418633:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54835
2025-11-19T13:59:03.703590Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435718497418633:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.703627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435718497418546:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:03.700728Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435715996190093:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.700745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435715996190093:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:03.701198Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435715996190113:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.701499Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435715996190093:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:03.701578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435715996190027:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.701593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:03.701599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435715996190027:2048] [node 2] ICP22 created new session: [1:7574435715996190121:2048]
2025-11-19T13:59:03.701605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435715996190121:2048] [node 2] ICS09 handshake done sender: [1:7574435715996190093:2108] self: [1:7574435715996190083:0] peer: [2:7574435717121859930:0] socket: 229 qp: -1
2025-11-19T13:59:03.701610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435715996190121:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:03.701625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:03.703887Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435715996190108:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.703894Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435715996190108:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:03.704076Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435717121859904:2099] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.704089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435717121859904:2099] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:03.704112Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435717121859901:2098] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.704116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435717121859901:2098] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:03.704019Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435719094579023:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.704038Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435719094579023:2103] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:03.707561Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435715996190109:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.707571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435715996190109:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.707763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435718497418635:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54835
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.708979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435719094579044:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53111
2025-11-19T13:59:03.709178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435719094579045:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53111
2025-11-19T13:59:03.709209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435719094579046:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53111
2025-11-19T13:59:03.709237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435719094579018:2097] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.710758Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435719094579046:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.710868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435719094579047:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53111
2025-11-19T13:59:03.710710Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435715996190108:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:03.711033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435715996190082:2100] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.711052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:03.711057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435715996190082:2100] [node 5] ICP22 created new session: [1:7574435715996190125:2100]
2025-11-19T13:59:03.711063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435715996190125:2100] [node 5] ICS09 handshake done sender: [1:7574435715996190108:2116] self: [1:7574435715996190103:0] peer: [5:7574435719746525599:0] socket: 254 qp: -1
2025-11-19T13:59:03.711068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435715996190125:2100] [node 5] ICS10 traffic start
2025-11-19T13:59:03.711085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:03.707566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435717121859846:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:03.707598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435717121859846:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435717121859923:2114] held: no
2025-11-19T13:59:03.712938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435719094579045:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.712964Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435719094579047:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.713005Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435719094579023:2103] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:03.710685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435719746525600:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21027
2025-11-19T13:59:03.712737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435719746525512:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435719746525597:2118] held: no
2025-11-19T13:59:03.712788Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435719746525597:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:03.713130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435719746525512:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.713145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:03.713150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435719746525512:2048] [node 1] ICP22 created new session: [5:7574435719746525601:2048]
2025-11-19T13:59:03.713155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435719746525601:2048] [node 1] ICS09 handshake done sender: [5:7574435719746525597:2118] self: [5:7574435719746525599:0] peer: [1:7574435715996190103:0] socket: 255 qp: -1
2025-11-19T13:59:03.713161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435719746525601:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:03.713180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:03.713467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435719094579006:2092] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.713481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:03.713487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435719094579006:2092] [node 4] ICP22 created new session: [3:7574435719094579050:2092]
2025-11-19T13:59:03.713491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435719094579050:2092] [node 4] ICS09 handshake done sender: [3:7574435719094579023:2103] self: [3:7574435719094579009:0] peer: [4:7574435718497418630:0] socket: 228 qp: -1
2025-11-19T13:59:03.713495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435719094579050:2092] [node 4] ICS10 traffic start
2025-11-19T13:59:03.713500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:03.713987Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435717121859924:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.713995Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435717121859924:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:03.713786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435719094578959:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:03.713799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435719094578960:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.714830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435719746525603:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21027
2025-11-19T13:59:03.716253Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435717121859924:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:03.716464Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435717121859904:2099] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:03.716550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435717121859902:2096] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.716565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:03.716571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435717121859902:2096] [node 4] ICP22 created new session: [2:7574435717121859934:2096]
2025-11-19T13:59:03.716576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435717121859934:2096] [node 4] ICS09 handshake done sender: [2:7574435717121859904:2099] self: [2:7574435717121859903:0] peer: [4:7574435718497418631:0] socket: 213 qp: -1
2025-11-19T13:59:03.716581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435717121859934:2096] [node 4] ICS10 traffic start
2025-11-19T13:59:03.716595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:03.719872Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435718497418635:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.720211Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435715996190097:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.720225Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435715996190097:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:03.720349Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435718497418547:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2115) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435718497418627:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.721522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435715754512067:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T13:59:03.723926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435719094579055:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53111
2025-11-19T13:59:03.723929Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435715996190097:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:03.724002Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435715996190109:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.724288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435717121859847:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.724303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:03.724309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435717121859847:2049] [node 3] ICP22 created new session: [2:7574435717121859936:2049]
2025-11-19T13:59:03.724316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435717121859936:2049] [node 3] ICS09 handshake done sender: [2:7574435717121859924:2115] self: [2:7574435717121859895:0] peer: [3:7574435719094579053:0] socket: 221 qp: -1
2025-11-19T13:59:03.724321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435717121859936:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:03.724328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:03.724188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435719094579056:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53111
2025-11-19T13:59:03.724545Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435719094579045:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:03.724911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435718497418548:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30671) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435718497418628:2115] held: no
2025-11-19T13:59:03.724649Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435715754512067:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.724693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435715754511996:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.725092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435715996190102:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.725110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:03.725114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435715996190102:2103] [node 4] ICP22 created new session: [1:7574435715996190129:2103]
2025-11-19T13:59:03.725119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435715996190129:2103] [node 4] ICS09 handshake done sender: [1:7574435715996190109:2117] self: [1:7574435715996190107:0] peer: [4:7574435718497418634:0] socket: 226 qp: -1
2025-11-19T13:59:03.725123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435715996190129:2103] [node 4] ICS10 traffic start
2025-11-19T13:59:03.725128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:03.725185Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435719746525603:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.725207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435719746525514:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:03.725245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435715996190028:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.725255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:03.725259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435715996190028:2049] [node 3] ICP22 created new session: [1:7574435715996190131:2049]
2025-11-19T13:59:03.725263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435715996190131:2049] [node 3] ICS09 handshake done sender: [1:7574435715996190097:2112] self: [1:7574435715996190085:0] peer: [3:7574435719094579052:0] socket: 230 qp: -1
2025-11-19T13:59:03.725266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435715996190131:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:03.725271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:03.725848Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435719094579039:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.725859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435719094579039:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:03.726749Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435719094579039:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:03.725281Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435719746525596:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.725285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435719746525596:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:03.727153Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435719746525596:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:03.727281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435719746525550:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.727295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:03.727299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435719746525550:2064] [node 6] ICP22 created new session: [5:7574435719746525608:2064]
2025-11-19T13:59:03.727304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435719746525608:2064] [node 6] ICS09 handshake done sender: [5:7574435719746525596:2117] self: [5:7574435719746525587:0] peer: [6:7574435715754512068:0] socket: 245 qp: -1
2025-11-19T13:59:03.727308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435719746525608:2064] [node 6] ICS10 traffic start
2025-11-19T13:59:03.727314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.726854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435719094579007:2090] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.726870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:03.726876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435719094579007:2090] [node 5] ICP22 created new session: [3:7574435719094579058:2090]
2025-11-19T13:59:03.726882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435719094579058:2090] [node 5] ICS09 handshake done sender: [3:7574435719094579039:2115] self: [3:7574435719094579036:0] peer: [5:7574435719746525606:0] socket: 253 qp: -1
2025-11-19T13:59:03.726887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435719094579058:2090] [node 5] ICS10 traffic start
2025-11-19T13:59:03.726893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:03.727052Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435719094579047:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:03.727899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435719094578959:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.727915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:03.727921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435719094578959:2048] [node 1] ICP22 created new session: [3:7574435719094579060:2048]
2025-11-19T13:59:03.727927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435719094579060:2048] [node 1] ICS09 handshake done sender: [3:7574435719094579045:2117] self: [3:7574435719094579052:0] peer: [1:7574435715996190085:0] socket: 259 qp: -1
2025-11-19T13:59:03.727933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435719094579060:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:03.727939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:03.728112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435719094578960:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.728119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:03.728135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435719094578960:2049] [node 2] ICP22 created new session: [3:7574435719094579062:2049]
2025-11-19T13:59:03.728139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435719094579062:2049] [node 2] ICS09 handshake done sender: [3:7574435719094579047:2119] self: [3:7574435719094579053:0] peer: [2:7574435717121859895:0] socket: 262 qp: -1
2025-11-19T13:59:03.728144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435719094579062:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:03.728148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:03.727638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435717121859940:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17160
2025-11-19T13:59:03.727686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435717121859941:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17160
2025-11-19T13:59:03.727716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435717121859942:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17160
2025-11-19T13:59:03.731528Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435717121859901:2098] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:03.732025Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435717121859940:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.732136Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435717121859923:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:03.732217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435717121859847:2049] [node 3] ICP18 handshake (actor [2:7574435717121859940:2118]) is too old
2025-11-19T13:59:03.732407Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435717121859941:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.732600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435717121859893:2093] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.732616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:03.732622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435717121859893:2093] [node 6] ICP22 created new session: [2:7574435717121859944:2093]
2025-11-19T13:59:03.732628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435717121859944:2093] [node 6] ICS09 handshake done sender: [2:7574435717121859901:2098] self: [2:7574435717121859894:0] peer: [6:7574435715754512064:0] socket: 212 qp: -1
2025-11-19T13:59:03.732633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435717121859944:2093] [node 6] ICS10 traffic start
2025-11-19T13:59:03.732653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:03.732979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435717121859846:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.732999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:03.733003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435717121859846:2048] [node 1] ICP22 created new session: [2:7574435717121859946:2048]
2025-11-19T13:59:03.733007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435717121859946:2048] [node 1] ICS09 handshake done sender: [2:7574435717121859923:2114] self: [2:7574435717121859930:0] peer: [1:7574435715996190083:0] socket: 232 qp: -1
2025-11-19T13:59:03.733012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435717121859946:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:03.733019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:03.733831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435715754511976:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2115) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435715754512041:2104] held: no
2025-11-19T13:59:03.739096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435719746525605:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21027
2025-11-19T13:59:03.745135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435718497418546:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31171) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435718497418633:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.746415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435715754511996:2065] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9042) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435715754512067:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.746586Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435719746525514:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30671) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435719746525603:2120] held: no
2025-11-19T13:59:03.746634Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435719746525605:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.746694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435719746525610:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21027
2025-11-19T13:59:03.747007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435715754512069:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T13:59:03.747110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435719746525513:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:03.747262Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435719746525550:2064] [node 6] ICP35 event processing took too much time PendingConnection:278(19952035ns) {HandleHandshakeStatus:329(19951624ns) {DropIncomingHandshake:441(2596ns) DropOutgoingHandshake:456(78ns) SwitchToState:178(67ns) ProcessPendingSessionEvents:495(19892008ns) {ForwardSessionEventToSession:612(19879943ns) ForwardSessionEventToSession:612(2121ns) ForwardSessionEventToSession:612(621ns) ForwardSessionEventToSession:612(705ns) ForwardSessionEventToSession:612(784ns) ForwardSessionEventToSession:612(819ns) ForwardSessionEventToSession:612(556ns) ForwardSessionEventToSession:612(567ns) ForwardSessionEventToSession:612(887ns) ForwardSessionEventToSession:612(597ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.747620Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435715754512067:2120] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:03.747689Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435719746525603:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:03.747771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435719746525514:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.747784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:03.747789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435719746525514:2050] [node 3] ICP22 created new session: [5:7574435719746525612:2050]
2025-11-19T13:59:03.747794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435719746525612:2050] [node 3] ICS09 handshake done sender: [5:7574435719746525603:2120] self: [5:7574435719746525606:0] peer: [3:7574435719094579036:0] socket: 263 qp: -1
2025-11-19T13:59:03.747800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435719746525612:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:03.747807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:03.748649Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435717121859928:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.748666Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435717121859928:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:03.748713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435715754512071:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T13:59:03.748732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435715754511996:2065] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.748743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:03.748749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435715754511996:2065] [node 5] ICP22 created new session: [6:7574435715754512072:2065]
2025-11-19T13:59:03.748755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435715754512072:2065] [node 5] ICS09 handshake done sender: [6:7574435715754512067:2120] self: [6:7574435715754512068:0] peer: [5:7574435719746525587:0] socket: 264 qp: -1
2025-11-19T13:59:03.748760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435715754512072:2065] [node 5] ICS10 traffic start
2025-11-19T13:59:03.748781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:03.748795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435715754512070:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.750024Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435715996190102:2103] [node 4] ICP35 event processing took too much time PendingConnection:278(24910218ns) {HandleHandshakeStatus:329(24909153ns) {DropIncomingHandshake:441(6105ns) DropOutgoingHandshake:456(77ns) SwitchToState:178(59ns) ProcessPendingSessionEvents:495(10189771ns) {ForwardSessionEventToSession:612(10168686ns) ForwardSessionEventToSession:612(1675ns) ForwardSessionEventToSession:612(1030ns) ForwardSessionEventToSession:612(928ns) ForwardSessionEventToSession:612(586ns) ForwardSessionEventToSession:612(1434ns) ForwardSessionEventToSession:612(883ns) ForwardSessionEventToSession:612(511ns) ForwardSessionEventToSession:612(571ns) ForwardSessionEventToSession:612(928ns)}}}
2025-11-19T13:59:03.749934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435718497418638:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54835
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.750258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435718497418639:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54835
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.750417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435718497418640:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54835
2025-11-19T13:59:03.750615Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435718497418627:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:03.750504Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435715754512041:2104] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:03.751950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435715754511976:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.751969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:03.751975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435715754511976:2049] [node 2] ICP22 created new session: [6:7574435715754512074:2049]
2025-11-19T13:59:03.751982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435715754512074:2049] [node 2] ICS09 handshake done sender: [6:7574435715754512041:2104] self: [6:7574435715754512064:0] peer: [2:7574435717121859894:0] socket: 218 qp: -1
2025-11-19T13:59:03.751989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435715754512074:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:03.751997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:03.753002Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435718497418628:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:03.753224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435718497418547:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.753255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:03.753262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435718497418547:2049] [node 2] ICP22 created new session: [4:7574435718497418641:2049]
2025-11-19T13:59:03.753268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435718497418641:2049] [node 2] ICS09 handshake done sender: [4:7574435718497418627:2114] self: [4:7574435718497418631:0] peer: [2:7574435717121859903:0] socket: 231 qp: -1
2025-11-19T13:59:03.753274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435718497418641:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:03.753290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:03.753456Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435718497418633:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:03.754061Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435717121859928:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.754286Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435715754512071:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.754339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435715754512002:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:03.754523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435717121859921:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.754541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:03.754547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435717121859921:2106] [node 5] ICP22 created new session: [2:7574435717121859949:2106]
2025-11-19T13:59:03.754553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435717121859949:2106] [node 5] ICS09 handshake done sender: [2:7574435717121859928:2117] self: [2:7574435717121859925:0] peer: [5:7574435719746525611:0] socket: 270 qp: -1
2025-11-19T13:59:03.754558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435717121859949:2106] [node 5] ICS10 traffic start
2025-11-19T13:59:03.754564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:03.754211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435719746525614:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21027
2025-11-19T13:59:03.754929Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435719746525605:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:03.754980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435719746525513:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.754989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:03.754994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435719746525513:2049] [node 2] ICP22 created new session: [5:7574435719746525615:2049]
2025-11-19T13:59:03.754999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435719746525615:2049] [node 2] ICS09 handshake done sender: [5:7574435719746525605:2121] self: [5:7574435719746525611:0] peer: [2:7574435717121859925:0] socket: 271 qp: -1
2025-11-19T13:59:03.755005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435719746525615:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:03.755010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:03.756453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435718497418546:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.756470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:03.756478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435718497418546:2048] [node 1] ICP22 created new session: [4:7574435718497418643:2048]
2025-11-19T13:59:03.756484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435718497418643:2048] [node 1] ICS09 handshake done sender: [4:7574435718497418633:2118] self: [4:7574435718497418634:0] peer: [1:7574435715996190107:0] socket: 256 qp: -1
2025-11-19T13:59:03.756490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435718497418643:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:03.756506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:03.756619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435718497418548:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.756626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:03.756630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435718497418548:2050] [node 3] ICP22 created new session: [4:7574435718497418645:2050]
2025-11-19T13:59:03.756634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435718497418645:2050] [node 3] ICS09 handshake done sender: [4:7574435718497418628:2115] self: [4:7574435718497418630:0] peer: [3:7574435719094579009:0] socket: 248 qp: -1
2025-11-19T13:59:03.756639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435718497418645:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:03.756643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:03.757395Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435718497418632:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:03.757408Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435718497418632:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:03.758357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435715754512077:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T13:59:03.759974Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435718497418632:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:03.760130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435718497418605:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:03.760154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:03.760160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435718497418605:2090] [node 6] ICP22 created new session: [4:7574435718497418648:2090]
2025-11-19T13:59:03.760165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435718497418648:2090] [node 6] ICS09 handshake done sender: [4:7574435718497418632:2117] self: [4:7574435718497418622:0] peer: [6:7574435715754512076:0] socket: 217 qp: -1
2025-11-19T13:59:03.760170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435718497418648:2090] [node 6] ICS10 traffic start
2025-11-19T13:59:03.760177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:03.763226Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435715754512002:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13270) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435715754512071:2123] held: no
2025-11-19T13:59:03.763968Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435715754512071:2123] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:03.764146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435715754512002:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:03.764157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:03.764164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435715754512002:2063] [node 4] ICP22 created new session: [6:7574435715754512078:2063]
2025-11-19T13:59:03.764170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435715754512078:2063] [node 4] ICS09 handshake done sender: [6:7574435715754512071:2123] self: [6:7574435715754512076:0] peer: [4:7574435718497418622:0] socket: 225 qp: -1
2025-11-19T13:59:03.764177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435715754512078:2063] [node 4] ICS10 traffic start
2025-11-19T13:59:03.764185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:03.772360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:03.772462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435717121859946:2048] [node 1] ICS01 socket: 232 reason# EndOfStream
2025-11-19T13:59:03.772473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435717121859846:2048] [node 1] ICP30 unregister session Session# [2:7574435717121859946:2048] VirtualId# [2:7574435717121859930:0]
2025-11-19T13:59:03.772479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435717121859946:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:03.772509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:03.772529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:03.772863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:03.772937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435719094579060:2048] [node 1] ICS01 socket: 259 reason# EndOfStream
2025-11-19T13:59:03.772942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435719094578959:2048] [node 1] ICP30 unregister session Session# [3:7574435719094579060:2048] VirtualId# [3:7574435719094579052:0]
2025-11-19T13:59:03.772946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435719094579060:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:03.772967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:03.772982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:03.783900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:03.784023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435715754512062:2048] [node 1] ICS01 socket: 240 reason# EndOfStream
2025-11-19T13:59:03.784035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435715754511975:2048] [node 1] ICP30 unregister session Session# [6:7574435715754512062:2048] VirtualId# [6:7574435715754512056:0]
2025-11-19T13:59:03.784044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435715754512062:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:03.784108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:03.784132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:03.784172Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:03.784252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435719746525601:2048] [node 1] ICS01 socket: 255 reason# EndOfStream
2025-11-19T13:59:03.784260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435719746525512:2048] [node 1] ICP30 unregister session Session# [5:7574435719746525601:2048] VirtualId# [5:7574435719746525599:0]
2025-11-19T13:59:03.784264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435719746525601:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:03.784301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:03.784322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:03.784351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:03.784406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435718497418643:2048] [node 1] ICS01 socket: 256 reason# EndOfStream
2025-11-19T13:59:03.784410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435718497418546:2048] [node 1] ICP30 unregister session Session# [4:7574435718497418643:2048] VirtualId# [4:7574435718497418634:0]
2025-11-19T13:59:03.784415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435718497418643:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:03.784447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:03.784467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:03.791397Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:03.791479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435719094579062:2049] [node 2] ICS01 socket: 262 reason# EndOfStream
2025-11-19T13:59:03.791487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435719094578960:2049] [node 2] ICP30 unregister session Session# [3:7574435719094579062:2049] VirtualId# [3:7574435719094579053:0]
2025-11-19T13:59:03.791492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435719094579062:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:03.791533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:03.791554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:03.807179Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:03.807276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435718497418641:2049] [node 2] ICS01 socket: 231 reason# EndOfStream
2025-11-19T13:59:03.807285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435718497418547:2049] [node 2] ICP30 unregister session Session# [4:7574435718497418641:2049] VirtualId# [4:7574435718497418631:0]
2025-11-19T13:59:03.807292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435718497418641:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:03.807331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:03.807354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:03.807385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:03.807440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435719746525615:2049] [node 2] ICS01 socket: 271 reason# EndOfStream
2025-11-19T13:59:03.807445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435719746525513:2049] [node 2] ICP30 unregister session Session# [5:7574435719746525615:2049] VirtualId# [5:7574435719746525611:0]
2025-11-19T13:59:03.807448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435719746525615:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:03.807491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:03.807509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:03.807716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:03.807836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435715754512074:2049] [node 2] ICS01 socket: 218 reason# EndOfStream
2025-11-19T13:59:03.807839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435715754511976:2049] [node 2] ICP30 unregister session Session# [6:7574435715754512074:2049] VirtualId# [6:7574435715754512064:0]
2025-11-19T13:59:03.807843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435715754512074:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:03.807870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:03.807888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:03.824025Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:03.824136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435719746525612:2050] [node 3] ICS01 socket: 263 reason# EndOfStream
2025-11-19T13:59:03.824146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435719746525514:2050] [node 3] ICP30 unregister session Session# [5:7574435719746525612:2050] VirtualId# [5:7574435719746525606:0]
2025-11-19T13:59:03.824154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435719746525612:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:03.824189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:03.824209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:03.824234Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T13:59:03.824292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435718497418645:2050] [node 3] ICS01 socket: 248 reason# EndOfStream
2025-11-19T13:59:03.824297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435718497418548:2050] [node 3] ICP30 unregister session Session# [4:7574435718497418645:2050] VirtualId# [4:7574435718497418630:0]
2025-11-19T13:59:03.824300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435718497418645:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:03.824322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:03.824339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:03.824360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:03.824415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435715754512059:2050] [node 3] ICS01 socket: 233 reason# EndOfStream
2025-11-19T13:59:03.824419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435715754511977:2050] [node 3] ICP30 unregister session Session# [6:7574435715754512059:2050] VirtualId# [6:7574435715754512047:0]
2025-11-19T13:59:03.824423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435715754512059:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:03.824445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:03.824462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:03.862005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:03.862104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435715754512078:2063] [node 4] ICS01 socket: 225 reason# EndOfStream
2025-11-19T13:59:03.862115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435715754512002:2063] [node 4] ICP30 unregister session Session# [6:7574435715754512078:2063] VirtualId# [6:7574435715754512076:0]
2025-11-19T13:59:03.862123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435715754512078:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:03.862171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:03.862193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:04.091895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435721590677638:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:04.091920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435721590677639:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:04.133447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435723346937488:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:04.133464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435723346937489:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:04.172012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435720565592224:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:04.172026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435720565592225:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:04.235958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435720041807930:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:04.235973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435720041807931:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:04.235977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435720041807932:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:04.325541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435720991635896:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:04.325560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435720991635897:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:04.325565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435720991635898:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:04.424613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435723327051840:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:04.424636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435723327051841:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:04.424642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435723327051842:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:04.448139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435720041807980:2096] [node 6] ICP01 ready to work
2025-11-19T13:59:04.450797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435721590677703:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:04.450885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435721590677704:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:04.452508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435721590677714:2113] [node 5] ICP01 ready to work
2025-11-19T13:59:04.452032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435720565592283:2102] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.460226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435720565592286:2105] [node 6] ICP01 ready to work
2025-11-19T13:59:04.461094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435720565592285:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11303
2025-11-19T13:59:04.461450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435723346937553:2099] [node 4] ICP01 ready to work
2025-11-19T13:59:04.461527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435723346937554:2101] [node 6] ICP01 ready to work
2025-11-19T13:59:04.461590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435723346937555:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:04.462742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435723346937551:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56347
2025-11-19T13:59:04.462799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435723346937552:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56347
2025-11-19T13:59:04.466947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435720991635898:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19039018ns) {Configure:104(19037784ns) {ProcessConfigured:135(18656980ns) {StartInitialHandshake:157(18656579ns) {DropHandshakes:469(637ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:476(2278ns) {SwitchToState:178(567ns)}}}}}
2025-11-19T13:59:04.467095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435720991635946:2061] [node 6] ICP01 ready to work
2025-11-19T13:59:04.467175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435720991635947:2066] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.471387Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435723327051841:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23480200ns) {Configure:104(23478280ns) {ProcessConfigured:135(23068324ns) {StartInitialHandshake:157(23067858ns) {DropHandshakes:469(470ns) {DropIncomingHandshake:441(39ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:476(681ns) {SwitchToState:178(512ns)}}}}}
2025-11-19T13:59:04.471517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435723327051899:2062] [node 4] ICP01 ready to work
2025-11-19T13:59:04.471591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435723327051900:2069] [node 5] ICP01 ready to work
2025-11-19T13:59:04.472602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435723327051901:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-19T13:59:04.472661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435723327051902:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-19T13:59:04.472671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435723327051903:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-19T13:59:04.474078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435720565592298:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11303
2025-11-19T13:59:04.474101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435720565592299:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11303
2025-11-19T13:59:04.477556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435720565592301:2115] [node 5] ICP01 ready to work
2025-11-19T13:59:04.483623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435721590677714:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(31055320ns) {Configure:104(31054901ns) {ProcessConfigured:135(30839306ns) {StartInitialHandshake:157(30839097ns) {DropHandshakes:469(372ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(50ns) PrepareNewSessionHandshake:476(747ns) {SwitchToState:178(22ns)}}}}}
2025-11-19T13:59:04.488975Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435723327051902:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.489997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435720041807997:2101] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.521382Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435723346937551:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.521442Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435723346937552:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.521508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435723346937569:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56347
2025-11-19T13:59:04.521530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435723346937488:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.531874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435720565592306:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11303
2025-11-19T13:59:04.532075Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721590677699:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.532092Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435721590677699:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:04.532323Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435720565592298:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.532362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435720565592225:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.539815Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435723346937561:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.539839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435723346937561:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.544594Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435720565592292:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.544607Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435720565592292:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:04.545299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435720565592308:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11303
2025-11-19T13:59:04.545666Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435721590677699:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:04.545778Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435723346937561:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:04.545877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435721590677638:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.545897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:04.545904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435721590677638:2048] [node 2] ICP22 created new session: [1:7574435721590677718:2048]
2025-11-19T13:59:04.545912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435721590677718:2048] [node 2] ICS09 handshake done sender: [1:7574435721590677699:2107] self: [1:7574435721590677698:0] peer: [2:7574435723346937570:0] socket: 216 qp: -1
2025-11-19T13:59:04.545930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435721590677718:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:04.545940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:04.543771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435723327051842:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.543814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435723327051921:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-19T13:59:04.546493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435723346937489:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.546513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:04.546518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435723346937489:2049] [node 3] ICP22 created new session: [2:7574435723346937574:2049]
2025-11-19T13:59:04.546523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435723346937574:2049] [node 3] ICS09 handshake done sender: [2:7574435723346937561:2114] self: [2:7574435723346937560:0] peer: [3:7574435720565592307:0] socket: 228 qp: -1
2025-11-19T13:59:04.546528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435723346937574:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:04.546535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.546861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435720041807998:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33017
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.546861Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435720565592285:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.546915Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435720565592292:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.546925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435720041807999:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33017
2025-11-19T13:59:04.546959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435720041808000:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33017
2025-11-19T13:59:04.546987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435720041808001:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33017
2025-11-19T13:59:04.547017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435720041808002:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33017
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.548408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435723346937573:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56347
2025-11-19T13:59:04.548622Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435723346937551:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.549760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435720565592286:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.549774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:04.549778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435720565592286:2105] [node 6] ICP22 created new session: [3:7574435720565592310:2105]
2025-11-19T13:59:04.549783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435720565592310:2105] [node 6] ICS09 handshake done sender: [3:7574435720565592292:2110] self: [3:7574435720565592291:0] peer: [6:7574435723327051922:0] socket: 224 qp: -1
2025-11-19T13:59:04.549788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435720565592310:2105] [node 6] ICS10 traffic start
2025-11-19T13:59:04.549795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:04.548692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435723346937576:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56347
2025-11-19T13:59:04.548707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435723346937488:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.548716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:04.548720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435723346937488:2048] [node 1] ICP22 created new session: [2:7574435723346937578:2048]
2025-11-19T13:59:04.548724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435723346937578:2048] [node 1] ICS09 handshake done sender: [2:7574435723346937551:2110] self: [2:7574435723346937570:0] peer: [1:7574435721590677698:0] socket: 219 qp: -1
2025-11-19T13:59:04.548729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435723346937578:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:04.548733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.559315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435720991635974:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-19T13:59:04.559343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435720991635975:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-19T13:59:04.559700Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435720565592298:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.560021Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435723327051901:2105] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.560100Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435723327051903:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.560214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435723327051923:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-19T13:59:04.561484Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435720565592225:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7132) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435720565592298:2113] held: no
2025-11-19T13:59:04.561510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435720565592225:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.561523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:04.561529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435720565592225:2049] [node 2] ICP22 created new session: [3:7574435720565592312:2049]
2025-11-19T13:59:04.561535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435720565592312:2049] [node 2] ICS09 handshake done sender: [3:7574435720565592298:2113] self: [3:7574435720565592307:0] peer: [2:7574435723346937560:0] socket: 229 qp: -1
2025-11-19T13:59:04.561541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435720565592312:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:04.561548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:04.561658Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435720565592299:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.561677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435720565592224:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.561886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435720565592286:2105] [node 6] ICP18 handshake (actor [3:7574435720565592299:2114]) is too old
2025-11-19T13:59:04.559822Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435720041807998:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.559852Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435720041808000:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.559869Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435720041807999:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.562313Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435723346937569:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.562688Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435720991635975:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.562753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435720991635976:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-19T13:59:04.562766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435720991635897:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.563043Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435723346937563:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.563052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435723346937563:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.563940Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435720041808001:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.563963Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435720041808002:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.564001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435720041807930:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.564010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435720041807931:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.564018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435720041807932:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.564140Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721590677706:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.564150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435721590677706:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:04.565217Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435720041807982:2097] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.565222Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435720041807982:2097] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.566312Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435720041807982:2097] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:04.566412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435720041807980:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.566424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:04.566429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435720041807980:2096] [node 6] ICP22 created new session: [4:7574435720041808019:2096]
2025-11-19T13:59:04.566433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435720041808019:2096] [node 6] ICS09 handshake done sender: [4:7574435720041807982:2097] self: [4:7574435720041807981:0] peer: [6:7574435723327051925:0] socket: 217 qp: -1
2025-11-19T13:59:04.566437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435720041808019:2096] [node 6] ICS10 traffic start
2025-11-19T13:59:04.566443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:04.566711Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435723346937563:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:04.564298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435723327051924:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-19T13:59:04.564340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435723327051899:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.564361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435723327051841:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.565112Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435723327051921:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.565171Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435723327051902:2106] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:04.565915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435723327051842:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10028) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435723327051902:2106] held: no
2025-11-19T13:59:04.565931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435723327051842:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.565941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:04.565946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435723327051842:2050] [node 3] ICP22 created new session: [6:7574435723327051927:2050]
2025-11-19T13:59:04.565952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435723327051927:2050] [node 3] ICS09 handshake done sender: [6:7574435723327051902:2106] self: [6:7574435723327051922:0] peer: [3:7574435720565592291:0] socket: 231 qp: -1
2025-11-19T13:59:04.565958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435723327051927:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:04.565965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:04.566142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435723327051900:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.566227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435723327051929:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-19T13:59:04.567863Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435723327051901:2105] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:04.568065Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435720565592306:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.568038Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435723346937557:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.568045Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435723346937557:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:04.568431Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435723346937559:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.568434Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435723346937559:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:04.568967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435723346937555:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.568977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:04.568982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435723346937555:2108] [node 5] ICP22 created new session: [2:7574435723346937582:2108]
2025-11-19T13:59:04.568987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435723346937582:2108] [node 5] ICS09 handshake done sender: [2:7574435723346937563:2115] self: [2:7574435723346937562:0] peer: [5:7574435720991635977:0] socket: 256 qp: -1
2025-11-19T13:59:04.568994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435723346937582:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:04.569003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:04.569629Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435720565592290:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.569642Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435720565592290:2109] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.571979Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721590677708:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.571991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435721590677708:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:04.572586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435721590677721:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30575
2025-11-19T13:59:04.572614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435721590677722:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30575
2025-11-19T13:59:04.572769Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435721590677706:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.573050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435720041808021:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33017
2025-11-19T13:59:04.573065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435720041808022:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33017
2025-11-19T13:59:04.573075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435720041808023:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33017
2025-11-19T13:59:04.573226Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435721590677708:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:04.573301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435721590677639:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.573314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:04.573320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435721590677639:2049] [node 3] ICP22 created new session: [1:7574435721590677725:2049]
2025-11-19T13:59:04.573325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435721590677725:2049] [node 3] ICS09 handshake done sender: [1:7574435721590677706:2110] self: [1:7574435721590677705:0] peer: [3:7574435720565592314:0] socket: 218 qp: -1
2025-11-19T13:59:04.573329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435721590677725:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:04.573335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:04.573416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435721590677703:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.573421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:04.573424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435721590677703:2108] [node 4] ICP22 created new session: [1:7574435721590677727:2108]
2025-11-19T13:59:04.573427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435721590677727:2108] [node 4] ICS09 handshake done sender: [1:7574435721590677708:2111] self: [1:7574435721590677707:0] peer: [4:7574435720041808013:0] socket: 221 qp: -1
2025-11-19T13:59:04.573430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435721590677727:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:04.573434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:04.575890Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435720565592290:2109] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.576099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435720565592283:2102] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.576109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:04.576114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435720565592283:2102] [node 4] ICP22 created new session: [3:7574435720565592321:2102]
2025-11-19T13:59:04.576118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435720565592321:2102] [node 4] ICS09 handshake done sender: [3:7574435720565592290:2109] self: [3:7574435720565592289:0] peer: [4:7574435720041808015:0] socket: 223 qp: -1
2025-11-19T13:59:04.576122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435720565592321:2102] [node 4] ICS10 traffic start
2025-11-19T13:59:04.576129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:04.576246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435720565592319:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11303
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.576414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435720565592320:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11303
2025-11-19T13:59:04.576430Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435720565592285:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:04.576479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435720565592224:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.576492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:04.576496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435720565592224:2048] [node 1] ICP22 created new session: [3:7574435720565592323:2048]
2025-11-19T13:59:04.576499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435720565592323:2048] [node 1] ICS09 handshake done sender: [3:7574435720565592285:2107] self: [3:7574435720565592314:0] peer: [1:7574435721590677705:0] socket: 220 qp: -1
2025-11-19T13:59:04.576504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435720565592323:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:04.576508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.579953Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435720991635974:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.579988Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435720991635964:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.579992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435720991635964:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:04.580768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435720991635978:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-19T13:59:04.580918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721590677721:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.582083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435721590677639:2049] [node 3] ICP18 handshake (actor [1:7574435721590677721:2115]) is too old
2025-11-19T13:59:04.582382Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435723327051924:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.584180Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435720041807998:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.584379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435720991635979:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-19T13:59:04.584442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435720991635947:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.584485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435720991635976:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.584538Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435720991635975:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:04.587934Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435723346937557:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:04.587891Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721590677722:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.588110Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435723346937559:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:04.591241Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435720991635964:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:04.591883Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435720041807999:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:04.591364Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435720991635979:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.591377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435720991635897:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.591393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:04.591400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435720991635897:2049] [node 2] ICP22 created new session: [5:7574435720991635982:2049]
2025-11-19T13:59:04.591406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435720991635982:2049] [node 2] ICS09 handshake done sender: [5:7574435720991635975:2116] self: [5:7574435720991635977:0] peer: [2:7574435723346937562:0] socket: 259 qp: -1
2025-11-19T13:59:04.591412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435720991635982:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:04.591420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:04.591583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435720991635898:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.591591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435720991635946:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.591597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:04.591600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435720991635946:2061] [node 6] ICP22 created new session: [5:7574435720991635985:2061]
2025-11-19T13:59:04.591603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435720991635985:2061] [node 6] ICS09 handshake done sender: [5:7574435720991635964:2111] self: [5:7574435720991635948:0] peer: [6:7574435723327051931:0] socket: 239 qp: -1
2025-11-19T13:59:04.591607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435720991635985:2061] [node 6] ICS10 traffic start
2025-11-19T13:59:04.591610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:04.591889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435720991635896:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.592283Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435720565592305:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.592292Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435720565592305:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:04.591992Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721590677716:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.592011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435721590677716:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:04.592694Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435721590677716:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.592791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435721590677714:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.592804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:04.592808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435721590677714:2113] [node 5] ICP22 created new session: [1:7574435721590677733:2113]
2025-11-19T13:59:04.592813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435721590677733:2113] [node 5] ICS09 handshake done sender: [1:7574435721590677716:2114] self: [1:7574435721590677715:0] peer: [5:7574435720991635987:0] socket: 275 qp: -1
2025-11-19T13:59:04.592818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435721590677733:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:04.592824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:04.592943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435721590677732:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30575
2025-11-19T13:59:04.593086Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435720041808004:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.593096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435720041808004:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.594571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435723327051899:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.594594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:04.594599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435723327051899:2062] [node 4] ICP22 created new session: [6:7574435723327051934:2062]
2025-11-19T13:59:04.594605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435723327051934:2062] [node 4] ICS09 handshake done sender: [6:7574435723327051901:2105] self: [6:7574435723327051925:0] peer: [4:7574435720041807981:0] socket: 230 qp: -1
2025-11-19T13:59:04.594612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435723327051934:2062] [node 4] ICS10 traffic start
2025-11-19T13:59:04.594618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:04.594689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435723327051899:2062] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:04.594847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435723327051932:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.595054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435723327051933:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-19T13:59:04.595213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435723327051840:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:04.595336Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435723327051903:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:04.595425Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721590677710:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.595433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435721590677710:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:04.596215Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435721590677710:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:04.596301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435721590677704:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.596315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:04.596320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435721590677704:2109] [node 6] ICP22 created new session: [1:7574435721590677736:2109]
2025-11-19T13:59:04.596324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435721590677736:2109] [node 6] ICS09 handshake done sender: [1:7574435721590677710:2112] self: [1:7574435721590677709:0] peer: [6:7574435723327051936:0] socket: 252 qp: -1
2025-11-19T13:59:04.596328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435721590677736:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:04.596335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:04.597156Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435723346937576:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.597174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435723346937553:2099] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.597182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:04.597188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435723346937553:2099] [node 4] ICP22 created new session: [2:7574435723346937586:2099]
2025-11-19T13:59:04.597192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435723346937586:2099] [node 4] ICS09 handshake done sender: [2:7574435723346937557:2112] self: [2:7574435723346937556:0] peer: [4:7574435720041808014:0] socket: 226 qp: -1
2025-11-19T13:59:04.597197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435723346937586:2099] [node 4] ICS10 traffic start
2025-11-19T13:59:04.597204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:04.597372Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435720565592305:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:04.596659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435720041807930:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.596673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:04.596679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435720041807930:2048] [node 1] ICP22 created new session: [4:7574435720041808025:2048]
2025-11-19T13:59:04.596683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435720041808025:2048] [node 1] ICS09 handshake done sender: [4:7574435720041807998:2111] self: [4:7574435720041808013:0] peer: [1:7574435721590677707:0] socket: 243 qp: -1
2025-11-19T13:59:04.596687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435720041808025:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:04.596692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:04.596786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435720041807932:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.596792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:04.596796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435720041807932:2050] [node 3] ICP22 created new session: [4:7574435720041808027:2050]
2025-11-19T13:59:04.596800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435720041808027:2050] [node 3] ICS09 handshake done sender: [4:7574435720041807999:2112] self: [4:7574435720041808015:0] peer: [3:7574435720565592289:0] socket: 248 qp: -1
2025-11-19T13:59:04.596803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435720041808027:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:04.596808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:04.597982Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435723327051921:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:04.598790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435723327051841:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.598802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:04.598808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435723327051841:2049] [node 2] ICP22 created new session: [6:7574435723327051939:2049]
2025-11-19T13:59:04.598812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435723327051939:2049] [node 2] ICS09 handshake done sender: [6:7574435723327051903:2107] self: [6:7574435723327051926:0] peer: [2:7574435723346937558:0] socket: 232 qp: -1
2025-11-19T13:59:04.598817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435723327051939:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:04.598824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:04.598891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435723327051900:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.598899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:04.598902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435723327051900:2069] [node 5] ICP22 created new session: [6:7574435723327051941:2069]
2025-11-19T13:59:04.598905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435723327051941:2069] [node 5] ICS09 handshake done sender: [6:7574435723327051921:2118] self: [6:7574435723327051931:0] peer: [5:7574435720991635948:0] socket: 242 qp: -1
2025-11-19T13:59:04.598908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435723327051941:2069] [node 5] ICS10 traffic start
2025-11-19T13:59:04.598912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:04.599217Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435720041808000:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:04.599316Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435723327051840:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28535) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435723327051924:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.601307Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721590677732:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.601328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435721590677714:2113] [node 5] ICP18 handshake (actor [1:7574435721590677732:2117]) is too old
2025-11-19T13:59:04.602451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435720041807931:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.602466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:04.602472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435720041807931:2049] [node 2] ICP22 created new session: [4:7574435720041808030:2049]
2025-11-19T13:59:04.602476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435720041808030:2049] [node 2] ICS09 handshake done sender: [4:7574435720041808000:2113] self: [4:7574435720041808014:0] peer: [2:7574435723346937556:0] socket: 249 qp: -1
2025-11-19T13:59:04.602481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435720041808030:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:04.602486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:04.600570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435720991635989:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-19T13:59:04.600900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435720991635990:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-19T13:59:04.601018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435720991635991:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-19T13:59:04.601173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435720991635992:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43369
2025-11-19T13:59:04.601246Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435720991635979:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:04.601571Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435720991635976:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:04.603336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435720991635896:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.603346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:04.603351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435720991635896:2048] [node 1] ICP22 created new session: [5:7574435720991635993:2048]
2025-11-19T13:59:04.603355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435720991635993:2048] [node 1] ICS09 handshake done sender: [5:7574435720991635979:2119] self: [5:7574435720991635987:0] peer: [1:7574435721590677715:0] socket: 278 qp: -1
2025-11-19T13:59:04.603361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435720991635993:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:04.603367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:04.603429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435720991635896:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:04.603486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435720991635898:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.603492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:04.603495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435720991635898:2050] [node 3] ICP22 created new session: [5:7574435720991635995:2050]
2025-11-19T13:59:04.603498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435720991635995:2050] [node 3] ICS09 handshake done sender: [5:7574435720991635976:2117] self: [5:7574435720991635984:0] peer: [3:7574435720565592304:0] socket: 236 qp: -1
2025-11-19T13:59:04.603501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435720991635995:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:04.603508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:04.604347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435723346937554:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.604363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:04.604367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435723346937554:2101] [node 6] ICP22 created new session: [2:7574435723346937588:2101]
2025-11-19T13:59:04.604371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435723346937588:2101] [node 6] ICS09 handshake done sender: [2:7574435723346937559:2113] self: [2:7574435723346937558:0] peer: [6:7574435723327051926:0] socket: 227 qp: -1
2025-11-19T13:59:04.604375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435723346937588:2101] [node 6] ICS10 traffic start
2025-11-19T13:59:04.604381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:04.604646Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435720565592320:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.604720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435720565592301:2115] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.604730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:04.604733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435720565592301:2115] [node 5] ICP22 created new session: [3:7574435720565592326:2115]
2025-11-19T13:59:04.604737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435720565592326:2115] [node 5] ICS09 handshake done sender: [3:7574435720565592305:2117] self: [3:7574435720565592304:0] peer: [5:7574435720991635984:0] socket: 261 qp: -1
2025-11-19T13:59:04.604740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435720565592326:2115] [node 5] ICS10 traffic start
2025-11-19T13:59:04.604747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:04.605117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435723346937553:2099] [node 4] ICP18 handshake (actor [2:7574435723346937576:2119]) is too old
2025-11-19T13:59:04.607965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435720565592283:2102] [node 4] ICP18 handshake (actor [3:7574435720565592320:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.608327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435723327051938:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11053
2025-11-19T13:59:04.608518Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435720041808004:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:04.608657Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435723327051924:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:04.608787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435720041807997:2101] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:04.608801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:04.608806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435720041807997:2101] [node 5] ICP22 created new session: [4:7574435720041808032:2101]
2025-11-19T13:59:04.608810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435720041808032:2101] [node 5] ICS09 handshake done sender: [4:7574435720041808004:2116] self: [4:7574435720041808003:0] peer: [5:7574435720991635981:0] socket: 254 qp: -1
2025-11-19T13:59:04.608814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435720041808032:2101] [node 5] ICS10 traffic start
2025-11-19T13:59:04.608820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:04.609039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435723327051840:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.609049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:04.609053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435723327051840:2048] [node 1] ICP22 created new session: [6:7574435723327051943:2048]
2025-11-19T13:59:04.609057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435723327051943:2048] [node 1] ICS09 handshake done sender: [6:7574435723327051924:2120] self: [6:7574435723327051936:0] peer: [1:7574435721590677709:0] socket: 222 qp: -1
2025-11-19T13:59:04.609062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435723327051943:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:04.609067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:04.611274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435720991635947:2066] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63872) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435720991635974:2115] held: no
2025-11-19T13:59:04.611303Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435720991635974:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:04.611386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435720991635947:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:04.611395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:04.611399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435720991635947:2066] [node 4] ICP22 created new session: [5:7574435720991635997:2066]
2025-11-19T13:59:04.611403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435720991635997:2066] [node 4] ICS09 handshake done sender: [5:7574435720991635974:2115] self: [5:7574435720991635981:0] peer: [4:7574435720041808003:0] socket: 258 qp: -1
2025-11-19T13:59:04.611408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435720991635997:2066] [node 4] ICS10 traffic start
2025-11-19T13:59:04.611414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:04.627078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:04.627175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435723346937578:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-11-19T13:59:04.627186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435723346937488:2048] [node 1] ICP30 unregister session Session# [2:7574435723346937578:2048] VirtualId# [2:7574435723346937570:0]
2025-11-19T13:59:04.627193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435723346937578:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.627236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:04.627257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:04.635096Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:04.635183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435720041808025:2048] [node 1] ICS01 socket: 243 reason# EndOfStream
2025-11-19T13:59:04.635191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435720041807930:2048] [node 1] ICP30 unregister session Session# [4:7574435720041808025:2048] VirtualId# [4:7574435720041808013:0]
2025-11-19T13:59:04.635197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435720041808025:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.635241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:04.635265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:04.636813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:04.636880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435723327051943:2048] [node 1] ICS01 socket: 222 reason# EndOfStream
2025-11-19T13:59:04.636885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435723327051840:2048] [node 1] ICP30 unregister session Session# [6:7574435723327051943:2048] VirtualId# [6:7574435723327051936:0]
2025-11-19T13:59:04.636889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435723327051943:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.636925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:04.636944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:04.636967Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:04.637024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435720991635993:2048] [node 1] ICS01 socket: 278 reason# EndOfStream
2025-11-19T13:59:04.637028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435720991635896:2048] [node 1] ICP30 unregister session Session# [5:7574435720991635993:2048] VirtualId# [5:7574435720991635987:0]
2025-11-19T13:59:04.637031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435720991635993:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.637052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:04.637070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:04.637210Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:04.637263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435720565592323:2048] [node 1] ICS01 socket: 220 reason# EndOfStream
2025-11-19T13:59:04.637266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435720565592224:2048] [node 1] ICP30 unregister session Session# [3:7574435720565592323:2048] VirtualId# [3:7574435720565592314:0]
2025-11-19T13:59:04.637269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435720565592323:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.637290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:04.637306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:04.655908Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:04.656006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435720565592312:2049] [node 2] ICS01 socket: 229 reason# EndOfStream
2025-11-19T13:59:04.656017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435720565592225:2049] [node 2] ICP30 unregister session Session# [3:7574435720565592312:2049] VirtualId# [3:7574435720565592307:0]
2025-11-19T13:59:04.656024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435720565592312:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.656056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:04.656075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:04.666780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:04.666876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435723327051939:2049] [node 2] ICS01 socket: 232 reason# EndOfStream
2025-11-19T13:59:04.666885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435723327051841:2049] [node 2] ICP30 unregister session Session# [6:7574435723327051939:2049] VirtualId# [6:7574435723327051926:0]
2025-11-19T13:59:04.666894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435723327051939:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.666945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:04.666968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:04.666999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:04.667060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435720991635982:2049] [node 2] ICS01 socket: 259 reason# EndOfStream
2025-11-19T13:59:04.667064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435720991635897:2049] [node 2] ICP30 unregister session Session# [5:7574435720991635982:2049] VirtualId# [5:7574435720991635977:0]
2025-11-19T13:59:04.667068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435720991635982:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.667089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:04.667106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:04.667888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:04.667974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435720041808030:2049] [node 2] ICS01 socket: 249 reason# EndOfStream
2025-11-19T13:59:04.667982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435720041807931:2049] [node 2] ICP30 unregister session Session# [4:7574435720041808030:2049] VirtualId# [4:7574435720041808014:0]
2025-11-19T13:59:04.667988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435720041808030:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.668041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:04.668064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:04.687831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T13:59:04.687938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435720041808027:2050] [node 3] ICS01 socket: 248 reason# EndOfStream
2025-11-19T13:59:04.687950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435720041807932:2050] [node 3] ICP30 unregister session Session# [4:7574435720041808027:2050] VirtualId# [4:7574435720041808015:0]
2025-11-19T13:59:04.687958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435720041808027:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.687988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:04.688005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:04.689828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:04.689894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435723327051927:2050] [node 3] ICS01 socket: 231 reason# EndOfStream
2025-11-19T13:59:04.689900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435723327051842:2050] [node 3] ICP30 unregister session Session# [6:7574435723327051927:2050] VirtualId# [6:7574435723327051922:0]
2025-11-19T13:59:04.689905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435723327051927:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.689940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:04.689959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:04.689983Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:04.690040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435720991635995:2050] [node 3] ICS01 socket: 236 reason# EndOfStream
2025-11-19T13:59:04.690044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435720991635898:2050] [node 3] ICP30 unregister session Session# [5:7574435720991635995:2050] VirtualId# [5:7574435720991635984:0]
2025-11-19T13:59:04.690047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435720991635995:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.690069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:04.690088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:04.703816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:04.703915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435723327051934:2062] [node 4] ICS01 socket: 230 reason# EndOfStream
2025-11-19T13:59:04.703925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435723327051899:2062] [node 4] ICP30 unregister session Session# [6:7574435723327051934:2062] VirtualId# [6:7574435723327051925:0]
2025-11-19T13:59:04.703933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435723327051934:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.703976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:04.704000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:04.708843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:59:04.708934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435723327051941:2069] [node 5] ICS01 socket: 242 reason# EndOfStream
2025-11-19T13:59:04.708942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435723327051900:2069] [node 5] ICP30 unregister session Session# [6:7574435723327051941:2069] VirtualId# [6:7574435723327051931:0]
2025-11-19T13:59:04.708948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435723327051941:2069] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:04.708986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:59:04.709005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:04.767971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435721132907838:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:04.767994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435721132907839:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:04.835388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435723683261838:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:04.835398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435723683261839:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:04.825278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435721016865590:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:04.825295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435721016865591:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:04.866636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435721801231703:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:04.866653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435721801231704:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:04.866657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435721801231705:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:04.897957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435722029038853:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:04.897973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435722029038854:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:04.897978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435722029038855:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:04.936005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435720032093809:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:04.936055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435720032093811:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:04.943781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435720032093810:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:04.948378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435720032093823:2060] [node 4] ICP01 ready to work
2025-11-19T13:59:04.948639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435721132907904:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:04.952097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435721016865642:2096] [node 4] ICP01 ready to work
2025-11-19T13:59:04.948719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435721132907905:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:04.953454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435722029038885:2063] [node 4] ICP01 ready to work
2025-11-19T13:59:04.953533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435722029038886:2066] [node 6] ICP01 ready to work
2025-11-19T13:59:04.956530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435720032093844:2066] [node 5] ICP01 ready to work
2025-11-19T13:59:04.957530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435721132907911:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.975937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435722029038925:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32353
2025-11-19T13:59:04.977408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435720032093881:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-11-19T13:59:04.980256Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435722029038925:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:04.981222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435721801231766:2105] [node 5] ICP01 ready to work
2025-11-19T13:59:04.981363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435721801231768:2106] [node 6] ICP01 ready to work
2025-11-19T13:59:04.978541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435721016865645:2098] [node 5] ICP01 ready to work
2025-11-19T13:59:04.977881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435723683261838:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25791662ns) {Configure:104(25790257ns) {ProcessConfigured:135(25456291ns) {StartInitialHandshake:157(25454966ns) {DropHandshakes:469(446ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(38ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:476(1123ns) {SwitchToState:178(620ns)}}}}}
2025-11-19T13:59:04.978060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435723683261907:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:04.978137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435723683261908:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:04.979496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435723683261905:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:04.986273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435721132907905:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(36544517ns) {Configure:104(36543858ns) {ProcessConfigured:135(36359673ns) {StartInitialHandshake:157(36359345ns) {DropHandshakes:469(198ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(33ns)} GenerateSessionVirtualId:988(138ns) PrepareNewSessionHandshake:476(1333ns) {SwitchToState:178(758ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:04.994731Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435721801231705:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(42225390ns) {Configure:104(42224757ns) {ProcessConfigured:135(41914085ns) {StartInitialHandshake:157(41913968ns) {DropHandshakes:469(223ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(30ns)} GenerateSessionVirtualId:988(152ns) PrepareNewSessionHandshake:476(556ns) {SwitchToState:178(234ns)}}}}}
2025-11-19T13:59:04.992771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435723683261902:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14646
2025-11-19T13:59:04.992844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435723683261903:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14646
2025-11-19T13:59:04.992858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435723683261904:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14646
2025-11-19T13:59:04.993778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435721016865660:2100] [node 6] ICP01 ready to work
2025-11-19T13:59:04.994430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435721016865655:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11361
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.000611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435721132907917:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1615
2025-11-19T13:59:05.000631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435721132907918:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1615
2025-11-19T13:59:05.000640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435721132907919:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1615
2025-11-19T13:59:04.996990Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435721016865655:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.006329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435725311832966:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11361
2025-11-19T13:59:05.006413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435725311832967:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11361
2025-11-19T13:59:05.006443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435725311832968:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11361
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.008402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435726324006229:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32353
2025-11-19T13:59:05.012199Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435720032093881:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.012244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435720032093809:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.012360Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435725311832966:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.012391Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435725311832967:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.012420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435721016865590:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.016151Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435723683261904:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.016231Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435723683261902:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.019886Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721132907910:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.019908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435721132907910:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.019954Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721132907907:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.019961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435721132907907:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.020975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435723683261907:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(41396476ns) {Configure:104(41395714ns) {ProcessConfigured:135(41134091ns) {StartInitialHandshake:157(41133223ns) {DropHandshakes:469(379ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(47ns)} GenerateSessionVirtualId:988(276ns) PrepareNewSessionHandshake:476(1261ns) {SwitchToState:178(554ns)}}}}}
2025-11-19T13:59:05.021006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435727978229215:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14646
2025-11-19T13:59:05.021019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435727978229216:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14646
2025-11-19T13:59:05.021047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435723683261838:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.021289Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435723683261903:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.030181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435727978229221:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14646
2025-11-19T13:59:05.030528Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435723683261902:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:05.030620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435723683261838:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.030638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:05.030644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435723683261838:2048] [node 1] ICP22 created new session: [3:7574435727978229222:2048]
2025-11-19T13:59:05.030649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435727978229222:2048] [node 1] ICS09 handshake done sender: [3:7574435723683261902:2110] self: [3:7574435727978229219:0] peer: [1:7574435721132907898:0] socket: 225 qp: -1
2025-11-19T13:59:05.030655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435727978229222:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:05.030662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:05.027536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435725311832972:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11361
2025-11-19T13:59:05.028823Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435721132907907:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:05.028985Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721132907899:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.028991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435721132907899:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.029461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435721132907838:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.029482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:05.029490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435721132907838:2048] [node 2] ICP22 created new session: [1:7574435725427875217:2048]
2025-11-19T13:59:05.029496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435725427875217:2048] [node 2] ICS09 handshake done sender: [1:7574435721132907907:2110] self: [1:7574435721132907900:0] peer: [2:7574435725311832971:0] socket: 234 qp: -1
2025-11-19T13:59:05.029506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435725427875217:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:05.029516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:05.030312Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435721132907899:2107] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.030908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435726324006230:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32353
2025-11-19T13:59:05.030853Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721132907917:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.030865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435721132907839:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.030873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:05.030879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435721132907839:2049] [node 3] ICP22 created new session: [1:7574435725427875221:2049]
2025-11-19T13:59:05.030883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435725427875221:2049] [node 3] ICS09 handshake done sender: [1:7574435721132907899:2107] self: [1:7574435721132907898:0] peer: [3:7574435727978229219:0] socket: 223 qp: -1
2025-11-19T13:59:05.030887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435725427875221:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:05.030893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.031367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435726096199077:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12150
2025-11-19T13:59:05.031417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435726096199078:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12150
2025-11-19T13:59:05.031446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435726096199079:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12150
2025-11-19T13:59:05.031896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435721132907839:2049] [node 3] ICP18 handshake (actor [1:7574435721132907917:2115]) is too old
2025-11-19T13:59:05.031900Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435726096199077:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.034089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435724327061185:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-11-19T13:59:05.032905Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435725311832967:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:05.032961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435721016865590:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.032970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:05.032976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435721016865590:2048] [node 1] ICP22 created new session: [2:7574435725311832973:2048]
2025-11-19T13:59:05.032981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435725311832973:2048] [node 1] ICS09 handshake done sender: [2:7574435725311832967:2117] self: [2:7574435725311832971:0] peer: [1:7574435721132907900:0] socket: 253 qp: -1
2025-11-19T13:59:05.033831Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435727978229215:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.033938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435723683261839:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.032986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435725311832973:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:05.032991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:05.035575Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435725311832968:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.035602Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435721016865657:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.035605Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435721016865657:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.035936Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435726096199079:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.035999Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435726096199078:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.036048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435721801231704:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.036057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435721801231705:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.036189Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435726324006229:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.036273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435722029038885:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.036289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435726324006231:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32353
2025-11-19T13:59:05.036546Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435724327061185:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.036575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435720032093811:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.036485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435727978229216:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.036432Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435721801231773:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.036438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435721801231773:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.036956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435726324006233:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32353
2025-11-19T13:59:05.037063Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435721801231773:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:05.037210Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435726324006229:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:05.037291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435722029038885:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.037304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:05.037308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435722029038885:2063] [node 4] ICP22 created new session: [5:7574435726324006234:2063]
2025-11-19T13:59:05.037313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435726324006234:2063] [node 4] ICS09 handshake done sender: [5:7574435726324006229:2116] self: [5:7574435726324006232:0] peer: [4:7574435721801231772:0] socket: 248 qp: -1
2025-11-19T13:59:05.037318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435726324006234:2063] [node 4] ICS10 traffic start
2025-11-19T13:59:05.037325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:05.039276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435721801231766:2105] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.039294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:05.039299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435721801231766:2105] [node 5] ICP22 created new session: [4:7574435726096199084:2105]
2025-11-19T13:59:05.039304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435726096199084:2105] [node 5] ICS09 handshake done sender: [4:7574435721801231773:2112] self: [4:7574435721801231772:0] peer: [5:7574435726324006232:0] socket: 247 qp: -1
2025-11-19T13:59:05.039309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435726096199084:2105] [node 5] ICS10 traffic start
2025-11-19T13:59:05.039324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:05.040041Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435723683261913:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.040051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435723683261913:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.039989Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435721016865659:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.040003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435721016865659:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.043519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435726096199086:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12150
2025-11-19T13:59:05.043608Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435721016865657:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.043887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435726096199087:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12150
2025-11-19T13:59:05.040418Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435723683261915:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.040424Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435723683261915:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.042877Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435723683261913:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:05.042981Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435723683261915:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:05.043067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435723683261908:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.043077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:05.043083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435723683261908:2109] [node 4] ICP22 created new session: [3:7574435727978229229:2109]
2025-11-19T13:59:05.043088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435727978229229:2109] [node 4] ICS09 handshake done sender: [3:7574435723683261913:2115] self: [3:7574435723683261912:0] peer: [4:7574435726096199082:0] socket: 241 qp: -1
2025-11-19T13:59:05.043091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435727978229229:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:05.043096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:05.043173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435723683261905:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.043181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:05.043185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435723683261905:2107] [node 6] ICP22 created new session: [3:7574435727978229231:2107]
2025-11-19T13:59:05.043188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435727978229231:2107] [node 6] ICS09 handshake done sender: [3:7574435723683261915:2116] self: [3:7574435723683261914:0] peer: [6:7574435724327061186:0] socket: 239 qp: -1
2025-11-19T13:59:05.043191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435727978229231:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:05.043194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:05.043944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435727978229233:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14646
2025-11-19T13:59:05.041190Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721132907919:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.041284Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721132907918:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.041382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435721132907838:2048] [node 2] ICP18 handshake (actor [1:7574435721132907918:2116]) is too old
2025-11-19T13:59:05.041604Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435721132907910:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:05.041676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435721132907904:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.041684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:05.041689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435721132907904:2108] [node 6] ICP22 created new session: [1:7574435725427875226:2108]
2025-11-19T13:59:05.041694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435725427875226:2108] [node 6] ICS09 handshake done sender: [1:7574435721132907910:2112] self: [1:7574435721132907906:0] peer: [6:7574435724327061184:0] socket: 235 qp: -1
2025-11-19T13:59:05.041698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435725427875226:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:05.041703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:05.044078Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435727978229215:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:05.044466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435721016865591:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.044486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:05.044493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435721016865591:2049] [node 3] ICP22 created new session: [2:7574435725311832978:2049]
2025-11-19T13:59:05.044499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435725311832978:2049] [node 3] ICS09 handshake done sender: [2:7574435721016865657:2111] self: [2:7574435721016865648:0] peer: [3:7574435727978229225:0] socket: 242 qp: -1
2025-11-19T13:59:05.044503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435725311832978:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:05.044509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.045745Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435726324006230:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.045781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435722029038853:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.045254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435724327061188:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.046316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435723683261839:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.046329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:05.046334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435723683261839:2049] [node 2] ICP22 created new session: [3:7574435727978229235:2049]
2025-11-19T13:59:05.046338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435727978229235:2049] [node 2] ICS09 handshake done sender: [3:7574435727978229215:2117] self: [3:7574435727978229225:0] peer: [2:7574435721016865648:0] socket: 249 qp: -1
2025-11-19T13:59:05.046356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435727978229235:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:05.046361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:05.046097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435724327061187:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-11-19T13:59:05.046151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435724327061189:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-11-19T13:59:05.046533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435724327061190:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-11-19T13:59:05.047451Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435721016865659:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.047796Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721132907913:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.047809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435721132907913:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.048397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435726324006238:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32353
2025-11-19T13:59:05.048435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435726324006239:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32353
2025-11-19T13:59:05.048570Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435723683261839:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:05.049197Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435721132907913:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:05.049356Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435726324006230:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:05.049437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435721132907911:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.049448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:05.049455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435721132907911:2111] [node 5] ICP22 created new session: [1:7574435725427875229:2111]
2025-11-19T13:59:05.049459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435725427875229:2111] [node 5] ICS09 handshake done sender: [1:7574435721132907913:2113] self: [1:7574435721132907912:0] peer: [5:7574435726324006237:0] socket: 261 qp: -1
2025-11-19T13:59:05.049464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435725427875229:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:05.049470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:05.049684Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435726324006238:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.049708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435722029038855:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.049790Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435727978229217:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.049795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435727978229217:2119] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.050496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435722029038853:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.050509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:05.050514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435722029038853:2048] [node 1] ICP22 created new session: [5:7574435726324006242:2048]
2025-11-19T13:59:05.050518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435726324006242:2048] [node 1] ICS09 handshake done sender: [5:7574435726324006230:2117] self: [5:7574435726324006237:0] peer: [1:7574435721132907912:0] socket: 263 qp: -1
2025-11-19T13:59:05.050523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435726324006242:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:05.050528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:05.050746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435726324006241:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32353
2025-11-19T13:59:05.051005Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435727978229217:2119] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:05.051144Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435726324006238:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:05.051216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435722029038855:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.051227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:05.051230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435722029038855:2050] [node 3] ICP22 created new session: [5:7574435726324006244:2050]
2025-11-19T13:59:05.051234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435726324006244:2050] [node 3] ICS09 handshake done sender: [5:7574435726324006238:2120] self: [5:7574435726324006240:0] peer: [3:7574435723683261910:0] socket: 254 qp: -1
2025-11-19T13:59:05.051238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435726324006244:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:05.051243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:05.052252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435721801231705:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13881) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435726096199078:2116] held: no
2025-11-19T13:59:05.052304Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435726096199087:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.053767Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435726096199078:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:05.052541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435721016865642:2096] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.052555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:05.052562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435721016865642:2096] [node 4] ICP22 created new session: [2:7574435725311832982:2096]
2025-11-19T13:59:05.052567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435725311832982:2096] [node 4] ICS09 handshake done sender: [2:7574435721016865659:2112] self: [2:7574435721016865658:0] peer: [4:7574435726096199081:0] socket: 243 qp: -1
2025-11-19T13:59:05.052572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435725311832982:2096] [node 4] ICS10 traffic start
2025-11-19T13:59:05.052580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.054409Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435724327061185:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:05.054465Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435720032093811:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13881) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435724327061185:2116] held: no
2025-11-19T13:59:05.054479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435720032093811:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.054487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:05.054491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435720032093811:2050] [node 3] ICP22 created new session: [6:7574435724327061191:2050]
2025-11-19T13:59:05.054496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435724327061191:2050] [node 3] ICS09 handshake done sender: [6:7574435724327061185:2116] self: [6:7574435724327061186:0] peer: [3:7574435723683261914:0] socket: 264 qp: -1
2025-11-19T13:59:05.054501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435724327061191:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:05.054506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:05.054629Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435724327061187:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.054663Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435720032093881:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:05.054756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435720032093809:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.054763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:05.054766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435720032093809:2048] [node 1] ICP22 created new session: [6:7574435724327061194:2048]
2025-11-19T13:59:05.054769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435724327061194:2048] [node 1] ICS09 handshake done sender: [6:7574435720032093881:2109] self: [6:7574435724327061184:0] peer: [1:7574435721132907906:0] socket: 236 qp: -1
2025-11-19T13:59:05.054773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435724327061194:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:05.054777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:05.055002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435724327061193:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-11-19T13:59:05.055053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435720032093844:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.055013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435723683261907:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.055028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:05.055033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435723683261907:2108] [node 5] ICP22 created new session: [3:7574435727978229238:2108]
2025-11-19T13:59:05.055038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435727978229238:2108] [node 5] ICS09 handshake done sender: [3:7574435727978229217:2119] self: [3:7574435723683261910:0] peer: [5:7574435726324006240:0] socket: 237 qp: -1
2025-11-19T13:59:05.055043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435727978229238:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:05.055048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:05.055510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435721801231705:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.055521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:05.055528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435721801231705:2050] [node 3] ICP22 created new session: [4:7574435726096199090:2050]
2025-11-19T13:59:05.055534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435726096199090:2050] [node 3] ICS09 handshake done sender: [4:7574435726096199078:2116] self: [4:7574435726096199082:0] peer: [3:7574435723683261912:0] socket: 258 qp: -1
2025-11-19T13:59:05.055538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435726096199090:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:05.055542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:05.055450Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435722029038895:2082] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.055459Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435722029038895:2082] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.056480Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435722029038895:2082] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:05.056567Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435726324006231:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.056577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435722029038886:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.056586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:05.056590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435722029038886:2066] [node 6] ICP22 created new session: [5:7574435726324006247:2066]
2025-11-19T13:59:05.056594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435726324006247:2066] [node 6] ICS09 handshake done sender: [5:7574435722029038895:2082] self: [5:7574435722029038894:0] peer: [6:7574435724327061197:0] socket: 270 qp: -1
2025-11-19T13:59:05.056598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435726324006247:2066] [node 6] ICS10 traffic start
2025-11-19T13:59:05.056602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:05.056856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435722029038854:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.056950Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435721016865664:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.056957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435721016865664:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.057553Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435721016865664:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:05.057622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435721016865645:2098] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.057637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:05.057643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435721016865645:2098] [node 5] ICP22 created new session: [2:7574435725311832985:2098]
2025-11-19T13:59:05.057648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435725311832985:2098] [node 5] ICS09 handshake done sender: [2:7574435721016865664:2114] self: [2:7574435721016865663:0] peer: [5:7574435726324006249:0] socket: 265 qp: -1
2025-11-19T13:59:05.057652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435725311832985:2098] [node 5] ICS10 traffic start
2025-11-19T13:59:05.057658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:05.058611Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435724327061193:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.058994Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435722029038854:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24876) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435726324006231:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.060406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435721801231703:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.058723Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435720032093844:2066] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24958) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435724327061187:2117] held: no
2025-11-19T13:59:05.060010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435724327061199:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-11-19T13:59:05.066513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435726324006250:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32353
2025-11-19T13:59:05.066639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435726096199089:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12150
2025-11-19T13:59:05.066806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435720032093810:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.066859Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435724327061190:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.066914Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435724327061187:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:05.067267Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435721132907916:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.067278Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435721132907916:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.067215Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435726096199079:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.069151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435726096199094:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12150
2025-11-19T13:59:05.069191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435726096199095:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12150
2025-11-19T13:59:05.069368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435720032093844:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.069382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:05.069388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435720032093844:2066] [node 5] ICP22 created new session: [6:7574435724327061201:2066]
2025-11-19T13:59:05.069395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435724327061201:2066] [node 5] ICS09 handshake done sender: [6:7574435724327061187:2117] self: [6:7574435724327061197:0] peer: [5:7574435722029038894:0] socket: 227 qp: -1
2025-11-19T13:59:05.069401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435724327061201:2066] [node 5] ICS10 traffic start
2025-11-19T13:59:05.069408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:05.069493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435720032093823:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.069678Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435726096199094:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.069706Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435721801231775:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.069709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435721801231775:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.070083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435721801231704:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.070094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:05.070098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435721801231704:2049] [node 2] ICP22 created new session: [4:7574435726096199096:2049]
2025-11-19T13:59:05.070102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435726096199096:2049] [node 2] ICS09 handshake done sender: [4:7574435726096199079:2117] self: [4:7574435726096199081:0] peer: [2:7574435721016865658:0] socket: 259 qp: -1
2025-11-19T13:59:05.070106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435726096199096:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:05.070111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.070918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435721016865666:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.070928Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435721016865666:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.071596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435725311832987:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11361
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.071734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435724327061204:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-11-19T13:59:05.071835Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435721132907916:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:05.071949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435721132907905:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.071967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:05.071972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435721132907905:2109] [node 4] ICP22 created new session: [1:7574435725427875232:2109]
2025-11-19T13:59:05.071978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435725427875232:2109] [node 4] ICS09 handshake done sender: [1:7574435721132907916:2114] self: [1:7574435721132907908:0] peer: [4:7574435726096199093:0] socket: 250 qp: -1
2025-11-19T13:59:05.071983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435725427875232:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:05.071991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:05.072392Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435726324006231:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:05.072584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435722029038854:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.072598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:05.072603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435722029038854:2049] [node 2] ICP22 created new session: [5:7574435726324006251:2049]
2025-11-19T13:59:05.072608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435726324006251:2049] [node 2] ICS09 handshake done sender: [5:7574435726324006231:2118] self: [5:7574435726324006249:0] peer: [2:7574435721016865663:0] socket: 266 qp: -1
2025-11-19T13:59:05.072613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435726324006251:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:05.072618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.072940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435724327061205:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-11-19T13:59:05.073862Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435721016865666:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:05.074040Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435724327061193:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:05.074101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435720032093810:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.074114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:05.074119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435720032093810:2049] [node 2] ICP22 created new session: [6:7574435724327061206:2049]
2025-11-19T13:59:05.074124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435724327061206:2049] [node 2] ICS09 handshake done sender: [6:7574435724327061193:2121] self: [6:7574435724327061200:0] peer: [2:7574435721016865665:0] socket: 229 qp: -1
2025-11-19T13:59:05.074129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435724327061206:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:05.074134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:05.075033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435721016865660:2100] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.075050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:05.075055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435721016865660:2100] [node 6] ICP22 created new session: [2:7574435725311832989:2100]
2025-11-19T13:59:05.075059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435725311832989:2100] [node 6] ICS09 handshake done sender: [2:7574435721016865666:2115] self: [2:7574435721016865665:0] peer: [6:7574435724327061200:0] socket: 233 qp: -1
2025-11-19T13:59:05.075063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435725311832989:2100] [node 6] ICS10 traffic start
2025-11-19T13:59:05.075068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:05.075435Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435721801231703:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20230) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435726096199087:2119] held: no
2025-11-19T13:59:05.075458Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435726096199087:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:05.075724Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435721801231775:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:05.075916Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435724327061190:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:05.076029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435721801231703:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.076042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:05.076046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435721801231703:2048] [node 1] ICP22 created new session: [4:7574435726096199100:2048]
2025-11-19T13:59:05.076050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435726096199100:2048] [node 1] ICS09 handshake done sender: [4:7574435726096199087:2119] self: [4:7574435726096199093:0] peer: [1:7574435721132907908:0] socket: 274 qp: -1
2025-11-19T13:59:05.076055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435726096199100:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:05.076061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:05.076134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435721801231768:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.076141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:05.076144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435721801231768:2106] [node 6] ICP22 created new session: [4:7574435726096199102:2106]
2025-11-19T13:59:05.076148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435726096199102:2106] [node 6] ICS09 handshake done sender: [4:7574435721801231775:2113] self: [4:7574435721801231774:0] peer: [6:7574435724327061203:0] socket: 275 qp: -1
2025-11-19T13:59:05.076151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435726096199102:2106] [node 6] ICS10 traffic start
2025-11-19T13:59:05.076155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:05.076392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435720032093823:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.076405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:05.076409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435720032093823:2060] [node 4] ICP22 created new session: [6:7574435724327061208:2060]
2025-11-19T13:59:05.076413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435724327061208:2060] [node 4] ICS09 handshake done sender: [6:7574435724327061190:2120] self: [6:7574435724327061203:0] peer: [4:7574435721801231774:0] socket: 277 qp: -1
2025-11-19T13:59:05.076418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435724327061208:2060] [node 4] ICS10 traffic start
2025-11-19T13:59:05.076423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:05.077779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435721016865642:2096] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:05.099699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:05.099871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435725311832973:2048] [node 1] ICS01 socket: 253 reason# EndOfStream
2025-11-19T13:59:05.099882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435721016865590:2048] [node 1] ICP30 unregister session Session# [2:7574435725311832973:2048] VirtualId# [2:7574435725311832971:0]
2025-11-19T13:59:05.099890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435725311832973:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.099933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:05.099954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:05.102892Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:05.102956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435724327061194:2048] [node 1] ICS01 socket: 236 reason# EndOfStream
2025-11-19T13:59:05.102962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435720032093809:2048] [node 1] ICP30 unregister session Session# [6:7574435724327061194:2048] VirtualId# [6:7574435724327061184:0]
2025-11-19T13:59:05.102967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435724327061194:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.103000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:05.103020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:05.103044Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:05.103098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435726324006242:2048] [node 1] ICS01 socket: 263 reason# EndOfStream
2025-11-19T13:59:05.103102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435722029038853:2048] [node 1] ICP30 unregister session Session# [5:7574435726324006242:2048] VirtualId# [5:7574435726324006237:0]
2025-11-19T13:59:05.103105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435726324006242:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.103125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:05.103142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:05.100307Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:05.100365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435727978229222:2048] [node 1] ICS01 socket: 225 reason# EndOfStream
2025-11-19T13:59:05.100370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435723683261838:2048] [node 1] ICP30 unregister session Session# [3:7574435727978229222:2048] VirtualId# [3:7574435727978229219:0]
2025-11-19T13:59:05.100374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435727978229222:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.100394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:05.100409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:05.103846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:05.103908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435726096199100:2048] [node 1] ICS01 socket: 274 reason# EndOfStream
2025-11-19T13:59:05.103913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435721801231703:2048] [node 1] ICP30 unregister session Session# [4:7574435726096199100:2048] VirtualId# [4:7574435726096199093:0]
2025-11-19T13:59:05.103916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435726096199100:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.103948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:05.103966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:05.112289Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:05.112356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435727978229235:2049] [node 2] ICS01 socket: 249 reason# EndOfStream
2025-11-19T13:59:05.112362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435723683261839:2049] [node 2] ICP30 unregister session Session# [3:7574435727978229235:2049] VirtualId# [3:7574435727978229225:0]
2025-11-19T13:59:05.112366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435727978229235:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.112391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:05.112407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:05.130991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:05.131085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435724327061206:2049] [node 2] ICS01 socket: 229 reason# EndOfStream
2025-11-19T13:59:05.131094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435720032093810:2049] [node 2] ICP30 unregister session Session# [6:7574435724327061206:2049] VirtualId# [6:7574435724327061200:0]
2025-11-19T13:59:05.131102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435724327061206:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.131145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:05.131165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:05.131248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:05.131305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435726324006251:2049] [node 2] ICS01 socket: 266 reason# EndOfStream
2025-11-19T13:59:05.131309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435722029038854:2049] [node 2] ICP30 unregister session Session# [5:7574435726324006251:2049] VirtualId# [5:7574435726324006249:0]
2025-11-19T13:59:05.131312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435726324006251:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.131335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:05.131352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:05.132006Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:05.132072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435726096199096:2049] [node 2] ICS01 socket: 259 reason# EndOfStream
2025-11-19T13:59:05.132078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435721801231704:2049] [node 2] ICP30 unregister session Session# [4:7574435726096199096:2049] VirtualId# [4:7574435726096199081:0]
2025-11-19T13:59:05.132081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435726096199096:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.132118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:05.132137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:05.150195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:05.150305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435724327061191:2050] [node 3] ICS01 socket: 264 reason# EndOfStream
2025-11-19T13:59:05.150318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435720032093811:2050] [node 3] ICP30 unregister session Session# [6:7574435724327061191:2050] VirtualId# [6:7574435724327061186:0]
2025-11-19T13:59:05.150326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435724327061191:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.150381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:05.150403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:05.150431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:05.150497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435726324006244:2050] [node 3] ICS01 socket: 254 reason# EndOfStream
2025-11-19T13:59:05.150501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435722029038855:2050] [node 3] ICP30 unregister session Session# [5:7574435726324006244:2050] VirtualId# [5:7574435726324006240:0]
2025-11-19T13:59:05.150505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435726324006244:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.150528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:05.150545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:05.160334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T13:59:05.160422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435726324006234:2063] [node 4] ICS01 socket: 248 reason# EndOfStream
2025-11-19T13:59:05.160431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435722029038885:2063] [node 4] ICP30 unregister session Session# [5:7574435726324006234:2063] VirtualId# [5:7574435726324006232:0]
2025-11-19T13:59:05.160438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435726324006234:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.160470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:05.160489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:05.160512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:05.160570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435724327061208:2060] [node 4] ICS01 socket: 277 reason# EndOfStream
2025-11-19T13:59:05.160574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435720032093823:2060] [node 4] ICP30 unregister session Session# [6:7574435724327061208:2060] VirtualId# [6:7574435724327061203:0]
2025-11-19T13:59:05.160577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435724327061208:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.160597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:05.160614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:05.255910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435724839561079:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:05.255923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435724839561080:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:05.291935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435727991950301:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:05.291950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435727991950302:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:05.300651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435728321341244:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:05.300665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435728321341245:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:05.323242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435725225387206:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:05.323257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435725225387207:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:05.323263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435725225387208:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:05.332030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435727714401815:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:05.332047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435727714401816:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:05.332051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435727714401817:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:05.349418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435725389422767:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:05.349430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435725389422768:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:05.349434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435725389422769:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:05.385424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435727991950357:2102] [node 5] ICP01 ready to work
2025-11-19T13:59:05.387989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435727991950366:2105] [node 4] ICP01 ready to work
2025-11-19T13:59:05.388083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435727991950367:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.389106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435724839561140:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:05.389580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435724839561141:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55291
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.390111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435727714401879:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43105
2025-11-19T13:59:05.393276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435727714401887:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:05.393616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435727714401886:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43105
2025-11-19T13:59:05.394128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435728321341306:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.395192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435724839561146:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55291
2025-11-19T13:59:05.391991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435725389422816:2087] [node 4] ICP01 ready to work
2025-11-19T13:59:05.392091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435725389422821:2090] [node 5] ICP01 ready to work
2025-11-19T13:59:05.395343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435727714401880:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43105
2025-11-19T13:59:05.399959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435724839561148:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.410754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435728321341314:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49222
2025-11-19T13:59:05.411196Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435724839561141:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.413195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435725389422835:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46369
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.415591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435728321341313:2105] [node 6] ICP01 ready to work
2025-11-19T13:59:05.415818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435728321341319:2113] [node 4] ICP01 ready to work
2025-11-19T13:59:05.418873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435728321341323:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49222
2025-11-19T13:59:05.416543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435724839561153:2113] [node 4] ICP01 ready to work
2025-11-19T13:59:05.420043Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435725225387206:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23616938ns) {Configure:104(23615743ns) {ProcessConfigured:135(23245102ns) {StartInitialHandshake:157(23244725ns) {DropHandshakes:469(495ns) {DropIncomingHandshake:441(92ns) DropOutgoingHandshake:456(38ns)} GenerateSessionVirtualId:988(179ns) PrepareNewSessionHandshake:476(1143ns) {SwitchToState:178(863ns)}}}}}
2025-11-19T13:59:05.420205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435725225387272:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.422303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435727714401894:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43105
2025-11-19T13:59:05.422322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435727714401895:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43105
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.423614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435724839561160:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55291
2025-11-19T13:59:05.423824Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435727714401880:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.423859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435727714401815:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.423982Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435728321341314:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.425252Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435728321341323:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.424138Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435724839561143:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.424147Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435724839561143:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.424663Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435724839561148:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(24648346ns) {Configure:104(24646869ns) {ProcessConfigured:135(24365938ns) {StartInitialHandshake:157(24365206ns) {DropHandshakes:469(600ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:476(638ns) {SwitchToState:178(304ns)}}}}}
2025-11-19T13:59:05.427911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435728321341244:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.428161Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435724839561155:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.428168Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435724839561155:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.428787Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435724839561155:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:05.429344Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435727714401894:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.429457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435727714401893:2116] [node 4] ICP01 ready to work
2025-11-19T13:59:05.428887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435724839561080:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.428904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:05.428912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435724839561080:2049] [node 3] ICP22 created new session: [1:7574435724839561163:2049]
2025-11-19T13:59:05.428918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435724839561163:2049] [node 3] ICS09 handshake done sender: [1:7574435724839561155:2114] self: [1:7574435724839561154:0] peer: [3:7574435728321341325:0] socket: 252 qp: -1
2025-11-19T13:59:05.428924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435724839561163:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:05.428950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:05.430205Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435724839561160:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.436926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435725225387281:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31788
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.437400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435725225387284:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31788
2025-11-19T13:59:05.438749Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435724839561146:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.438912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435724839561080:2049] [node 3] ICP18 handshake (actor [1:7574435724839561146:2110]) is too old
2025-11-19T13:59:05.439074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435724839561079:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17056787ns) {Configure:104(17055965ns) {ProcessConfigured:135(16780906ns) {StartInitialHandshake:157(16780651ns) {DropHandshakes:469(418ns) {DropIncomingHandshake:441(21ns) DropOutgoingHandshake:456(45ns)} GenerateSessionVirtualId:988(418ns) PrepareNewSessionHandshake:476(505ns) {SwitchToState:178(64ns)}}}}}
2025-11-19T13:59:05.439508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435725225387280:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31788
2025-11-19T13:59:05.439836Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435727714401886:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.440696Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435725389422835:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.440737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435725389422768:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.441077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435725225387270:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:05.441211Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435725225387280:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.441233Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435725225387281:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.441907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435725225387207:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.446762Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435727991950371:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.446777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435727991950371:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.447393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435727991950301:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17736985ns) {Configure:104(17736226ns) {ProcessConfigured:135(17423833ns) {StartInitialHandshake:157(17423301ns) {DropHandshakes:469(333ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:476(675ns) {SwitchToState:178(401ns)}}}}}
2025-11-19T13:59:05.447447Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435727991950369:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.447450Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435727991950369:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.448496Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435724839561143:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:05.448773Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435728321341308:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.448785Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435728321341308:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.450088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435727714401817:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.450127Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435727714401817:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(20736197ns) {IncomingHandshake:261(20735792ns) {DropIncomingHandshake:441(55ns) PrepareNewSessionHandshake:476(397ns) {SwitchToState:178(110ns)} GenerateSessionVirtualId:988(153ns) IssueIncomingHandshakeReply:193(20731611ns) {SwitchToState:178(446ns)}}}
2025-11-19T13:59:05.450200Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435728321341308:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:05.450172Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435727714401879:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.450208Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435727714401895:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.450231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435727714401816:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.450529Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435725225387284:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.450652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435724839561140:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.450669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:05.450677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435724839561140:2107] [node 5] ICP22 created new session: [1:7574435724839561168:2107]
2025-11-19T13:59:05.450682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435724839561168:2107] [node 5] ICS09 handshake done sender: [1:7574435724839561143:2109] self: [1:7574435724839561142:0] peer: [5:7574435727714401896:0] socket: 231 qp: -1
2025-11-19T13:59:05.450687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435724839561168:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:05.450695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:05.450832Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435727991950369:2109] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.455977Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435727991950359:2103] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.455987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435727991950359:2103] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.456671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435727991950366:2105] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.456684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:05.456692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435727991950366:2105] [node 4] ICP22 created new session: [2:7574435727991950386:2105]
2025-11-19T13:59:05.456698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435727991950386:2105] [node 4] ICS09 handshake done sender: [2:7574435727991950369:2109] self: [2:7574435727991950368:0] peer: [4:7574435725225387285:0] socket: 237 qp: -1
2025-11-19T13:59:05.456706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435727991950386:2105] [node 4] ICS10 traffic start
2025-11-19T13:59:05.456726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.459834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435725389422847:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46369
2025-11-19T13:59:05.457091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435727714401901:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43105
2025-11-19T13:59:05.457190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435727714401902:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43105
2025-11-19T13:59:05.457360Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435727714401894:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:05.457509Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435727714401880:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:05.457559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435727714401817:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.457567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:05.457572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435727714401817:2050] [node 3] ICP22 created new session: [5:7574435727714401903:2050]
2025-11-19T13:59:05.457578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435727714401903:2050] [node 3] ICS09 handshake done sender: [5:7574435727714401894:2117] self: [5:7574435727714401897:0] peer: [3:7574435728321341307:0] socket: 247 qp: -1
2025-11-19T13:59:05.457583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435727714401903:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:05.457597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:05.458826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435727714401815:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.458836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:05.458842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435727714401815:2048] [node 1] ICP22 created new session: [5:7574435727714401905:2048]
2025-11-19T13:59:05.458847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435727714401905:2048] [node 1] ICS09 handshake done sender: [5:7574435727714401880:2110] self: [5:7574435727714401896:0] peer: [1:7574435724839561142:0] socket: 233 qp: -1
2025-11-19T13:59:05.458852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435727714401905:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:05.458858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.461731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435725389422848:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46369
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.462979Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435727714401893:2116] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(33406880ns) {Configure:104(33406062ns) {ProcessConfigured:135(33116245ns) {StartInitialHandshake:157(33115139ns) {DropHandshakes:469(467ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(96ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(827ns) {SwitchToState:178(630ns)}}}}}
2025-11-19T13:59:05.462716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435727991950381:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52233
2025-11-19T13:59:05.462788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435727991950382:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52233
2025-11-19T13:59:05.462826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435727991950383:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52233
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.463508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435727991950384:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52233
2025-11-19T13:59:05.459863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435728321341313:2105] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(43997582ns) {Configure:104(43996607ns) {ProcessConfigured:135(43673646ns) {StartInitialHandshake:157(43672871ns) {DropHandshakes:469(861ns) {DropIncomingHandshake:441(205ns) DropOutgoingHandshake:456(126ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:476(1612ns) {SwitchToState:178(407ns)}}}}}
2025-11-19T13:59:05.459889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435728321341306:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.459902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:05.459907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435728321341306:2107] [node 5] ICP22 created new session: [3:7574435728321341328:2107]
2025-11-19T13:59:05.459912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435728321341328:2107] [node 5] ICS09 handshake done sender: [3:7574435728321341308:2109] self: [3:7574435728321341307:0] peer: [5:7574435727714401897:0] socket: 241 qp: -1
2025-11-19T13:59:05.459918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435728321341328:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:05.459935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:05.461458Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435728321341244:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64471) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435728321341323:2115] held: no
2025-11-19T13:59:05.464680Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435727991950382:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.464957Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435727991950359:2103] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:05.466560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435725225387270:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25276806ns) {Configure:104(25275858ns) {ProcessConfigured:135(16600873ns) {StartInitialHandshake:157(16599830ns) {DropHandshakes:469(878ns) {DropIncomingHandshake:441(361ns) DropOutgoingHandshake:456(115ns)} GenerateSessionVirtualId:988(400ns) PrepareNewSessionHandshake:476(1671ns) {SwitchToState:178(750ns)}}}}}
2025-11-19T13:59:05.465074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435727714401893:2116] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.467805Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435725389422847:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.467961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435725389422821:2090] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.470255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435725225387290:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31788
2025-11-19T13:59:05.470268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435725225387291:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31788
2025-11-19T13:59:05.470763Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435724839561157:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.470774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435724839561157:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.471333Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435724839561140:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(20653851ns) {HandleHandshakeStatus:329(20653306ns) {DropIncomingHandshake:441(3135ns) DropOutgoingHandshake:456(179ns) SwitchToState:178(98ns) ProcessPendingSessionEvents:495(20563581ns) {ForwardSessionEventToSession:612(20553401ns) ForwardSessionEventToSession:612(831ns) ForwardSessionEventToSession:612(789ns) ForwardSessionEventToSession:612(864ns) ForwardSessionEventToSession:612(787ns) ForwardSessionEventToSession:612(492ns) ForwardSessionEventToSession:612(658ns) ForwardSessionEventToSession:612(865ns) ForwardSessionEventToSession:612(811ns)}}}
2025-11-19T13:59:05.471644Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435727991950371:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:05.471714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435725225387206:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.471740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435725225387206:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(21173094ns) {IncomingHandshake:261(21172852ns) {DropIncomingHandshake:441(15ns) PrepareNewSessionHandshake:476(219ns) {SwitchToState:178(108ns)} GenerateSessionVirtualId:988(153ns) IssueIncomingHandshakeReply:193(21169783ns) {SwitchToState:178(167ns)}}}
2025-11-19T13:59:05.472018Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435725225387291:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.472043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435725225387208:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.472134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435727991950385:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52233
2025-11-19T13:59:05.472179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435727991950357:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.472193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:05.472199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435727991950357:2102] [node 5] ICP22 created new session: [2:7574435727991950392:2102]
2025-11-19T13:59:05.472206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435727991950392:2102] [node 5] ICS09 handshake done sender: [2:7574435727991950359:2103] self: [2:7574435727991950358:0] peer: [5:7574435727714401900:0] socket: 228 qp: -1
2025-11-19T13:59:05.472212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435727991950392:2102] [node 5] ICS10 traffic start
2025-11-19T13:59:05.472261Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435724839561157:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:05.472220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:05.472567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435724839561153:2113] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.472589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:05.472596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435724839561153:2113] [node 4] ICP22 created new session: [1:7574435724839561172:2113]
2025-11-19T13:59:05.472600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435724839561172:2113] [node 4] ICS09 handshake done sender: [1:7574435724839561157:2115] self: [1:7574435724839561156:0] peer: [4:7574435725225387287:0] socket: 261 qp: -1
2025-11-19T13:59:05.472604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435724839561172:2113] [node 4] ICS10 traffic start
2025-11-19T13:59:05.472610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:05.474656Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435728321341322:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.474671Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435728321341322:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.475797Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435728321341322:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:05.475896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435728321341319:2113] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.475910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:05.475917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435728321341319:2113] [node 4] ICP22 created new session: [3:7574435728321341333:2113]
2025-11-19T13:59:05.475922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435728321341333:2113] [node 4] ICS09 handshake done sender: [3:7574435728321341322:2114] self: [3:7574435728321341321:0] peer: [4:7574435725225387292:0] socket: 243 qp: -1
2025-11-19T13:59:05.475928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435728321341333:2113] [node 4] ICS10 traffic start
2025-11-19T13:59:05.475935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.477178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435725389422850:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46369
2025-11-19T13:59:05.477474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435725389422851:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46369
2025-11-19T13:59:05.477610Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435727991950381:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.477632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435727991950367:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.477646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:05.477653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435727991950367:2107] [node 6] ICP22 created new session: [2:7574435727991950394:2107]
2025-11-19T13:59:05.477658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435727991950394:2107] [node 6] ICS09 handshake done sender: [2:7574435727991950371:2110] self: [2:7574435727991950370:0] peer: [6:7574435725389422846:0] socket: 238 qp: -1
2025-11-19T13:59:05.477663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435727991950394:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:05.477670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:05.477814Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435727991950383:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.477833Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435727991950384:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.478209Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435727714401891:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.478223Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435727714401891:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.478574Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435725389422835:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:05.476502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435728321341335:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49222
2025-11-19T13:59:05.476598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435728321341336:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49222
2025-11-19T13:59:05.479849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435725389422768:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.479866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:05.479872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435725389422768:2049] [node 2] ICP22 created new session: [6:7574435725389422852:2049]
2025-11-19T13:59:05.479878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435725389422852:2049] [node 2] ICS09 handshake done sender: [6:7574435725389422835:2108] self: [6:7574435725389422846:0] peer: [2:7574435727991950370:0] socket: 242 qp: -1
2025-11-19T13:59:05.479882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435725389422852:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:05.479888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:05.480154Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435728321341323:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:05.480228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435728321341244:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.480238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:05.480245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435728321341244:2048] [node 1] ICP22 created new session: [3:7574435728321341337:2048]
2025-11-19T13:59:05.480250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435728321341337:2048] [node 1] ICS09 handshake done sender: [3:7574435728321341323:2115] self: [3:7574435728321341325:0] peer: [1:7574435724839561154:0] socket: 253 qp: -1
2025-11-19T13:59:05.480254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435728321341337:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:05.480265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:05.479883Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435725225387276:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.479902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435725225387276:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.480574Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435725225387281:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:05.485133Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435727991950385:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.485173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435727991950301:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.485259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435727991950366:2105] [node 4] ICP18 handshake (actor [2:7574435727991950385:2119]) is too old
2025-11-19T13:59:05.486714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435725225387207:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.486739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:05.486751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435725225387207:2049] [node 2] ICP22 created new session: [4:7574435725225387294:2049]
2025-11-19T13:59:05.486758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435725225387294:2049] [node 2] ICS09 handshake done sender: [4:7574435725225387281:2114] self: [4:7574435725225387285:0] peer: [2:7574435727991950368:0] socket: 260 qp: -1
2025-11-19T13:59:05.486767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435725225387294:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:05.486776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.491647Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435724839561151:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.491663Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435724839561151:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.493345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435724839561174:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55291
2025-11-19T13:59:05.493770Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435724839561151:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:05.496743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435727991950357:2102] [node 5] ICP35 event processing took too much time PendingConnection:278(24535160ns) {HandleHandshakeStatus:329(24534439ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(252ns) SwitchToState:178(69ns) ProcessPendingSessionEvents:495(18233927ns) {ForwardSessionEventToSession:612(18227004ns) ForwardSessionEventToSession:612(2179ns) ForwardSessionEventToSession:612(968ns) ForwardSessionEventToSession:612(609ns)}}}
2025-11-19T13:59:05.496994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435727991950357:2102] [node 5] ICP18 handshake (actor [2:7574435727991950381:2115]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.497484Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435727714401891:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:05.497568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435727991950399:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52233
2025-11-19T13:59:05.502400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435727714401887:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.502425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:05.502433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435727714401887:2113] [node 6] ICP22 created new session: [5:7574435727714401912:2113]
2025-11-19T13:59:05.502441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435727714401912:2113] [node 6] ICS09 handshake done sender: [5:7574435727714401891:2115] self: [5:7574435727714401890:0] peer: [6:7574435725389422849:0] socket: 229 qp: -1
2025-11-19T13:59:05.502448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435727714401912:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:05.502476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:05.504056Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435728321341244:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(23801595ns) {HandleHandshakeStatus:329(23801323ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(105ns) SwitchToState:178(64ns) ProcessPendingSessionEvents:495(23727899ns) {ForwardSessionEventToSession:612(23723609ns) ForwardSessionEventToSession:612(2147ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.505286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435725389422768:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(25414977ns) {HandleHandshakeStatus:329(25414417ns) {DropIncomingHandshake:441(105ns) DropOutgoingHandshake:456(1258ns) SwitchToState:178(60ns) ProcessPendingSessionEvents:495(11733888ns) {ForwardSessionEventToSession:612(11722509ns) ForwardSessionEventToSession:612(1817ns) ForwardSessionEventToSession:612(817ns) ForwardSessionEventToSession:612(677ns) ForwardSessionEventToSession:612(580ns) ForwardSessionEventToSession:612(942ns) ForwardSessionEventToSession:612(563ns) ForwardSessionEventToSession:612(642ns) ForwardSessionEventToSession:612(676ns) ForwardSessionEventToSession:612(704ns)}}}
2025-11-19T13:59:05.505336Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435725389422848:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.505410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435725389422768:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.505961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435725389422769:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.505986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435725389422854:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46369
2025-11-19T13:59:05.506125Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435728321341336:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.506148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435728321341245:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.504331Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435727991950384:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:05.506414Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435727991950373:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.506419Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435727991950373:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.504453Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435724839561153:2113] [node 4] ICP35 event processing took too much time PendingConnection:278(31864425ns) {HandleHandshakeStatus:329(31863836ns) {DropIncomingHandshake:441(4013ns) DropOutgoingHandshake:456(104ns) SwitchToState:178(67ns) ProcessPendingSessionEvents:495(31784211ns) {ForwardSessionEventToSession:612(31774159ns) ForwardSessionEventToSession:612(1694ns) ForwardSessionEventToSession:612(537ns) ForwardSessionEventToSession:612(579ns) ForwardSessionEventToSession:612(536ns) ForwardSessionEventToSession:612(455ns) ForwardSessionEventToSession:612(717ns) ForwardSessionEventToSession:612(980ns) ForwardSessionEventToSession:612(584ns) ForwardSessionEventToSession:612(864ns)}}}
2025-11-19T13:59:05.504469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435724839561079:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.504480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:05.504486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435724839561079:2048] [node 2] ICP22 created new session: [1:7574435724839561176:2048]
2025-11-19T13:59:05.504491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435724839561176:2048] [node 2] ICS09 handshake done sender: [1:7574435724839561151:2112] self: [1:7574435724839561145:0] peer: [2:7574435727991950398:0] socket: 251 qp: -1
2025-11-19T13:59:05.504496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435724839561176:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:05.504503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:05.505729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435724839561174:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.507037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435727991950301:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.507057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:05.507071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435727991950301:2048] [node 1] ICP22 created new session: [2:7574435727991950400:2048]
2025-11-19T13:59:05.507076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435727991950400:2048] [node 1] ICS09 handshake done sender: [2:7574435727991950384:2118] self: [2:7574435727991950398:0] peer: [1:7574435724839561145:0] socket: 270 qp: -1
2025-11-19T13:59:05.507081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435727991950400:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:05.507088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:05.507362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435724839561079:2048] [node 2] ICP18 handshake (actor [1:7574435724839561174:2118]) is too old
2025-11-19T13:59:05.505186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435727714401914:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43105
2025-11-19T13:59:05.507537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435727714401915:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43105
2025-11-19T13:59:05.508736Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435727714401816:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2508) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435727714401879:2109] held: no
2025-11-19T13:59:05.510873Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435725225387276:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:05.511146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435725225387206:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64471) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435725225387284:2116] held: no
2025-11-19T13:59:05.511205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435725225387272:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.511214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:05.511220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435725225387272:2109] [node 5] ICP22 created new session: [4:7574435725225387299:2109]
2025-11-19T13:59:05.511228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435725225387299:2109] [node 5] ICS09 handshake done sender: [4:7574435725225387276:2112] self: [4:7574435725225387275:0] peer: [5:7574435727714401909:0] socket: 256 qp: -1
2025-11-19T13:59:05.511234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435725225387299:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:05.511240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.516385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435728321341340:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49222
2025-11-19T13:59:05.516528Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435727991950373:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:05.516729Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435728321341336:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:05.516844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435728321341245:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.516861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:05.516867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435728321341245:2049] [node 2] ICP22 created new session: [3:7574435728321341341:2049]
2025-11-19T13:59:05.516873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435728321341341:2049] [node 2] ICS09 handshake done sender: [3:7574435728321341336:2118] self: [3:7574435728321341339:0] peer: [2:7574435727991950372:0] socket: 283 qp: -1
2025-11-19T13:59:05.516879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435728321341341:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:05.516885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:05.516914Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435725389422851:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.517279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435727991950302:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.517295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:05.517299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435727991950302:2049] [node 3] ICP22 created new session: [2:7574435727991950403:2049]
2025-11-19T13:59:05.517304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435727991950403:2049] [node 3] ICS09 handshake done sender: [2:7574435727991950373:2111] self: [2:7574435727991950372:0] peer: [3:7574435728321341339:0] socket: 276 qp: -1
2025-11-19T13:59:05.517309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435727991950403:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:05.517315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:05.523895Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435728321341327:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.523918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435728321341327:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.525478Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435728321341327:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:05.525617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435728321341313:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.525632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:05.525640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435728321341313:2105] [node 6] ICP22 created new session: [3:7574435728321341344:2105]
2025-11-19T13:59:05.525646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435728321341344:2105] [node 6] ICS09 handshake done sender: [3:7574435728321341327:2116] self: [3:7574435728321341320:0] peer: [6:7574435725389422855:0] socket: 246 qp: -1
2025-11-19T13:59:05.525652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435728321341344:2105] [node 6] ICS10 traffic start
2025-11-19T13:59:05.525662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:05.526258Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435725389422847:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.527581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435725225387296:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31788
2025-11-19T13:59:05.527597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435725225387297:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31788
2025-11-19T13:59:05.527010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435725389422821:2090] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25000) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435725389422847:2116] held: no
2025-11-19T13:59:05.527169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435725389422767:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.527403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435725389422856:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46369
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.527637Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435727714401879:2109] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:05.529181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435727714401816:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.529195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:05.529205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435727714401816:2049] [node 2] ICP22 created new session: [5:7574435727714401916:2049]
2025-11-19T13:59:05.529211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435727714401916:2049] [node 2] ICS09 handshake done sender: [5:7574435727714401879:2109] self: [5:7574435727714401900:0] peer: [2:7574435727991950358:0] socket: 232 qp: -1
2025-11-19T13:59:05.529212Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435727714401895:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:05.529217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435727714401916:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:05.529225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:05.529099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435725225387298:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31788
2025-11-19T13:59:05.527704Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435724839561161:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.529185Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435725225387284:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:05.527716Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435724839561161:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:05.529002Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435724839561161:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:05.529300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435724839561179:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55291
2025-11-19T13:59:05.529505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435727714401893:2116] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.529517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:05.529522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435727714401893:2116] [node 4] ICP22 created new session: [5:7574435727714401918:2116]
2025-11-19T13:59:05.529527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435727714401918:2116] [node 4] ICS09 handshake done sender: [5:7574435727714401895:2118] self: [5:7574435727714401909:0] peer: [4:7574435725225387275:0] socket: 257 qp: -1
2025-11-19T13:59:05.529531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435727714401918:2116] [node 4] ICS10 traffic start
2025-11-19T13:59:05.529536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:05.529799Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435725225387298:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.531542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435724839561148:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.531557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:05.531566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435724839561148:2111] [node 6] ICP22 created new session: [1:7574435724839561180:2111]
2025-11-19T13:59:05.531571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435724839561180:2111] [node 6] ICS09 handshake done sender: [1:7574435724839561161:2117] self: [1:7574435724839561149:0] peer: [6:7574435725389422857:0] socket: 281 qp: -1
2025-11-19T13:59:05.531577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435724839561180:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:05.531584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:05.530461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435728321341346:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49222
2025-11-19T13:59:05.530479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435728321341347:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49222
2025-11-19T13:59:05.531943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435725225387206:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.531957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:05.531962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435725225387206:2048] [node 1] ICP22 created new session: [4:7574435725225387303:2048]
2025-11-19T13:59:05.531966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435725225387303:2048] [node 1] ICS09 handshake done sender: [4:7574435725225387284:2116] self: [4:7574435725225387287:0] peer: [1:7574435724839561156:0] socket: 263 qp: -1
2025-11-19T13:59:05.531971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435725225387303:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:05.531976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:05.532061Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435725225387291:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:05.534476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435725225387272:2109] [node 5] ICP18 handshake (actor [4:7574435725225387298:2122]) is too old
2025-11-19T13:59:05.535418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435725225387208:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.535432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:05.535437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435725225387208:2050] [node 3] ICP22 created new session: [4:7574435725225387305:2050]
2025-11-19T13:59:05.535442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435725225387305:2050] [node 3] ICS09 handshake done sender: [4:7574435725225387291:2119] self: [4:7574435725225387292:0] peer: [3:7574435728321341321:0] socket: 279 qp: -1
2025-11-19T13:59:05.535447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435725225387305:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:05.535452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:05.534930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435725389422821:2090] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.534949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:05.534955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435725389422821:2090] [node 5] ICP22 created new session: [6:7574435725389422860:2090]
2025-11-19T13:59:05.534961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435725389422860:2090] [node 5] ICS09 handshake done sender: [6:7574435725389422847:2116] self: [6:7574435725389422849:0] peer: [5:7574435727714401890:0] socket: 273 qp: -1
2025-11-19T13:59:05.534966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435725389422860:2090] [node 5] ICS10 traffic start
2025-11-19T13:59:05.534985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:05.535162Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435725389422848:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.537129Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435728321341347:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.537230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435728321341319:2113] [node 4] ICP18 handshake (actor [3:7574435728321341347:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.536897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435725389422769:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.536911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:05.536918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435725389422769:2050] [node 3] ICP22 created new session: [6:7574435725389422864:2050]
2025-11-19T13:59:05.536923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435725389422864:2050] [node 3] ICS09 handshake done sender: [6:7574435725389422848:2117] self: [6:7574435725389422855:0] peer: [3:7574435728321341320:0] socket: 274 qp: -1
2025-11-19T13:59:05.536928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435725389422864:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:05.536934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:05.537082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435725389422862:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46369
2025-11-19T13:59:05.537565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435725389422863:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46369
2025-11-19T13:59:05.539640Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435725389422851:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:05.540611Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435724839561179:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.540708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435724839561148:2111] [node 6] ICP18 handshake (actor [1:7574435724839561179:2119]) is too old
2025-11-19T13:59:05.539723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435725389422767:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.539736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:05.539741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435725389422767:2048] [node 1] ICP22 created new session: [6:7574435725389422866:2048]
2025-11-19T13:59:05.539780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435725389422866:2048] [node 1] ICS09 handshake done sender: [6:7574435725389422851:2119] self: [6:7574435725389422857:0] peer: [1:7574435724839561149:0] socket: 284 qp: -1
2025-11-19T13:59:05.539785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435725389422866:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:05.539792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:05.540958Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435725389422863:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.540994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435725389422816:2087] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:05.541372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435728321341313:2105] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:05.541506Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435725225387288:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:05.541516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435725225387288:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.542500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435725389422869:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46369
2025-11-19T13:59:05.542700Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435725225387288:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:05.542862Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435725389422863:2123] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:05.542917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435725389422816:2087] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:05.542930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:05.542934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435725389422816:2087] [node 4] ICP22 created new session: [6:7574435725389422870:2087]
2025-11-19T13:59:05.542939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435725389422870:2087] [node 4] ICS09 handshake done sender: [6:7574435725389422863:2123] self: [6:7574435725389422868:0] peer: [4:7574435725225387286:0] socket: 295 qp: -1
2025-11-19T13:59:05.542944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435725389422870:2087] [node 4] ICS10 traffic start
2025-11-19T13:59:05.542950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:05.544250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435725225387270:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:05.544268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:05.544274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435725225387270:2108] [node 6] ICP22 created new session: [4:7574435725225387308:2108]
2025-11-19T13:59:05.544281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435725225387308:2108] [node 6] ICS09 handshake done sender: [4:7574435725225387288:2117] self: [4:7574435725225387286:0] peer: [6:7574435725389422868:0] socket: 271 qp: -1
2025-11-19T13:59:05.544287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435725225387308:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:05.544294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:05.572727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:05.572822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435727991950400:2048] [node 1] ICS01 socket: 270 reason# EndOfStream
2025-11-19T13:59:05.572831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435727991950301:2048] [node 1] ICP30 unregister session Session# [2:7574435727991950400:2048] VirtualId# [2:7574435727991950398:0]
2025-11-19T13:59:05.572838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435727991950400:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.572889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:05.572915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:05.573466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:05.573556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435728321341337:2048] [node 1] ICS07 socket disconnect 253 reason# EndOfStream
2025-11-19T13:59:05.573563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435728321341337:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.573609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:05.573627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435728321341337:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:05.580872Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435728321341244:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64471) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:05.580893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435728321341337:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:05.581644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435728321341244:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64471) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:05.581655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435728321341337:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:05.581659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435728321341244:2048] [node 1] ICP30 unregister session Session# [3:7574435728321341337:2048] VirtualId# [3:7574435728321341325:0]
2025-11-19T13:59:05.581673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:05.581678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435728321341244:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64471) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:05.581681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64471) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:05.581602Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:05.581697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435727714401905:2048] [node 1] ICS01 socket: 233 reason# EndOfStream
2025-11-19T13:59:05.581705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435727714401815:2048] [node 1] ICP30 unregister session Session# [5:7574435727714401905:2048] VirtualId# [5:7574435727714401896:0]
2025-11-19T13:59:05.581711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435727714401905:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.581767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:05.581792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:05.582108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:05.582184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435725225387303:2048] [node 1] ICS01 socket: 263 reason# EndOfStream
2025-11-19T13:59:05.582189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435725225387206:2048] [node 1] ICP30 unregister session Session# [4:7574435725225387303:2048] VirtualId# [4:7574435725225387287:0]
2025-11-19T13:59:05.582195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435725225387303:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.582231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:05.582250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:05.583457Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:05.583532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435725389422866:2048] [node 1] ICS01 socket: 284 reason# EndOfStream
2025-11-19T13:59:05.583538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435725389422767:2048] [node 1] ICP30 unregister session Session# [6:7574435725389422866:2048] VirtualId# [6:7574435725389422857:0]
2025-11-19T13:59:05.583543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435725389422866:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.583580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:05.583604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:05.592307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574435728321341244:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:05.592340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:05.592419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435728321341341:2049] [node 2] ICS01 socket: 283 reason# EndOfStream
2025-11-19T13:59:05.592427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435728321341245:2049] [node 2] ICP30 unregister session Session# [3:7574435728321341341:2049] VirtualId# [3:7574435728321341339:0]
2025-11-19T13:59:05.592434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435728321341341:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.592468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:05.592487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:05.603864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:05.603962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435725225387294:2049] [node 2] ICS01 socket: 260 reason# EndOfStream
2025-11-19T13:59:05.603971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435725225387207:2049] [node 2] ICP30 unregister session Session# [4:7574435725225387294:2049] VirtualId# [4:7574435725225387285:0]
2025-11-19T13:59:05.603978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435725225387294:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.604024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:05.604048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:05.604081Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:05.604143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435725389422852:2049] [node 2] ICS07 socket disconnect 242 reason# EndOfStream
2025-11-19T13:59:05.604148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435725389422852:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.604171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:05.604190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435725389422852:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:05.604735Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435725389422768:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2508) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:05.604748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435725389422852:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:05.604752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435725389422768:2049] [node 2] ICP30 unregister session Session# [6:7574435725389422852:2049] VirtualId# [6:7574435725389422846:0]
2025-11-19T13:59:05.604764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:05.604769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435725389422768:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2508) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:05.604771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2508) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:05.607800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:05.607884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435727714401916:2049] [node 2] ICS01 socket: 232 reason# EndOfStream
2025-11-19T13:59:05.607892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435727714401816:2049] [node 2] ICP30 unregister session Session# [5:7574435727714401916:2049] VirtualId# [5:7574435727714401900:0]
2025-11-19T13:59:05.607905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435727714401916:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.607977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:05.608006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:05.616532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435725389422768:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:05.631220Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:05.631321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435727714401903:2050] [node 3] ICS07 socket disconnect 247 reason# EndOfStream
2025-11-19T13:59:05.631328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435727714401903:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.631405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:05.631430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435727714401903:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:05.632966Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:05.633052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435725389422864:2050] [node 3] ICS01 socket: 274 reason# EndOfStream
2025-11-19T13:59:05.633060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435725389422769:2050] [node 3] ICP30 unregister session Session# [6:7574435725389422864:2050] VirtualId# [6:7574435725389422855:0]
2025-11-19T13:59:05.633067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435725389422864:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.633125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:05.633151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:05.640053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435727714401817:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18112) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:05.640074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435727714401903:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:05.640080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435727714401817:2050] [node 3] ICP30 unregister session Session# [5:7574435727714401903:2050] VirtualId# [5:7574435727714401897:0]
2025-11-19T13:59:05.640096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:05.640103Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435727714401817:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18112) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:05.640105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18112) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:05.647263Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:05.647349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435725389422870:2087] [node 4] ICS01 socket: 295 reason# EndOfStream
2025-11-19T13:59:05.647366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435725389422816:2087] [node 4] ICP30 unregister session Session# [6:7574435725389422870:2087] VirtualId# [6:7574435725389422868:0]
2025-11-19T13:59:05.647373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435725389422870:2087] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:05.647423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:05.647444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:05.772228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435727186301432:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:05.772247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435727186301433:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:05.811309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435726573222771:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:05.811322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435726573222772:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:05.842162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435726166055991:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:05.842174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435726166055992:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:05.863875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435725847693899:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:05.863887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435725847693900:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:05.863892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435725847693901:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:05.924056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435725053119733:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:05.924067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435725053119734:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:05.924069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435725053119735:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:05.951960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435724419256928:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:05.951975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435724419256929:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:05.951979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435724419256930:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:05.982618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435726166056044:2097] [node 5] ICP01 ready to work
2025-11-19T13:59:05.982731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435726166056045:2099] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.983816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435725053119773:2084] [node 6] ICP01 ready to work
2025-11-19T13:59:05.983868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435725053119771:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1546
2025-11-19T13:59:05.984212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435727186301494:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:05.984271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435727186301495:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:05.984369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435725053119777:2089] [node 4] ICP01 ready to work
2025-11-19T13:59:05.986509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435727186301501:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:05.983978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435724419256946:2061] [node 4] ICP01 ready to work
2025-11-19T13:59:05.984175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435724419256949:2064] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.987955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435726166056053:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65131
2025-11-19T13:59:05.988254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435726573222832:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:05.992261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435725847693956:2098] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.994074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435726166056058:2105] [node 4] ICP01 ready to work
2025-11-19T13:59:05.996749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435727186301509:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7284
2025-11-19T13:59:05.996886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435724419256983:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8763
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:05.999975Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435724419256983:2092] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.000021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435724419256930:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.004524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435730868190134:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3123
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.006457Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435725053119771:2085] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.006763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435725053119735:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.006783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729348087098:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1546
2025-11-19T13:59:06.006795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729348087099:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1546
2025-11-19T13:59:06.006841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729348087100:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1546
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.008105Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435727186301433:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(23797570ns) {Configure:104(23797014ns) {ProcessConfigured:135(19893149ns) {StartInitialHandshake:157(19892266ns) {DropHandshakes:469(552ns) {DropIncomingHandshake:441(16ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(463ns) {SwitchToState:178(224ns)}}}}}
2025-11-19T13:59:06.008200Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435724419256929:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20128231ns) {Configure:104(20127681ns) {ProcessConfigured:135(19880872ns) {StartInitialHandshake:157(19880722ns) {DropHandshakes:469(678ns) {DropIncomingHandshake:441(26ns) DropOutgoingHandshake:456(24ns)} GenerateSessionVirtualId:988(130ns) PrepareNewSessionHandshake:476(667ns) {SwitchToState:178(213ns)}}}}}
2025-11-19T13:59:06.008306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435726573222832:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19998724ns) {Configure:104(19998244ns) {ProcessConfigured:135(19644409ns) {StartInitialHandshake:157(19644237ns) {DropHandshakes:469(485ns) {DropIncomingHandshake:441(77ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(62ns) PrepareNewSessionHandshake:476(667ns) {SwitchToState:178(125ns)}}}}}
2025-11-19T13:59:06.008356Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730868190134:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.008392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435726573222771:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.008969Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435726166055991:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(20191271ns) {Configure:104(20190935ns) {ProcessConfigured:135(19913466ns) {StartInitialHandshake:157(19913202ns) {DropHandshakes:469(269ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:476(1170ns) {SwitchToState:178(141ns)}}}}}
2025-11-19T13:59:06.009026Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435726166056049:2101] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.009030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435726166056049:2101] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.009712Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435726166056047:2100] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.009714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435726166056047:2100] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.010453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435730461023367:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65131
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.012153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435728714224304:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8763
2025-11-19T13:59:06.012171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435728714224305:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8763
2025-11-19T13:59:06.010517Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435725053119733:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(26657339ns) {Configure:104(26653626ns) {ProcessConfigured:135(21343279ns) {StartInitialHandshake:157(21342642ns) {DropHandshakes:469(609ns) {DropIncomingHandshake:441(165ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(713ns) {SwitchToState:178(328ns)}}}}}
2025-11-19T13:59:06.011884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435725053119734:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23849522ns) {Configure:104(23848077ns) {ProcessConfigured:135(22573229ns) {StartInitialHandshake:157(22572898ns) {DropHandshakes:469(211ns) {DropIncomingHandshake:441(36ns) DropOutgoingHandshake:456(35ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:476(545ns) {SwitchToState:178(131ns)}}}}}
2025-11-19T13:59:06.012054Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435729348087099:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.012128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435725053119733:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.012145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729348087111:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1546
2025-11-19T13:59:06.012246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435725053119773:2084] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(28347223ns) {Configure:104(28346433ns) {ProcessConfigured:135(22905347ns) {StartInitialHandshake:157(22904388ns) {DropHandshakes:469(387ns) {DropIncomingHandshake:441(16ns) DropOutgoingHandshake:456(118ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:476(407ns) {SwitchToState:178(192ns)}}}}}
2025-11-19T13:59:06.012842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435727186301501:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26279181ns) {Configure:104(26278888ns) {ProcessConfigured:135(26080178ns) {StartInitialHandshake:157(26080089ns) {DropHandshakes:469(199ns) {DropIncomingHandshake:441(31ns) DropOutgoingHandshake:456(22ns)} GenerateSessionVirtualId:988(43ns) PrepareNewSessionHandshake:476(550ns) {SwitchToState:178(270ns)}}}}}
2025-11-19T13:59:06.012435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435726166055992:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23064643ns) {Configure:104(23063700ns) {ProcessConfigured:135(22835854ns) {StartInitialHandshake:157(22835540ns) {DropHandshakes:469(155ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(142ns) PrepareNewSessionHandshake:476(473ns) {SwitchToState:178(261ns)}}}}}
2025-11-19T13:59:06.012498Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435726166056049:2101] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:06.013270Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435724419256983:2092] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:06.013428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435724419256930:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.013445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:06.013451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435724419256930:2050] [node 3] ICP22 created new session: [6:7574435728714224306:2050]
2025-11-19T13:59:06.013457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435728714224306:2050] [node 3] ICS09 handshake done sender: [6:7574435724419256983:2092] self: [6:7574435724419257001:0] peer: [3:7574435726166056048:0] socket: 239 qp: -1
2025-11-19T13:59:06.013464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435728714224306:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:06.013484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:06.015566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435730461023368:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65131
2025-11-19T13:59:06.015657Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435726166056047:2100] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:06.015895Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435725053119771:2085] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:06.015986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435725053119735:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.016009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:06.016030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435725053119735:2050] [node 3] ICP22 created new session: [5:7574435729348087114:2050]
2025-11-19T13:59:06.016040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435729348087114:2050] [node 3] ICS09 handshake done sender: [5:7574435725053119771:2085] self: [5:7574435729348087105:0] peer: [3:7574435726166056046:0] socket: 236 qp: -1
2025-11-19T13:59:06.016049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435729348087114:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:06.016070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:06.016147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435731481268806:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7284
2025-11-19T13:59:06.016212Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435727186301498:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.016219Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435727186301498:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.015951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435726166056062:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65131
2025-11-19T13:59:06.016059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435726166056045:2099] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.016077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:06.016090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435726166056045:2099] [node 6] ICP22 created new session: [3:7574435730461023372:2099]
2025-11-19T13:59:06.016096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435730461023372:2099] [node 6] ICS09 handshake done sender: [3:7574435726166056049:2101] self: [3:7574435726166056048:0] peer: [6:7574435724419257001:0] socket: 235 qp: -1
2025-11-19T13:59:06.016104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435730461023372:2099] [node 6] ICS10 traffic start
2025-11-19T13:59:06.016124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.018540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435726166056044:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.018558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:06.018567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435726166056044:2097] [node 5] ICP22 created new session: [3:7574435730461023374:2097]
2025-11-19T13:59:06.018573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435730461023374:2097] [node 5] ICS09 handshake done sender: [3:7574435726166056047:2100] self: [3:7574435726166056046:0] peer: [5:7574435729348087105:0] socket: 234 qp: -1
2025-11-19T13:59:06.018579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435730461023374:2097] [node 5] ICS10 traffic start
2025-11-19T13:59:06.018588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:06.019353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435730142661271:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45114
2025-11-19T13:59:06.019374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435730142661272:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45114
2025-11-19T13:59:06.020291Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435730142661272:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.020405Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435727186301509:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.023882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435730868190144:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3123
2025-11-19T13:59:06.025670Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435727186301500:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.025688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435727186301500:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.024967Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435726166056062:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.027961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435725847693966:2104] [node 6] ICP01 ready to work
2025-11-19T13:59:06.028552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435725847693899:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.027769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435730868190132:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:06.028169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435730868190133:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.031742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435731481268810:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7284
2025-11-19T13:59:06.031776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435731481268811:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7284
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.033332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435730142661275:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45114
2025-11-19T13:59:06.033382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435730142661276:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45114
2025-11-19T13:59:06.038920Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435729348087098:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.040627Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435730142661275:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.040694Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435730142661271:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.040723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435725847693901:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.044066Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435726166056060:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.044078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435726166056060:2106] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.043670Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435727186301498:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:06.044463Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435730142661276:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.044739Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435729348087100:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.044764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435725053119777:2089] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.044976Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435727186301504:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.044989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435727186301504:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.047094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435730868190148:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3123
2025-11-19T13:59:06.045469Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435731481268810:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.045491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435727186301432:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.045503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:06.045510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435727186301432:2048] [node 2] ICP22 created new session: [1:7574435731481268813:2048]
2025-11-19T13:59:06.045515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435731481268813:2048] [node 2] ICS09 handshake done sender: [1:7574435727186301498:2110] self: [1:7574435727186301497:0] peer: [2:7574435730868190142:0] socket: 244 qp: -1
2025-11-19T13:59:06.045521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435731481268813:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:06.045529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:06.045864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435731481268815:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7284
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.048531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435730868190149:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3123
2025-11-19T13:59:06.048964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435730868190132:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(21122562ns) {Configure:104(21121540ns) {ProcessConfigured:135(20782654ns) {StartInitialHandshake:157(20782204ns) {DropHandshakes:469(1107ns) {DropIncomingHandshake:441(405ns) DropOutgoingHandshake:456(115ns)} GenerateSessionVirtualId:988(390ns) PrepareNewSessionHandshake:476(386ns) {SwitchToState:178(70ns)}}}}}
2025-11-19T13:59:06.049152Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730868190144:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.049452Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730868190138:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.049457Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435730868190138:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.050258Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435730868190138:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:06.050492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435726573222772:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.050508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:06.050513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435726573222772:2049] [node 3] ICP22 created new session: [2:7574435730868190153:2049]
2025-11-19T13:59:06.050519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435730868190153:2049] [node 3] ICS09 handshake done sender: [2:7574435730868190138:2112] self: [2:7574435730868190137:0] peer: [3:7574435730461023380:0] socket: 255 qp: -1
2025-11-19T13:59:06.050524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435730868190153:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:06.050539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:06.051108Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435727186301504:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:06.048628Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435726166056045:2099] [node 6] ICP35 event processing took too much time PendingConnection:278(32541487ns) {HandleHandshakeStatus:329(32540773ns) {DropIncomingHandshake:441(77ns) DropOutgoingHandshake:456(268ns) SwitchToState:178(141ns) ProcessPendingSessionEvents:495(20702733ns) {ForwardSessionEventToSession:612(20690874ns) ForwardSessionEventToSession:612(1793ns) ForwardSessionEventToSession:612(924ns) ForwardSessionEventToSession:612(585ns) ForwardSessionEventToSession:612(548ns) ForwardSessionEventToSession:612(972ns) ForwardSessionEventToSession:612(590ns) ForwardSessionEventToSession:612(630ns) ForwardSessionEventToSession:612(673ns) ForwardSessionEventToSession:612(1085ns)}}}
2025-11-19T13:59:06.048817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435730461023379:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65131
2025-11-19T13:59:06.049349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435730461023367:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.049364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435726166055992:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.050169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435730461023381:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65131
2025-11-19T13:59:06.050428Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435730461023367:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:06.051186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435726166055992:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.051195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:06.051200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435726166055992:2049] [node 2] ICP22 created new session: [3:7574435730461023382:2049]
2025-11-19T13:59:06.051204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435730461023382:2049] [node 2] ICS09 handshake done sender: [3:7574435730461023367:2116] self: [3:7574435730461023380:0] peer: [2:7574435730868190137:0] socket: 264 qp: -1
2025-11-19T13:59:06.051209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435730461023382:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:06.051215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:06.052966Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435730868190134:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:06.052095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435725847693966:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23997541ns) {Configure:104(23996461ns) {ProcessConfigured:135(23668916ns) {StartInitialHandshake:157(23668531ns) {DropHandshakes:469(540ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(797ns) {SwitchToState:178(419ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.053155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729348087119:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1546
2025-11-19T13:59:06.053172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729348087120:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1546
2025-11-19T13:59:06.053413Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435725847693958:2100] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.053422Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435725847693958:2100] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.053802Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435727186301500:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:06.054000Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435726166056060:2106] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:06.055058Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435728714224304:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.055093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435724419256929:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.057139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435724419256930:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(43641742ns) {HandleHandshakeStatus:329(43641053ns) {DropIncomingHandshake:441(236ns) DropOutgoingHandshake:456(2037ns) SwitchToState:178(69ns) ProcessPendingSessionEvents:495(32783692ns) {ForwardSessionEventToSession:612(32771603ns) ForwardSessionEventToSession:612(1038ns) ForwardSessionEventToSession:612(951ns) ForwardSessionEventToSession:612(906ns) ForwardSessionEventToSession:612(573ns) ForwardSessionEventToSession:612(974ns) ForwardSessionEventToSession:612(714ns) ForwardSessionEventToSession:612(848ns) ForwardSessionEventToSession:612(513ns) ForwardSessionEventToSession:612(1185ns)}}}
2025-11-19T13:59:06.057392Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435725053119735:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(41385320ns) {HandleHandshakeStatus:329(41384810ns) {DropIncomingHandshake:441(239ns) DropOutgoingHandshake:456(2231ns) SwitchToState:178(65ns) ProcessPendingSessionEvents:495(32942662ns) {ForwardSessionEventToSession:612(32932871ns) ForwardSessionEventToSession:612(1416ns) ForwardSessionEventToSession:612(984ns) ForwardSessionEventToSession:612(578ns) ForwardSessionEventToSession:612(724ns) ForwardSessionEventToSession:612(967ns) ForwardSessionEventToSession:612(620ns) ForwardSessionEventToSession:612(583ns) ForwardSessionEventToSession:612(574ns) ForwardSessionEventToSession:612(733ns)}}}
2025-11-19T13:59:06.057534Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435729348087099:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:06.059909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435725053119733:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.059931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:06.059939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435725053119733:2048] [node 1] ICP22 created new session: [5:7574435729348087121:2048]
2025-11-19T13:59:06.059945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435729348087121:2048] [node 1] ICS09 handshake done sender: [5:7574435729348087099:2112] self: [5:7574435729348087112:0] peer: [1:7574435727186301499:0] socket: 258 qp: -1
2025-11-19T13:59:06.059951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435729348087121:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:06.059958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:06.057688Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730868190141:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.057697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435730868190141:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.058878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435726573222771:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.058892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:06.058905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435726573222771:2048] [node 1] ICP22 created new session: [2:7574435730868190157:2048]
2025-11-19T13:59:06.058910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435730868190157:2048] [node 1] ICS09 handshake done sender: [2:7574435730868190134:2110] self: [2:7574435730868190142:0] peer: [1:7574435727186301497:0] socket: 248 qp: -1
2025-11-19T13:59:06.058915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435730868190157:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:06.058921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:06.058575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435726166056058:2105] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.058598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:06.058605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435726166056058:2105] [node 4] ICP22 created new session: [3:7574435730461023385:2105]
2025-11-19T13:59:06.058610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435730461023385:2105] [node 4] ICS09 handshake done sender: [3:7574435726166056060:2106] self: [3:7574435726166056059:0] peer: [4:7574435730142661277:0] socket: 253 qp: -1
2025-11-19T13:59:06.058616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435730461023385:2105] [node 4] ICS10 traffic start
2025-11-19T13:59:06.058625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:06.059838Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435731481268815:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.059859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435727186301495:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.059878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:06.059885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435727186301495:2108] [node 5] ICP22 created new session: [1:7574435731481268820:2108]
2025-11-19T13:59:06.059890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435731481268820:2108] [node 5] ICS09 handshake done sender: [1:7574435727186301500:2111] self: [1:7574435727186301499:0] peer: [5:7574435729348087112:0] socket: 245 qp: -1
2025-11-19T13:59:06.059897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435731481268820:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:06.059904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:06.060038Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435731481268811:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.060086Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435731481268806:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.060401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435727186301494:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.060411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:06.060416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435727186301494:2107] [node 4] ICP22 created new session: [1:7574435731481268822:2107]
2025-11-19T13:59:06.060420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435731481268822:2107] [node 4] ICS09 handshake done sender: [1:7574435727186301504:2112] self: [1:7574435727186301503:0] peer: [4:7574435730142661273:0] socket: 246 qp: -1
2025-11-19T13:59:06.060424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435731481268822:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:06.060428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:06.060563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435727186301432:2048] [node 2] ICP18 handshake (actor [1:7574435731481268806:2114]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.062380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435725847693901:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27646) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435730142661275:2116] held: no
2025-11-19T13:59:06.062468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435730142661281:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45114
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.061683Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435730868190141:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:06.061795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435726573222832:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.061802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:06.061808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435726573222832:2107] [node 6] ICP22 created new session: [2:7574435730868190159:2107]
2025-11-19T13:59:06.061813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435730868190159:2107] [node 6] ICS09 handshake done sender: [2:7574435730868190141:2113] self: [2:7574435726573222833:0] peer: [6:7574435728714224309:0] socket: 263 qp: -1
2025-11-19T13:59:06.061823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435730868190159:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:06.061830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.064630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435730142661283:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45114
2025-11-19T13:59:06.067774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435730142661282:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45114
2025-11-19T13:59:06.067893Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730868190149:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.068017Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435730461023379:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.068052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435726166055991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.069732Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435731481268807:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.069741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435731481268807:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.070995Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435731481268807:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:06.071164Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435729348087120:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.071197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435725053119734:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.071349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435727186301433:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.071364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:06.071370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435727186301433:2049] [node 3] ICP22 created new session: [1:7574435731481268828:2049]
2025-11-19T13:59:06.071376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435731481268828:2049] [node 3] ICS09 handshake done sender: [1:7574435731481268807:2115] self: [1:7574435727186301508:0] peer: [3:7574435730461023387:0] socket: 280 qp: -1
2025-11-19T13:59:06.071381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435731481268828:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:06.071396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:06.069033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435728714224310:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8763
2025-11-19T13:59:06.069502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435728714224311:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8763
2025-11-19T13:59:06.069513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435728714224312:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8763
2025-11-19T13:59:06.071487Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435728714224304:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:06.071553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435724419256929:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.071565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:06.071573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435724419256929:2049] [node 2] ICP22 created new session: [6:7574435728714224313:2049]
2025-11-19T13:59:06.071577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435728714224313:2049] [node 2] ICS09 handshake done sender: [6:7574435728714224304:2116] self: [6:7574435728714224309:0] peer: [2:7574435726573222833:0] socket: 269 qp: -1
2025-11-19T13:59:06.071581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435728714224313:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:06.071585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:06.075485Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435730142661272:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:06.075693Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435725847693899:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29724) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435730142661272:2115] held: no
2025-11-19T13:59:06.075720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435725847693899:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.075732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:06.075738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435725847693899:2048] [node 1] ICP22 created new session: [4:7574435730142661285:2048]
2025-11-19T13:59:06.075759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435730142661285:2048] [node 1] ICS09 handshake done sender: [4:7574435730142661272:2115] self: [4:7574435730142661273:0] peer: [1:7574435727186301503:0] socket: 256 qp: -1
2025-11-19T13:59:06.075770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435730142661285:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:06.075810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:06.076033Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435730142661275:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:06.076832Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435725847693958:2100] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.077858Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435730142661283:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.077879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435725847693901:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.077887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:06.077893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435725847693901:2050] [node 3] ICP22 created new session: [4:7574435730142661288:2050]
2025-11-19T13:59:06.077898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435730142661288:2050] [node 3] ICS09 handshake done sender: [4:7574435730142661275:2116] self: [4:7574435730142661277:0] peer: [3:7574435726166056059:0] socket: 257 qp: -1
2025-11-19T13:59:06.077903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435730142661288:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:06.077909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:06.077984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435725847693956:2098] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.077994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:06.077998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435725847693956:2098] [node 5] ICP22 created new session: [4:7574435730142661290:2098]
2025-11-19T13:59:06.078002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435730142661290:2098] [node 5] ICS09 handshake done sender: [4:7574435725847693958:2100] self: [4:7574435725847693957:0] peer: [5:7574435729348087117:0] socket: 251 qp: -1
2025-11-19T13:59:06.078005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435730142661290:2098] [node 5] ICS10 traffic start
2025-11-19T13:59:06.078010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:06.077225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730868190147:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.077241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435730868190147:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.078351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435730868190163:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3123
2025-11-19T13:59:06.077535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729348087125:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1546
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.080118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435728714224315:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8763
2025-11-19T13:59:06.081048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435726166055991:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29724) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435730461023379:2119] held: no
2025-11-19T13:59:06.079885Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730868190150:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.079897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435730868190150:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.081872Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730868190163:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.078568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435725847693900:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.082019Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435728714224312:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.082041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435724419256928:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.084117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435730461023388:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65131
2025-11-19T13:59:06.085460Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435725053119777:2089] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15024) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435729348087100:2113] held: no
2025-11-19T13:59:06.085643Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435729348087100:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:06.086033Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435728714224311:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.086060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435724419256949:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.087116Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435730461023379:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:06.087324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435726166055991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.087341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:06.087348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435726166055991:2048] [node 1] ICP22 created new session: [3:7574435730461023389:2048]
2025-11-19T13:59:06.087353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435730461023389:2048] [node 1] ICS09 handshake done sender: [3:7574435730461023379:2119] self: [3:7574435730461023387:0] peer: [1:7574435727186301508:0] socket: 286 qp: -1
2025-11-19T13:59:06.087359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435730461023389:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:06.087369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:06.087710Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435730868190147:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:06.087813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435730868190133:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.087830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:06.087835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435730868190133:2109] [node 5] ICP22 created new session: [2:7574435730868190166:2109]
2025-11-19T13:59:06.087840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435730868190166:2109] [node 5] ICS09 handshake done sender: [2:7574435730868190147:2115] self: [2:7574435730868190146:0] peer: [5:7574435729348087124:0] socket: 289 qp: -1
2025-11-19T13:59:06.087844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435730868190166:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:06.087849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.087192Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435729348087113:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.087203Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435729348087113:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.087864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435725053119777:2089] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.087874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:06.087881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435725053119777:2089] [node 4] ICP22 created new session: [5:7574435729348087126:2089]
2025-11-19T13:59:06.087887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435729348087126:2089] [node 4] ICS09 handshake done sender: [5:7574435729348087100:2113] self: [5:7574435729348087117:0] peer: [4:7574435725847693957:0] socket: 259 qp: -1
2025-11-19T13:59:06.087894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435729348087126:2089] [node 4] ICS10 traffic start
2025-11-19T13:59:06.087902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:06.090049Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435729348087113:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:06.091103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435725053119773:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.091121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:06.091127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435725053119773:2084] [node 6] ICP22 created new session: [5:7574435729348087129:2084]
2025-11-19T13:59:06.091133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435729348087129:2084] [node 6] ICS09 handshake done sender: [5:7574435729348087113:2119] self: [5:7574435725053119784:0] peer: [6:7574435728714224317:0] socket: 254 qp: -1
2025-11-19T13:59:06.091138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435729348087129:2084] [node 6] ICS10 traffic start
2025-11-19T13:59:06.091145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:06.091244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435725053119734:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13068) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435729348087120:2121] held: no
2025-11-19T13:59:06.089022Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435731481268808:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.089037Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435731481268808:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.089895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435730142661294:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45114
2025-11-19T13:59:06.088991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435724419256929:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(17409568ns) {HandleHandshakeStatus:329(17409213ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(4414ns) SwitchToState:178(44ns) ProcessPendingSessionEvents:495(17346296ns) {ForwardSessionEventToSession:612(17333505ns) ForwardSessionEventToSession:612(1652ns) ForwardSessionEventToSession:612(611ns) ForwardSessionEventToSession:612(1349ns) ForwardSessionEventToSession:612(581ns) ForwardSessionEventToSession:612(704ns) ForwardSessionEventToSession:612(596ns) ForwardSessionEventToSession:612(583ns) ForwardSessionEventToSession:612(694ns) ForwardSessionEventToSession:612(1102ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.091940Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435730868190150:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:06.092003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435724419256949:2064] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2566) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435728714224311:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.092543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435728714224318:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8763
2025-11-19T13:59:06.092550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435728714224319:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8763
2025-11-19T13:59:06.092705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729348087131:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1546
2025-11-19T13:59:06.092744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435730868190132:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.092760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:06.092765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435730868190132:2108] [node 4] ICP22 created new session: [2:7574435730868190169:2108]
2025-11-19T13:59:06.092770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435730868190169:2108] [node 4] ICS09 handshake done sender: [2:7574435730868190150:2118] self: [2:7574435730868190145:0] peer: [4:7574435730142661292:0] socket: 249 qp: -1
2025-11-19T13:59:06.092775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435730868190169:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:06.092781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:06.092974Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435729348087120:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:06.093215Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435728714224315:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.093234Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435728714224311:2119] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:06.093284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435724419256946:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.093077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435725053119734:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.093091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:06.093096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435725053119734:2049] [node 2] ICP22 created new session: [5:7574435729348087132:2049]
2025-11-19T13:59:06.093111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435729348087132:2049] [node 2] ICS09 handshake done sender: [5:7574435729348087120:2121] self: [5:7574435729348087124:0] peer: [2:7574435730868190146:0] socket: 290 qp: -1
2025-11-19T13:59:06.093117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435729348087132:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:06.093130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:06.093411Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435730142661279:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.093426Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435730142661279:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.094462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435728714224321:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8763
2025-11-19T13:59:06.095931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435724419256949:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.095949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:06.095954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435724419256949:2064] [node 5] ICP22 created new session: [6:7574435728714224322:2064]
2025-11-19T13:59:06.095960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435728714224322:2064] [node 5] ICS09 handshake done sender: [6:7574435728714224311:2119] self: [6:7574435728714224317:0] peer: [5:7574435725053119784:0] socket: 252 qp: -1
2025-11-19T13:59:06.095966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435728714224322:2064] [node 5] ICS10 traffic start
2025-11-19T13:59:06.095977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:06.096997Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435731481268808:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:06.096433Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435730142661279:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:06.096495Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435730142661283:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:06.096555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435725847693900:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13068) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435730142661283:2121] held: no
2025-11-19T13:59:06.096572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435725847693900:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.096580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:06.096585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435725847693900:2049] [node 2] ICP22 created new session: [4:7574435730142661296:2049]
2025-11-19T13:59:06.096589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435730142661296:2049] [node 2] ICS09 handshake done sender: [4:7574435730142661283:2121] self: [4:7574435730142661292:0] peer: [2:7574435730868190145:0] socket: 292 qp: -1
2025-11-19T13:59:06.096594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435730142661296:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:06.096601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:06.096811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435725847693966:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.096821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:06.096824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435725847693966:2104] [node 6] ICP22 created new session: [4:7574435730142661298:2104]
2025-11-19T13:59:06.096828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435730142661298:2104] [node 6] ICS09 handshake done sender: [4:7574435730142661279:2118] self: [4:7574435730142661274:0] peer: [6:7574435728714224320:0] socket: 243 qp: -1
2025-11-19T13:59:06.096832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435730142661298:2104] [node 6] ICS10 traffic start
2025-11-19T13:59:06.096837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:06.097105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435727186301501:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.097141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:06.097151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435727186301501:2109] [node 6] ICP22 created new session: [1:7574435731481268831:2109]
2025-11-19T13:59:06.097156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435731481268831:2109] [node 6] ICS09 handshake done sender: [1:7574435731481268808:2116] self: [1:7574435727186301502:0] peer: [6:7574435728714224316:0] socket: 293 qp: -1
2025-11-19T13:59:06.097160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435731481268831:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:06.097165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:06.097510Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435728714224315:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:06.097656Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435728714224312:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:06.098461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435724419256946:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.098475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:06.098481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435724419256946:2061] [node 4] ICP22 created new session: [6:7574435728714224324:2061]
2025-11-19T13:59:06.098486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435728714224324:2061] [node 4] ICS09 handshake done sender: [6:7574435728714224315:2121] self: [6:7574435728714224320:0] peer: [4:7574435730142661274:0] socket: 277 qp: -1
2025-11-19T13:59:06.098492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435728714224324:2061] [node 4] ICS10 traffic start
2025-11-19T13:59:06.098511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:06.098585Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435724419256946:2061] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:06.098943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435724419256928:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.098956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:06.098961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435724419256928:2048] [node 1] ICP22 created new session: [6:7574435728714224326:2048]
2025-11-19T13:59:06.098966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435728714224326:2048] [node 1] ICS09 handshake done sender: [6:7574435728714224312:2120] self: [6:7574435728714224316:0] peer: [1:7574435727186301502:0] socket: 262 qp: -1
2025-11-19T13:59:06.098971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435728714224326:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:06.098976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:06.112331Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:06.112427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435730868190157:2048] [node 1] ICS01 socket: 248 reason# EndOfStream
2025-11-19T13:59:06.112437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435726573222771:2048] [node 1] ICP30 unregister session Session# [2:7574435730868190157:2048] VirtualId# [2:7574435730868190142:0]
2025-11-19T13:59:06.112445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435730868190157:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.112491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:06.112516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:06.119810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:06.119913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435730461023389:2048] [node 1] ICS01 socket: 286 reason# EndOfStream
2025-11-19T13:59:06.119925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435726166055991:2048] [node 1] ICP30 unregister session Session# [3:7574435730461023389:2048] VirtualId# [3:7574435730461023387:0]
2025-11-19T13:59:06.119932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435730461023389:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.119992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:06.120025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:06.123439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:06.124060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:06.124133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435728714224326:2048] [node 1] ICS01 socket: 262 reason# EndOfStream
2025-11-19T13:59:06.124140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435724419256928:2048] [node 1] ICP30 unregister session Session# [6:7574435728714224326:2048] VirtualId# [6:7574435728714224316:0]
2025-11-19T13:59:06.124147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435728714224326:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.124188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:06.124211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:06.123542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435729348087121:2048] [node 1] ICS07 socket disconnect 258 reason# EndOfStream
2025-11-19T13:59:06.123561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435729348087121:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.123609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:06.123632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435729348087121:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:06.127040Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435725053119733:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29724) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:06.127052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435729348087121:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:06.127058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435725053119733:2048] [node 1] ICP30 unregister session Session# [5:7574435729348087121:2048] VirtualId# [5:7574435729348087112:0]
2025-11-19T13:59:06.127076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:06.127082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435725053119733:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29724) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:06.127085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29724) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:06.129330Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:06.129400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435730461023382:2049] [node 2] ICS01 socket: 264 reason# EndOfStream
2025-11-19T13:59:06.129407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435726166055992:2049] [node 2] ICP30 unregister session Session# [3:7574435730461023382:2049] VirtualId# [3:7574435730461023380:0]
2025-11-19T13:59:06.129413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435730461023382:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.129451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:06.129472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:06.124417Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:06.124479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435730142661285:2048] [node 1] ICS01 socket: 256 reason# EndOfStream
2025-11-19T13:59:06.124484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435725847693899:2048] [node 1] ICP30 unregister session Session# [4:7574435730142661285:2048] VirtualId# [4:7574435730142661273:0]
2025-11-19T13:59:06.124489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435730142661285:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.124519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:06.124538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:06.132910Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:06.132986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435730142661296:2049] [node 2] ICS01 socket: 292 reason# EndOfStream
2025-11-19T13:59:06.132993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435725847693900:2049] [node 2] ICP30 unregister session Session# [4:7574435730142661296:2049] VirtualId# [4:7574435730142661292:0]
2025-11-19T13:59:06.132999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435730142661296:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.133034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:06.133058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:06.133318Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:06.133387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435729348087132:2049] [node 2] ICS01 socket: 290 reason# EndOfStream
2025-11-19T13:59:06.133392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435725053119734:2049] [node 2] ICP30 unregister session Session# [5:7574435729348087132:2049] VirtualId# [5:7574435729348087124:0]
2025-11-19T13:59:06.133398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435729348087132:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.133447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:06.133483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:06.136065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:06.136131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435728714224313:2049] [node 2] ICS07 socket disconnect 269 reason# EndOfStream
2025-11-19T13:59:06.136136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435728714224313:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.136179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:06.136206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435728714224313:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:06.136679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435724419256929:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13068) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:06.136689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435728714224313:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:06.136692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435724419256929:2049] [node 2] ICP30 unregister session Session# [6:7574435728714224313:2049] VirtualId# [6:7574435728714224309:0]
2025-11-19T13:59:06.136703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:06.136707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435724419256929:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13068) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:06.136709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13068) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:06.136399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435725053119733:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:06.148215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435724419256929:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:06.150039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:06.150126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435729348087114:2050] [node 3] ICS07 socket disconnect 236 reason# EndOfStream
2025-11-19T13:59:06.150132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435729348087114:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.150185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:06.150206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435729348087114:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:06.151099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435725053119735:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27646) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:06.151110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435729348087114:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:06.151115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435725053119735:2050] [node 3] ICP30 unregister session Session# [5:7574435729348087114:2050] VirtualId# [5:7574435729348087105:0]
2025-11-19T13:59:06.151129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:06.151134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435725053119735:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27646) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:06.151136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27646) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:06.151643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:06.151721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435728714224306:2050] [node 3] ICS07 socket disconnect 239 reason# EndOfStream
2025-11-19T13:59:06.151727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435728714224306:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.151788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:06.151810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435728714224306:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:06.152362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435724419256930:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27646) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:06.152370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435728714224306:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:06.152375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435724419256930:2050] [node 3] ICP30 unregister session Session# [6:7574435728714224306:2050] VirtualId# [6:7574435724419257001:0]
2025-11-19T13:59:06.152390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:06.152395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435724419256930:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27646) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:06.152397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27646) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:06.162328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435725053119735:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:59:06.164174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435724419256930:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:59:06.164203Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:06.164279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435728714224324:2061] [node 4] ICS01 socket: 277 reason# EndOfStream
2025-11-19T13:59:06.164287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435724419256946:2061] [node 4] ICP30 unregister session Session# [6:7574435728714224324:2061] VirtualId# [6:7574435728714224320:0]
2025-11-19T13:59:06.164295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435728714224324:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.164355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:06.164377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:06.268697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435730303151998:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:06.268717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435730303151999:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:06.315307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435730049600796:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:06.315318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435730049600797:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:06.391853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435731231831863:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:06.391865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435731231831864:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:06.439928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435728905909726:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:06.439948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435728905909727:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:06.439952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435728905909728:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:06.464820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435729377077854:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:06.464846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435729377077855:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:06.464851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435729377077856:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:06.496192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435729554963498:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:06.503390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435729554963497:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:06.506044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435729554963496:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:06.555598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435730303152061:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:06.555684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435730303152062:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:06.555763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435730303152063:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:06.557497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435730049600862:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.558347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435730303152066:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27261
2025-11-19T13:59:06.559934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435729554963544:2088] [node 5] ICP01 ready to work
2025-11-19T13:59:06.560021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435729554963545:2090] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.565824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435731231831924:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.566949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435730049600867:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19208
2025-11-19T13:59:06.567121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435730049600871:2113] [node 5] ICP01 ready to work
2025-11-19T13:59:06.567193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435730049600872:2114] [node 6] ICP01 ready to work
2025-11-19T13:59:06.567547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435730049600868:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19208
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.573648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435730303152063:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17701460ns) {Configure:104(17700731ns) {ProcessConfigured:135(11677326ns) {StartInitialHandshake:157(11676596ns) {DropHandshakes:469(443ns) {DropIncomingHandshake:441(45ns) DropOutgoingHandshake:456(52ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(1052ns) {SwitchToState:178(888ns)}}}}}
2025-11-19T13:59:06.572045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729377077915:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5169
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.576511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435728905909788:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15143
2025-11-19T13:59:06.576522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435731231831929:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:06.576669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435731231831930:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
2025-11-19T13:59:06.576717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435731231831931:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
2025-11-19T13:59:06.577038Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435730303152066:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.576806Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730049600867:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.577594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435728905909789:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15143
2025-11-19T13:59:06.577610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435728905909790:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15143
2025-11-19T13:59:06.577181Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435731231831930:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.582324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435728905909794:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:06.583149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435729554963544:2088] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23099075ns) {Configure:104(23098215ns) {ProcessConfigured:135(22799406ns) {StartInitialHandshake:157(22798971ns) {DropHandshakes:469(522ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(154ns) PrepareNewSessionHandshake:476(654ns) {SwitchToState:178(331ns)}}}}}
2025-11-19T13:59:06.583768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435728905909787:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15143
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.588833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435730303152078:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27261
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.589311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729377077922:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5169
2025-11-19T13:59:06.584261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435728905909792:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.594487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435730303152079:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27261
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.596683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435730049600878:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19208
2025-11-19T13:59:06.597147Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435728905909788:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.597176Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435728905909790:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.597196Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435728905909787:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.596303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435731231831940:2114] [node 5] ICP01 ready to work
2025-11-19T13:59:06.597929Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435731231831931:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.597914Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435729377077922:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.597914Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730049600868:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.597961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435730049600796:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.598134Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435730303152079:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.598346Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435729377077915:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.605092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435729377077928:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:06.605867Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435730303152074:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.605881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435730303152074:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.607265Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435730303151999:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(50561322ns) {Configure:104(50560316ns) {ProcessConfigured:135(43537042ns) {StartInitialHandshake:157(43536397ns) {DropHandshakes:469(486ns) {DropIncomingHandshake:441(95ns) DropOutgoingHandshake:456(31ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:476(852ns) {SwitchToState:178(80ns)}}}}}
2025-11-19T13:59:06.607368Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435730303152074:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:06.607853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729377077932:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5169
2025-11-19T13:59:06.608032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435729554963496:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25010537ns) {Configure:104(25007442ns) {ProcessConfigured:135(24679823ns) {StartInitialHandshake:157(24679373ns) {DropHandshakes:469(720ns) {DropIncomingHandshake:441(157ns) DropOutgoingHandshake:456(201ns)} GenerateSessionVirtualId:988(581ns) PrepareNewSessionHandshake:476(1724ns) {SwitchToState:178(958ns)}}}}}
2025-11-19T13:59:06.608039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435730303152062:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.608057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:06.608063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435730303152062:2108] [node 5] ICP22 created new session: [1:7574435730303152083:2108]
2025-11-19T13:59:06.608074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435730303152083:2108] [node 5] ICS09 handshake done sender: [1:7574435730303152074:2113] self: [1:7574435730303152071:0] peer: [5:7574435729377077926:0] socket: 252 qp: -1
2025-11-19T13:59:06.608082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435730303152083:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:06.608104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.608310Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435728905909728:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(23944522ns) {Configure:104(23944004ns) {ProcessConfigured:135(23603975ns) {StartInitialHandshake:157(23603811ns) {DropHandshakes:469(594ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(30ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:476(802ns) {SwitchToState:178(435ns)}}}}}
2025-11-19T13:59:06.608342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435728905909728:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.608704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435729554963574:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31264
2025-11-19T13:59:06.608774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435729554963575:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31264
2025-11-19T13:59:06.608805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435729554963576:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31264
2025-11-19T13:59:06.608980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435728905909726:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24153550ns) {Configure:104(24152853ns) {ProcessConfigured:135(23953880ns) {StartInitialHandshake:157(23953675ns) {DropHandshakes:469(224ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(175ns) PrepareNewSessionHandshake:476(403ns) {SwitchToState:178(259ns)}}}}}
2025-11-19T13:59:06.609006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435728905909726:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.608148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435729377077856:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(24284996ns) {Configure:104(24284137ns) {ProcessConfigured:135(23900759ns) {StartInitialHandshake:157(23900242ns) {DropHandshakes:469(438ns) {DropIncomingHandshake:441(77ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:476(861ns) {SwitchToState:178(293ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.611444Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435729554963574:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.611491Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435729554963576:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.611534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435729554963498:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.611544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435729554963496:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.609641Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435731231831935:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.609657Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435731231831935:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.611415Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435731231831935:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:06.611499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435731231831924:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.611509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:06.611515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435731231831924:2107] [node 4] ICP22 created new session: [3:7574435731231831946:2107]
2025-11-19T13:59:06.611520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435731231831946:2107] [node 4] ICS09 handshake done sender: [3:7574435731231831935:2112] self: [3:7574435731231831925:0] peer: [4:7574435728905909803:0] socket: 258 qp: -1
2025-11-19T13:59:06.611527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435731231831946:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:06.611534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:06.610357Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435730303152069:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.610381Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435730303152069:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.611216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435730303152086:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27261
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.612160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729377077934:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5169
2025-11-19T13:59:06.612768Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435729554963575:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.612807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435729554963544:2088] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.613839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435728905909808:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15143
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.614014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435728905909809:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15143
2025-11-19T13:59:06.614031Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435728905909790:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:06.614088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435728905909728:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.614104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:06.614109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435728905909728:2050] [node 3] ICP22 created new session: [4:7574435728905909810:2050]
2025-11-19T13:59:06.614115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435728905909810:2050] [node 3] ICS09 handshake done sender: [4:7574435728905909790:2111] self: [4:7574435728905909803:0] peer: [3:7574435731231831925:0] socket: 260 qp: -1
2025-11-19T13:59:06.614121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435728905909810:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:06.614139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.612912Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435729377077930:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.612918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435729377077930:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.614603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435729554963580:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31264
2025-11-19T13:59:06.614763Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435729377077930:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:06.614945Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435729554963575:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:06.615120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435729377077928:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.615136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:06.615141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435729377077928:2111] [node 6] ICP22 created new session: [5:7574435729377077936:2111]
2025-11-19T13:59:06.615146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435729377077936:2111] [node 6] ICS09 handshake done sender: [5:7574435729377077930:2114] self: [5:7574435729377077929:0] peer: [6:7574435729554963579:0] socket: 270 qp: -1
2025-11-19T13:59:06.615153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435729377077936:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:06.615160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:06.615509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435729554963544:2088] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.615525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:06.615532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435729554963544:2088] [node 5] ICP22 created new session: [6:7574435729554963581:2088]
2025-11-19T13:59:06.615537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435729554963581:2088] [node 5] ICS09 handshake done sender: [6:7574435729554963575:2116] self: [6:7574435729554963579:0] peer: [5:7574435729377077929:0] socket: 275 qp: -1
2025-11-19T13:59:06.615541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435729554963581:2088] [node 5] ICS10 traffic start
2025-11-19T13:59:06.615560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.618017Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435730303152069:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:06.618075Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435730303152065:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.618086Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435730303152065:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.617926Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435728905909789:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.619864Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435731231831933:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.619882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435731231831933:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.620141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435731231831864:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(44219071ns) {Configure:104(44218104ns) {ProcessConfigured:135(43928659ns) {StartInitialHandshake:157(43928202ns) {DropHandshakes:469(461ns) {DropIncomingHandshake:441(119ns) DropOutgoingHandshake:456(30ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:476(729ns) {SwitchToState:178(300ns)}}}}}
2025-11-19T13:59:06.620264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435731231831864:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.620517Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730049600861:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.620529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435730049600861:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.621546Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435730049600861:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:06.621649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435730049600797:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.621665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:06.621671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435730049600797:2049] [node 3] ICP22 created new session: [2:7574435730049600881:2049]
2025-11-19T13:59:06.621677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435730049600881:2049] [node 3] ICS09 handshake done sender: [2:7574435730049600861:2109] self: [2:7574435730049600860:0] peer: [3:7574435731231831950:0] socket: 242 qp: -1
2025-11-19T13:59:06.621682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435730049600881:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:06.621716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:06.623885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435728905909727:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(36071860ns) {Configure:104(36068262ns) {ProcessConfigured:135(35768907ns) {StartInitialHandshake:157(35768366ns) {DropHandshakes:469(503ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(107ns)} GenerateSessionVirtualId:988(250ns) PrepareNewSessionHandshake:476(865ns) {SwitchToState:178(702ns)}}}}}
2025-11-19T13:59:06.623998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435728905909727:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.624011Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435728905909788:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:06.624454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435728905909726:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.624470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:06.624475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435728905909726:2048] [node 1] ICP22 created new session: [4:7574435728905909815:2048]
2025-11-19T13:59:06.624480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435728905909815:2048] [node 1] ICS09 handshake done sender: [4:7574435728905909788:2109] self: [4:7574435728905909806:0] peer: [1:7574435730303152068:0] socket: 251 qp: -1
2025-11-19T13:59:06.624485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435728905909815:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:06.624492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.625946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435729554963584:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31264
2025-11-19T13:59:06.627921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435729377077854:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.627952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435729377077854:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(29573914ns) {IncomingHandshake:261(29573171ns) {DropIncomingHandshake:441(31ns) PrepareNewSessionHandshake:476(364ns) {SwitchToState:178(52ns)} GenerateSessionVirtualId:988(191ns) IssueIncomingHandshakeReply:193(29570910ns) {SwitchToState:178(757ns)}}}
2025-11-19T13:59:06.628009Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435729377077934:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.628029Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435729377077854:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32106) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435729377077915:2108] held: no
2025-11-19T13:59:06.628054Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435729377077915:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:06.628261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435729377077927:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:06.628708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435729377077856:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.628715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435729377077854:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.628723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:06.628728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435729377077854:2048] [node 1] ICP22 created new session: [5:7574435729377077941:2048]
2025-11-19T13:59:06.628732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435729377077941:2048] [node 1] ICS09 handshake done sender: [5:7574435729377077915:2108] self: [5:7574435729377077926:0] peer: [1:7574435730303152071:0] socket: 254 qp: -1
2025-11-19T13:59:06.628739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435729377077941:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:06.628746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:06.628069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435731231831944:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
2025-11-19T13:59:06.629436Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730049600878:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.629226Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435729554963584:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.629287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435729554963497:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.630862Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435730303152076:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.630877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435730303152076:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.631777Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435731231831924:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:06.632024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435730303152061:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.632046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:06.632054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435730303152061:2107] [node 4] ICP22 created new session: [1:7574435730303152088:2107]
2025-11-19T13:59:06.632061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435730303152088:2107] [node 4] ICS09 handshake done sender: [1:7574435730303152069:2112] self: [1:7574435730303152068:0] peer: [4:7574435728905909806:0] socket: 246 qp: -1
2025-11-19T13:59:06.632067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435730303152088:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:06.632076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:06.632876Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435730303152061:2107] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.633741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435730303152092:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27261
2025-11-19T13:59:06.634421Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435730303152065:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:06.638301Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730049600864:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.638323Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435730049600864:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.640794Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435731231831942:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.640810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435731231831942:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.640612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435728905909794:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(29962084ns) {Configure:104(29960489ns) {ProcessConfigured:135(29709934ns) {StartInitialHandshake:157(29709258ns) {DropHandshakes:469(496ns) {DropIncomingHandshake:441(46ns) DropOutgoingHandshake:456(191ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:476(637ns) {SwitchToState:178(77ns)}}}}}
2025-11-19T13:59:06.640950Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730049600875:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.640964Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435730049600875:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.644018Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435730303152078:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.644058Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435730303152092:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.644238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435730303151998:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.644257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:06.644265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435730303151998:2048] [node 2] ICP22 created new session: [1:7574435730303152095:2048]
2025-11-19T13:59:06.644272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435730303152095:2048] [node 2] ICS09 handshake done sender: [1:7574435730303152065:2110] self: [1:7574435730303152064:0] peer: [2:7574435730049600879:0] socket: 245 qp: -1
2025-11-19T13:59:06.644281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435730303152095:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:06.644290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.648824Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435730049600796:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32106) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435730049600868:2112] held: no
2025-11-19T13:59:06.651532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729377077943:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5169
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.652527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435729554963586:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31264
2025-11-19T13:59:06.653348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435729554963587:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31264
2025-11-19T13:59:06.651965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435729377077927:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.652012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435729377077927:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(23620455ns) {Configure:104(23619931ns) {ProcessConfigured:135(23320909ns) {StartInitialHandshake:157(23293976ns) {DropHandshakes:469(507ns) {DropIncomingHandshake:441(170ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(366ns) PrepareNewSessionHandshake:476(663ns) {SwitchToState:178(240ns)}} IncomingHandshake:261(18156ns) {DropIncomingHandshake:441(123ns) PrepareNewSessionHandshake:476(172ns) {SwitchToState:178(129ns)} GenerateSessionVirtualId:988(225ns) IssueIncomingHandshakeReply:193(12950ns) {SwitchToState:178(59ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.654258Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435728905909800:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.654271Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435728905909800:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.654922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435728905909820:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15143
2025-11-19T13:59:06.652374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435730049600886:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19208
2025-11-19T13:59:06.652416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435730049600887:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19208
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.655236Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435729377077943:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.656201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435729377077855:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.655023Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435730303152076:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:06.656510Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730049600886:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.657324Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435728905909800:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:06.657446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435728905909792:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.657455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:06.657460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435728905909792:2112] [node 5] ICP22 created new session: [4:7574435728905909822:2112]
2025-11-19T13:59:06.657465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435728905909822:2112] [node 5] ICS09 handshake done sender: [4:7574435728905909800:2114] self: [4:7574435728905909799:0] peer: [5:7574435729377077945:0] socket: 264 qp: -1
2025-11-19T13:59:06.657469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435728905909822:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:06.657475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:06.656719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435730303152063:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.656745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:06.656751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435730303152063:2109] [node 6] ICP22 created new session: [1:7574435730303152098:2109]
2025-11-19T13:59:06.656757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435730303152098:2109] [node 6] ICS09 handshake done sender: [1:7574435730303152076:2114] self: [1:7574435730303152072:0] peer: [6:7574435729554963578:0] socket: 273 qp: -1
2025-11-19T13:59:06.656763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435730303152098:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:06.656778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:06.656455Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435731231831933:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:06.659024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435731231831929:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.659045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:06.659053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435731231831929:2108] [node 6] ICP22 created new session: [3:7574435731231831952:2108]
2025-11-19T13:59:06.659061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435731231831952:2108] [node 6] ICS09 handshake done sender: [3:7574435731231831933:2111] self: [3:7574435731231831932:0] peer: [6:7574435729554963577:0] socket: 256 qp: -1
2025-11-19T13:59:06.659069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435731231831952:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:06.659077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.660091Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435730049600868:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:06.660144Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435729554963586:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.660282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435729554963545:2090] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.660303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435729554963588:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31264
2025-11-19T13:59:06.660579Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435728905909812:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.660588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435728905909812:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.662298Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435728905909812:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:06.662427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435730049600796:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.662447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:06.662458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435730049600796:2048] [node 1] ICP22 created new session: [2:7574435730049600889:2048]
2025-11-19T13:59:06.662464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435730049600889:2048] [node 1] ICS09 handshake done sender: [2:7574435730049600868:2112] self: [2:7574435730049600879:0] peer: [1:7574435730303152064:0] socket: 248 qp: -1
2025-11-19T13:59:06.662472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435730049600889:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:06.662481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.662887Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435729554963576:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:06.662934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435729554963590:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31264
2025-11-19T13:59:06.663110Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435729554963586:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:06.663259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435728905909794:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.663289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:06.663296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435728905909794:2113] [node 6] ICP22 created new session: [4:7574435728905909825:2113]
2025-11-19T13:59:06.663301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435728905909825:2113] [node 6] ICS09 handshake done sender: [4:7574435728905909812:2119] self: [4:7574435728905909797:0] peer: [6:7574435729554963589:0] socket: 274 qp: -1
2025-11-19T13:59:06.663307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435728905909825:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:06.663313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:06.663454Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435729554963574:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.664449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435731231831955:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
2025-11-19T13:59:06.664516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435731231831956:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
2025-11-19T13:59:06.665146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435731231831957:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
2025-11-19T13:59:06.667845Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435730049600864:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:06.668353Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435731231831955:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.668420Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435731231831942:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:06.668229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729377077947:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5169
2025-11-19T13:59:06.668379Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435730049600877:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.668392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435730049600877:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.668693Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435729377077922:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:06.668760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435729377077927:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.668773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:06.668779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435729377077927:2110] [node 4] ICP22 created new session: [5:7574435729377077949:2110]
2025-11-19T13:59:06.668785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435729377077949:2110] [node 4] ICS09 handshake done sender: [5:7574435729377077922:2112] self: [5:7574435729377077945:0] peer: [4:7574435728905909799:0] socket: 266 qp: -1
2025-11-19T13:59:06.668790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435729377077949:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:06.668799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:06.669328Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435731231831957:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.669342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435731231831940:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.669352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:06.669359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435731231831940:2114] [node 5] ICP22 created new session: [3:7574435731231831959:2114]
2025-11-19T13:59:06.669365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435731231831959:2114] [node 5] ICS09 handshake done sender: [3:7574435731231831942:2115] self: [3:7574435731231831941:0] peer: [5:7574435729377077940:0] socket: 282 qp: -1
2025-11-19T13:59:06.669370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435731231831959:2114] [node 5] ICS10 traffic start
2025-11-19T13:59:06.669378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:06.672774Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435730303152063:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(16020491ns) {HandleHandshakeStatus:329(16019636ns) {DropIncomingHandshake:441(8731ns) DropOutgoingHandshake:456(241ns) SwitchToState:178(83ns) ProcessPendingSessionEvents:495(15917565ns) {ForwardSessionEventToSession:612(15902378ns) ForwardSessionEventToSession:612(1501ns) ForwardSessionEventToSession:612(821ns) ForwardSessionEventToSession:612(718ns) ForwardSessionEventToSession:612(673ns) ForwardSessionEventToSession:612(1842ns) ForwardSessionEventToSession:612(955ns) ForwardSessionEventToSession:612(514ns) ForwardSessionEventToSession:612(660ns) ForwardSessionEventToSession:612(479ns)}}}
2025-11-19T13:59:06.673124Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435731231831930:2109] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:06.673361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435729554963545:2090] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.673385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:06.673392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435729554963545:2090] [node 4] ICP22 created new session: [6:7574435729554963591:2090]
2025-11-19T13:59:06.673398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435729554963591:2090] [node 4] ICS09 handshake done sender: [6:7574435729554963586:2120] self: [6:7574435729554963589:0] peer: [4:7574435728905909797:0] socket: 295 qp: -1
2025-11-19T13:59:06.673405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435729554963591:2090] [node 4] ICS10 traffic start
2025-11-19T13:59:06.673426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:06.673263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435731231831864:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.673281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:06.673289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435731231831864:2049] [node 2] ICP22 created new session: [3:7574435731231831961:2049]
2025-11-19T13:59:06.673294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435731231831961:2049] [node 2] ICS09 handshake done sender: [3:7574435731231831930:2109] self: [3:7574435731231831950:0] peer: [2:7574435730049600860:0] socket: 250 qp: -1
2025-11-19T13:59:06.673301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435731231831961:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:06.673308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:06.673470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435731231831863:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:06.669534Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435730049600875:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.674887Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435730303152082:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:06.674896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435730303152082:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:06.675631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435728905909827:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15143
2025-11-19T13:59:06.676185Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435730303152082:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:06.676257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435730303151999:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.676271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:06.676276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435730303151999:2049] [node 3] ICP22 created new session: [1:7574435730303152101:2049]
2025-11-19T13:59:06.676281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435730303152101:2049] [node 3] ICS09 handshake done sender: [1:7574435730303152082:2117] self: [1:7574435730303152073:0] peer: [3:7574435731231831963:0] socket: 296 qp: -1
2025-11-19T13:59:06.676286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435730303152101:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:06.676292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:06.673568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435729554963496:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.673579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:06.673586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435729554963496:2048] [node 1] ICP22 created new session: [6:7574435729554963593:2048]
2025-11-19T13:59:06.673590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435729554963593:2048] [node 1] ICS09 handshake done sender: [6:7574435729554963576:2117] self: [6:7574435729554963578:0] peer: [1:7574435730303152072:0] socket: 276 qp: -1
2025-11-19T13:59:06.673596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435729554963593:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:06.673599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:06.676251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435729554963498:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.676267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:06.676275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435729554963498:2050] [node 3] ICP22 created new session: [6:7574435729554963596:2050]
2025-11-19T13:59:06.676282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435729554963596:2050] [node 3] ICS09 handshake done sender: [6:7574435729554963574:2115] self: [6:7574435729554963577:0] peer: [3:7574435731231831932:0] socket: 261 qp: -1
2025-11-19T13:59:06.676287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435729554963596:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:06.676316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:06.679649Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435728905909787:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:06.682232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435728905909727:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.682254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:06.682261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435728905909727:2049] [node 2] ICP22 created new session: [4:7574435728905909828:2049]
2025-11-19T13:59:06.682266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435728905909828:2049] [node 2] ICS09 handshake done sender: [4:7574435728905909787:2108] self: [4:7574435728905909814:0] peer: [2:7574435730049600863:0] socket: 249 qp: -1
2025-11-19T13:59:06.682274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435728905909828:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:06.682281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:06.683850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435731231831929:2108] [node 6] ICP35 event processing took too much time PendingConnection:278(24797267ns) {HandleHandshakeStatus:329(24796340ns) {DropIncomingHandshake:441(5365ns) DropOutgoingHandshake:456(189ns) SwitchToState:178(225ns) ProcessPendingSessionEvents:495(15531696ns) {ForwardSessionEventToSession:612(15516799ns) ForwardSessionEventToSession:612(2354ns) ForwardSessionEventToSession:612(844ns) ForwardSessionEventToSession:612(586ns) ForwardSessionEventToSession:612(773ns) ForwardSessionEventToSession:612(1141ns) ForwardSessionEventToSession:612(760ns) ForwardSessionEventToSession:612(567ns) ForwardSessionEventToSession:612(696ns) ForwardSessionEventToSession:612(804ns)}}}
2025-11-19T13:59:06.684146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435730049600872:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.684170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:06.684176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435730049600872:2114] [node 6] ICP22 created new session: [2:7574435730049600893:2114]
2025-11-19T13:59:06.684183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435730049600893:2114] [node 6] ICS09 handshake done sender: [2:7574435730049600875:2115] self: [2:7574435730049600874:0] peer: [6:7574435729554963585:0] socket: 292 qp: -1
2025-11-19T13:59:06.684189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435730049600893:2114] [node 6] ICS10 traffic start
2025-11-19T13:59:06.684207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.687511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729377077951:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5169
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.687675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435729377077952:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5169
2025-11-19T13:59:06.687880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435731231831940:2114] [node 5] ICP35 event processing took too much time PendingConnection:278(18513143ns) {HandleHandshakeStatus:329(18512470ns) {DropIncomingHandshake:441(113ns) DropOutgoingHandshake:456(215ns) SwitchToState:178(200ns) ProcessPendingSessionEvents:495(18442530ns) {ForwardSessionEventToSession:612(18432471ns) ForwardSessionEventToSession:612(1464ns) ForwardSessionEventToSession:612(780ns) ForwardSessionEventToSession:612(675ns) ForwardSessionEventToSession:612(483ns) ForwardSessionEventToSession:612(755ns) ForwardSessionEventToSession:612(668ns) ForwardSessionEventToSession:612(554ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(859ns)}}}
2025-11-19T13:59:06.687901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435731231831940:2114] [node 5] ICP18 handshake (actor [3:7574435731231831955:2118]) is too old
2025-11-19T13:59:06.688130Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435730049600877:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:06.688443Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435729377077943:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:06.688505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435729377077855:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.688518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:06.688523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435729377077855:2049] [node 2] ICP22 created new session: [5:7574435729377077953:2049]
2025-11-19T13:59:06.688529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435729377077953:2049] [node 2] ICS09 handshake done sender: [5:7574435729377077943:2118] self: [5:7574435729377077948:0] peer: [2:7574435730049600873:0] socket: 244 qp: -1
2025-11-19T13:59:06.688534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435729377077953:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:06.688550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:06.689578Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435729377077934:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:06.690516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435729377077856:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.690532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:06.690538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435729377077856:2050] [node 3] ICP22 created new session: [5:7574435729377077955:2050]
2025-11-19T13:59:06.690543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435729377077955:2050] [node 3] ICS09 handshake done sender: [5:7574435729377077934:2117] self: [5:7574435729377077940:0] peer: [3:7574435731231831941:0] socket: 285 qp: -1
2025-11-19T13:59:06.690548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435729377077955:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:06.690554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:06.690612Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435729377077856:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:06.686314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435728905909792:2112] [node 5] ICP35 event processing took too much time PendingConnection:278(28837028ns) {HandleHandshakeStatus:329(28836464ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(122ns) SwitchToState:178(44ns) ProcessPendingSessionEvents:495(17809223ns) {ForwardSessionEventToSession:612(17799399ns) ForwardSessionEventToSession:612(1409ns) ForwardSessionEventToSession:612(866ns) ForwardSessionEventToSession:612(519ns) ForwardSessionEventToSession:612(533ns) ForwardSessionEventToSession:612(1019ns) ForwardSessionEventToSession:612(708ns) ForwardSessionEventToSession:612(549ns) ForwardSessionEventToSession:612(673ns) ForwardSessionEventToSession:612(932ns)}}}
2025-11-19T13:59:06.692178Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435729554963496:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(18576366ns) {HandleHandshakeStatus:329(18575568ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(895ns) SwitchToState:178(58ns) ProcessPendingSessionEvents:495(18520582ns) {ForwardSessionEventToSession:612(18508216ns) ForwardSessionEventToSession:612(2016ns) ForwardSessionEventToSession:612(774ns) ForwardSessionEventToSession:612(1112ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(895ns) ForwardSessionEventToSession:612(605ns) ForwardSessionEventToSession:612(579ns) ForwardSessionEventToSession:612(1195ns) ForwardSessionEventToSession:612(938ns)}}}
2025-11-19T13:59:06.692217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435729554963545:2090] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:06.694860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435730049600871:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.694891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:06.694899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435730049600871:2113] [node 5] ICP22 created new session: [2:7574435730049600896:2113]
2025-11-19T13:59:06.694905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435730049600896:2113] [node 5] ICS09 handshake done sender: [2:7574435730049600877:2116] self: [2:7574435730049600873:0] peer: [5:7574435729377077948:0] socket: 240 qp: -1
2025-11-19T13:59:06.694912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435730049600896:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:06.694919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.696236Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435729554963497:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435729554963584:2119] held: no
2025-11-19T13:59:06.696294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435729554963595:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31264
2025-11-19T13:59:06.697356Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435729554963584:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:06.695581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435730049600862:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:06.695614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:06.695622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435730049600862:2108] [node 4] ICP22 created new session: [2:7574435730049600898:2108]
2025-11-19T13:59:06.695628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435730049600898:2108] [node 4] ICS09 handshake done sender: [2:7574435730049600864:2110] self: [2:7574435730049600863:0] peer: [4:7574435728905909814:0] socket: 243 qp: -1
2025-11-19T13:59:06.695633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435730049600898:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:06.695640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:06.696202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435730049600900:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19208
2025-11-19T13:59:06.696548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435730049600797:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:06.697531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435729554963497:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.697545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:06.697550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435729554963497:2049] [node 2] ICP22 created new session: [6:7574435729554963598:2049]
2025-11-19T13:59:06.697555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435729554963598:2049] [node 2] ICS09 handshake done sender: [6:7574435729554963584:2119] self: [6:7574435729554963585:0] peer: [2:7574435730049600874:0] socket: 293 qp: -1
2025-11-19T13:59:06.697561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435729554963598:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:06.697567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:06.700177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435730303151999:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(23885482ns) {HandleHandshakeStatus:329(23884904ns) {DropIncomingHandshake:441(2505ns) DropOutgoingHandshake:456(109ns) SwitchToState:178(104ns) ProcessPendingSessionEvents:495(23821110ns) {ForwardSessionEventToSession:612(23807386ns) ForwardSessionEventToSession:612(1039ns) ForwardSessionEventToSession:612(927ns) ForwardSessionEventToSession:612(545ns) ForwardSessionEventToSession:612(722ns) ForwardSessionEventToSession:612(1123ns) ForwardSessionEventToSession:612(641ns) ForwardSessionEventToSession:612(494ns) ForwardSessionEventToSession:612(545ns) ForwardSessionEventToSession:612(942ns)}}}
2025-11-19T13:59:06.700524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435731231831964:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
2025-11-19T13:59:06.701047Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435731231831957:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:06.701265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435731231831863:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:06.701279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:06.701285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435731231831863:2048] [node 1] ICP22 created new session: [3:7574435731231831965:2048]
2025-11-19T13:59:06.701290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435731231831965:2048] [node 1] ICS09 handshake done sender: [3:7574435731231831957:2120] self: [3:7574435731231831963:0] peer: [1:7574435730303152073:0] socket: 308 qp: -1
2025-11-19T13:59:06.701296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435731231831965:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:06.701304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:06.704363Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435730049600862:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(34832571ns) {HandleHandshakeStatus:329(34831889ns) {DropIncomingHandshake:441(13903ns) DropOutgoingHandshake:456(198ns) SwitchToState:178(160ns) ProcessPendingSessionEvents:495(29454ns) {ForwardSessionEventToSession:612(19889ns) ForwardSessionEventToSession:612(655ns) ForwardSessionEventToSession:612(1115ns) ForwardSessionEventToSession:612(550ns) ForwardSessionEventToSession:612(512ns) ForwardSessionEventToSession:612(799ns) ForwardSessionEventToSession:612(922ns) ForwardSessionEventToSession:612(596ns) ForwardSessionEventToSession:612(517ns) ForwardSessionEventToSession:612(1058ns)}}}
2025-11-19T13:59:06.705612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435729554963498:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(42056916ns) {HandleHandshakeStatus:329(42056531ns) {DropIncomingHandshake:441(637ns) DropOutgoingHandshake:456(1841ns) SwitchToState:178(236ns) ProcessPendingSessionEvents:495(9896552ns) {ForwardSessionEventToSession:612(9884902ns) ForwardSessionEventToSession:612(1514ns) ForwardSessionEventToSession:612(919ns) ForwardSessionEventToSession:612(648ns) ForwardSessionEventToSession:612(539ns) ForwardSessionEventToSession:612(918ns) ForwardSessionEventToSession:612(703ns) ForwardSessionEventToSession:612(769ns) ForwardSessionEventToSession:612(742ns) ForwardSessionEventToSession:612(1126ns)}}}
2025-11-19T13:59:06.705641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435729554963498:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:06.709632Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435728905909792:2112] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:06.716449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:06.716530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435731231831965:2048] [node 1] ICS01 socket: 308 reason# EndOfStream
2025-11-19T13:59:06.716539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435731231831863:2048] [node 1] ICP30 unregister session Session# [3:7574435731231831965:2048] VirtualId# [3:7574435731231831963:0]
2025-11-19T13:59:06.716545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435731231831965:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.716577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:06.716595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:06.719820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:06.719913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435730049600889:2048] [node 1] ICS01 socket: 248 reason# EndOfStream
2025-11-19T13:59:06.719921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435730049600796:2048] [node 1] ICP30 unregister session Session# [2:7574435730049600889:2048] VirtualId# [2:7574435730049600879:0]
2025-11-19T13:59:06.719927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435730049600889:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.719966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:06.719990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:06.721183Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:06.721257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435729377077941:2048] [node 1] ICS01 socket: 254 reason# EndOfStream
2025-11-19T13:59:06.721263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435729377077854:2048] [node 1] ICP30 unregister session Session# [5:7574435729377077941:2048] VirtualId# [5:7574435729377077926:0]
2025-11-19T13:59:06.721269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435729377077941:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.721315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:06.721338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:06.721370Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:06.721437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435729554963593:2048] [node 1] ICS07 socket disconnect 276 reason# EndOfStream
2025-11-19T13:59:06.721442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435729554963593:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.721478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:06.721504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435729554963593:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:06.723430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:06.723494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435728905909815:2048] [node 1] ICS01 socket: 251 reason# EndOfStream
2025-11-19T13:59:06.723499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435728905909726:2048] [node 1] ICP30 unregister session Session# [4:7574435728905909815:2048] VirtualId# [4:7574435728905909806:0]
2025-11-19T13:59:06.723505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435728905909815:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.723544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:06.723564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:06.725730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435729554963496:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32106) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:06.725744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435729554963593:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:06.725748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435729554963496:2048] [node 1] ICP30 unregister session Session# [6:7574435729554963593:2048] VirtualId# [6:7574435729554963578:0]
2025-11-19T13:59:06.725761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:06.725766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435729554963496:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:06.725769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:06.728370Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:06.728433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435731231831961:2049] [node 2] ICS01 socket: 250 reason# EndOfStream
2025-11-19T13:59:06.728439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435731231831864:2049] [node 2] ICP30 unregister session Session# [3:7574435731231831961:2049] VirtualId# [3:7574435731231831950:0]
2025-11-19T13:59:06.728444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435731231831961:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.728471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:06.728486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:06.735887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435729554963496:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:06.735919Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:06.735986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435729554963598:2049] [node 2] ICS01 socket: 293 reason# EndOfStream
2025-11-19T13:59:06.735992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435729554963497:2049] [node 2] ICP30 unregister session Session# [6:7574435729554963598:2049] VirtualId# [6:7574435729554963585:0]
2025-11-19T13:59:06.735997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435729554963598:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.736030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:06.736053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:06.737881Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:06.737944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435728905909828:2049] [node 2] ICS01 socket: 249 reason# EndOfStream
2025-11-19T13:59:06.737950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435728905909727:2049] [node 2] ICP30 unregister session Session# [4:7574435728905909828:2049] VirtualId# [4:7574435728905909814:0]
2025-11-19T13:59:06.737955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435728905909828:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.737990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:06.738010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:06.738039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:06.738093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435729377077953:2049] [node 2] ICS01 socket: 244 reason# EndOfStream
2025-11-19T13:59:06.738097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435729377077855:2049] [node 2] ICP30 unregister session Session# [5:7574435729377077953:2049] VirtualId# [5:7574435729377077948:0]
2025-11-19T13:59:06.738100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435729377077953:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.738122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:06.738140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:06.749908Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:06.749975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435729377077955:2050] [node 3] ICS01 socket: 285 reason# EndOfStream
2025-11-19T13:59:06.749980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435729377077856:2050] [node 3] ICP30 unregister session Session# [5:7574435729377077955:2050] VirtualId# [5:7574435729377077940:0]
2025-11-19T13:59:06.749984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435729377077955:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.750021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:06.750039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:06.752094Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:06.752160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435729554963596:2050] [node 3] ICS07 socket disconnect 261 reason# EndOfStream
2025-11-19T13:59:06.752166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435729554963596:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.752205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:06.752224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435729554963596:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:06.753223Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435729554963498:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25710) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:06.753236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435729554963596:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:06.753240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435729554963498:2050] [node 3] ICP30 unregister session Session# [6:7574435729554963596:2050] VirtualId# [6:7574435729554963577:0]
2025-11-19T13:59:06.753251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:06.753256Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435729554963498:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25710) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:06.753259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25710) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:06.756889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:06.756956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435729554963591:2090] [node 4] ICS01 socket: 295 reason# EndOfStream
2025-11-19T13:59:06.756962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435729554963545:2090] [node 4] ICP30 unregister session Session# [6:7574435729554963591:2090] VirtualId# [6:7574435729554963589:0]
2025-11-19T13:59:06.756967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435729554963591:2090] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:06.757011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:06.757030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:06.767342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435729554963498:2050] [node 3] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:06.960915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435730372663569:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:06.960932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435730372663570:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:06.995844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435729637107764:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:06.995864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435729637107765:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:07.003836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435731427634893:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:07.003852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435731427634894:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:07.031935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435734142243441:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:07.031954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435734142243442:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:07.031959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435734142243443:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:07.041475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435736133288782:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:07.041494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435736133288783:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:07.041499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435736133288784:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:07.071875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435736442975698:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:07.071902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435736442975699:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:07.071907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435736442975700:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.149899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435735722602249:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57941
2025-11-19T13:59:07.150001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435734142243500:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-19T13:59:07.148777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435734667630926:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:07.151395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435734667630935:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:07.151469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435734667630936:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:07.155328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435734142243503:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:07.155891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435733932075121:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:07.156330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435736442975744:2086] [node 4] ICP01 ready to work
2025-11-19T13:59:07.157256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435734142243509:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.159140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435736442975745:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37434
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.159763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435733932075122:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21511
2025-11-19T13:59:07.159811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435733932075129:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21511
2025-11-19T13:59:07.160057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435733932075134:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:07.160587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435736442975756:2095] [node 5] ICP01 ready to work
2025-11-19T13:59:07.160004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435736133288843:2101] [node 4] ICP01 ready to work
2025-11-19T13:59:07.160084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435736133288844:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.165047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435734667630944:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42311
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.166516Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435735722602249:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.166859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435731427634893:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:07.166887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435735722602252:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57941
2025-11-19T13:59:07.166933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435735722602253:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57941
2025-11-19T13:59:07.167688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435733932075138:2115] [node 4] ICP01 ready to work
2025-11-19T13:59:07.173800Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435734667630928:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.173814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435734667630928:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:07.173809Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435736442975745:2091] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.173943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435736442975744:2086] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.183084Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435733932075129:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.183172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435733932075139:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21511
2025-11-19T13:59:07.183616Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435735722602252:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.185690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435735722602260:2112] [node 6] ICP01 ready to work
2025-11-19T13:59:07.185974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435735722602260:2112] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:07.186459Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435736133288843:2101] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25682923ns) {Configure:104(25682215ns) {ProcessConfigured:135(25506954ns) {StartInitialHandshake:157(25506480ns) {DropHandshakes:469(164ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(21ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:476(723ns) {SwitchToState:178(240ns)}}}}}
2025-11-19T13:59:07.186994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435735722602263:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.192910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435734667630945:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42311
2025-11-19T13:59:07.193027Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435734142243500:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.193060Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435734142243505:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.193064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435734142243505:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:07.193547Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435734667630928:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:07.195858Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435733932075122:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.195935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435729637107764:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.196191Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435735722602253:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.196321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435731427634894:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:07.196342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435735722602266:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57941
2025-11-19T13:59:07.196711Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435735722602249:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:07.197612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435735722602270:2116] [node 4] ICP01 ready to work
2025-11-19T13:59:07.197661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435731427634893:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.197674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:07.197678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435731427634893:2048] [node 1] ICP22 created new session: [3:7574435735722602272:2048]
2025-11-19T13:59:07.197685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435735722602272:2048] [node 1] ICS09 handshake done sender: [3:7574435735722602249:2107] self: [3:7574435735722602256:0] peer: [1:7574435734667630927:0] socket: 248 qp: -1
2025-11-19T13:59:07.197692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435735722602272:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:07.197701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:07.196433Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435733932075139:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.196784Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435733932075126:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.196787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435733932075126:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:07.198510Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435733932075126:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:07.198570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435729637107765:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.198580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:07.198585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435729637107765:2049] [node 3] ICP22 created new session: [2:7574435733932075146:2049]
2025-11-19T13:59:07.198590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435733932075146:2049] [node 3] ICS09 handshake done sender: [2:7574435733932075126:2110] self: [2:7574435733932075125:0] peer: [3:7574435735722602267:0] socket: 260 qp: -1
2025-11-19T13:59:07.198594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435733932075146:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:07.198599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:07.193449Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435736442975741:2088] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.193469Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435736442975741:2088] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.208947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435736442975778:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37434
2025-11-19T13:59:07.209066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435730372663570:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.209090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:07.209098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435730372663570:2049] [node 3] ICP22 created new session: [1:7574435734667630947:2049]
2025-11-19T13:59:07.209107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435734667630947:2049] [node 3] ICS09 handshake done sender: [1:7574435734667630928:2108] self: [1:7574435734667630927:0] peer: [3:7574435735722602256:0] socket: 246 qp: -1
2025-11-19T13:59:07.209114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435734667630947:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:07.209124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:07.209341Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435734667630944:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.209368Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435734667630938:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.209371Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435734667630938:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:07.209824Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435734667630945:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.210018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435730372663570:2049] [node 3] ICP18 handshake (actor [1:7574435734667630945:2116]) is too old
2025-11-19T13:59:07.210389Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435736442975741:2088] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.210531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435736442975700:2050] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.210544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:07.210550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435736442975700:2050] [node 3] ICP22 created new session: [6:7574435736442975781:2050]
2025-11-19T13:59:07.210555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435736442975781:2050] [node 3] ICS09 handshake done sender: [6:7574435736442975741:2088] self: [6:7574435736442975740:0] peer: [3:7574435735722602261:0] socket: 256 qp: -1
2025-11-19T13:59:07.210559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435736442975781:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:07.210567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:07.210694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435736442975780:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37434
2025-11-19T13:59:07.211211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435736133288844:2105] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(24699732ns) {Configure:104(24698951ns) {ProcessConfigured:135(24452602ns) {StartInitialHandshake:157(24451587ns) {DropHandshakes:469(602ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(160ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:476(1189ns) {SwitchToState:178(437ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.220034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435734142243441:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:07.220677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435734142243513:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-19T13:59:07.221315Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435736442975778:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.221432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435736442975699:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:07.221449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435736442975784:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37434
2025-11-19T13:59:07.226098Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435734667630938:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:07.226410Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435734667630930:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.226419Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435734667630930:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:07.226905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435730372663569:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.226923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:07.226929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435730372663569:2048] [node 2] ICP22 created new session: [1:7574435734667630954:2048]
2025-11-19T13:59:07.226935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435734667630954:2048] [node 2] ICS09 handshake done sender: [1:7574435734667630938:2112] self: [1:7574435734667630937:0] peer: [2:7574435733932075143:0] socket: 250 qp: -1
2025-11-19T13:59:07.226941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435734667630954:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:07.226950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:07.227623Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435734142243505:2109] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:07.234234Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435734142243513:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.234251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435734142243503:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.234262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:07.234269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435734142243503:2108] [node 6] ICP22 created new session: [4:7574435734142243523:2108]
2025-11-19T13:59:07.234227Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435733932075136:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.234241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435733932075136:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:07.234274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435734142243523:2108] [node 6] ICS09 handshake done sender: [4:7574435734142243505:2109] self: [4:7574435734142243504:0] peer: [6:7574435736442975772:0] socket: 251 qp: -1
2025-11-19T13:59:07.234280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435734142243523:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:07.234287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.242691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435736133288845:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43119
2025-11-19T13:59:07.242746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435736133288846:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43119
2025-11-19T13:59:07.243170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435736133288857:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43119
2025-11-19T13:59:07.243181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435736133288858:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43119
2025-11-19T13:59:07.241687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435734142243503:2108] [node 6] ICP18 handshake (actor [4:7574435734142243513:2113]) is too old
2025-11-19T13:59:07.242366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435734667630952:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42311
2025-11-19T13:59:07.242430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435734667630953:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42311
2025-11-19T13:59:07.244925Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435734667630930:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:07.245067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435734667630926:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.245081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:07.245087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435734667630926:2107] [node 4] ICP22 created new session: [1:7574435734667630958:2107]
2025-11-19T13:59:07.245093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435734667630958:2107] [node 4] ICS09 handshake done sender: [1:7574435734667630930:2109] self: [1:7574435734667630929:0] peer: [4:7574435734142243517:0] socket: 247 qp: -1
2025-11-19T13:59:07.245099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435734667630958:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:07.245108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.248847Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435729637107764:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18341) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435733932075122:2108] held: no
2025-11-19T13:59:07.249079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435733932075149:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21511
2025-11-19T13:59:07.251974Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435734667630952:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.252092Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435736442975745:2091] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:07.254969Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435736442975744:2086] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19498) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435736442975745:2091] held: no
2025-11-19T13:59:07.254987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435736442975744:2086] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.254997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:07.255005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435736442975744:2086] [node 4] ICP22 created new session: [6:7574435736442975786:2086]
2025-11-19T13:59:07.255010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435736442975786:2086] [node 4] ICS09 handshake done sender: [6:7574435736442975745:2091] self: [6:7574435736442975772:0] peer: [4:7574435734142243504:0] socket: 253 qp: -1
2025-11-19T13:59:07.255016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435736442975786:2086] [node 4] ICS10 traffic start
2025-11-19T13:59:07.255024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:07.255196Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435736442975780:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.255334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435736442975788:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37434
2025-11-19T13:59:07.255352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435736442975698:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:07.252126Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435736133288846:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.255657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435736133288783:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:07.255960Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435734667630940:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.255969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435734667630940:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.257688Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435734667630940:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:07.257784Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435734667630953:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.257794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435734667630935:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.257807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:07.257813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435734667630935:2110] [node 6] ICP22 created new session: [1:7574435734667630962:2110]
2025-11-19T13:59:07.257817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435734667630962:2110] [node 6] ICS09 handshake done sender: [1:7574435734667630940:2113] self: [1:7574435734667630939:0] peer: [6:7574435736442975790:0] socket: 280 qp: -1
2025-11-19T13:59:07.257822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435734667630962:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:07.257827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:07.257004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435734142243526:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-19T13:59:07.257584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435734142243527:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-19T13:59:07.257627Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435734142243500:2106] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:07.259258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435734142243441:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.259269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:07.259275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435734142243441:2048] [node 1] ICP22 created new session: [4:7574435734142243528:2048]
2025-11-19T13:59:07.259280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435734142243528:2048] [node 1] ICS09 handshake done sender: [4:7574435734142243500:2106] self: [4:7574435734142243517:0] peer: [1:7574435734667630929:0] socket: 249 qp: -1
2025-11-19T13:59:07.259286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435734142243528:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:07.259291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.264002Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435733932075128:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.264015Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435733932075128:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:07.264724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435733932075151:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21511
2025-11-19T13:59:07.264849Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435733932075122:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:07.265810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435734667630926:2107] [node 4] ICP18 handshake (actor [1:7574435734667630953:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.266155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435734142243530:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.266317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435736442975698:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18341) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435736442975780:2117] held: no
2025-11-19T13:59:07.266388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435736442975791:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37434
2025-11-19T13:59:07.266538Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435736442975780:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:07.266602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435736442975698:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.266615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:07.266620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435736442975698:2048] [node 1] ICP22 created new session: [6:7574435736442975792:2048]
2025-11-19T13:59:07.266626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435736442975792:2048] [node 1] ICS09 handshake done sender: [6:7574435736442975780:2117] self: [6:7574435736442975790:0] peer: [1:7574435734667630939:0] socket: 281 qp: -1
2025-11-19T13:59:07.266632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435736442975792:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:07.266638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.267626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435735722602271:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57941
2025-11-19T13:59:07.267938Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435731427634894:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1876) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435735722602253:2110] held: no
2025-11-19T13:59:07.269329Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435735722602253:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:07.269561Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435733932075136:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.269660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435729637107764:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.269675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:07.269680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435729637107764:2048] [node 1] ICP22 created new session: [2:7574435733932075155:2048]
2025-11-19T13:59:07.269686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435733932075155:2048] [node 1] ICS09 handshake done sender: [2:7574435733932075122:2108] self: [2:7574435733932075143:0] peer: [1:7574435734667630937:0] socket: 252 qp: -1
2025-11-19T13:59:07.269690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435733932075155:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:07.269695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:07.269785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435733932075153:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21511
2025-11-19T13:59:07.269825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435733932075134:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.269869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:07.269873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435733932075134:2113] [node 6] ICP22 created new session: [2:7574435733932075157:2113]
2025-11-19T13:59:07.269877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435733932075157:2113] [node 6] ICS09 handshake done sender: [2:7574435733932075136:2114] self: [2:7574435733932075135:0] peer: [6:7574435736442975785:0] socket: 270 qp: -1
2025-11-19T13:59:07.269880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435733932075157:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:07.269885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:07.271466Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435733932075128:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.271639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435731427634894:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.271652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:07.271656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435731427634894:2049] [node 2] ICP22 created new session: [3:7574435735722602279:2049]
2025-11-19T13:59:07.271661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435735722602279:2049] [node 2] ICS09 handshake done sender: [3:7574435735722602253:2110] self: [3:7574435735722602267:0] peer: [2:7574435733932075125:0] socket: 267 qp: -1
2025-11-19T13:59:07.271665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435735722602279:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:07.271670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:07.271787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435735722602276:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57941
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.271997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435735722602277:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57941
2025-11-19T13:59:07.272066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435735722602278:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57941
2025-11-19T13:59:07.272114Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435733932075153:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.272206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435733932075121:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.272213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:07.272217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435733932075121:2107] [node 5] ICP22 created new session: [2:7574435733932075159:2107]
2025-11-19T13:59:07.272222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435733932075159:2107] [node 5] ICS09 handshake done sender: [2:7574435733932075128:2111] self: [2:7574435733932075127:0] peer: [5:7574435736133288864:0] socket: 261 qp: -1
2025-11-19T13:59:07.272226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435733932075159:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:07.272232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:07.272349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435734142243530:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.272376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435734142243442:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:07.272830Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435736442975699:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1876) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435736442975778:2116] held: no
2025-11-19T13:59:07.272867Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435736442975778:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:07.275917Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435733932075141:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.275926Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435733932075141:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:07.276668Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435736133288858:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.276718Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435736133288845:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.280139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435734142243533:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.281708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435736133288865:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43119
2025-11-19T13:59:07.281745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435736133288866:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43119
2025-11-19T13:59:07.281866Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435734142243527:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.282122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435736133288782:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:07.282137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435736133288843:2101] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:07.282155Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435736133288846:2109] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:07.282277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435736133288783:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.282293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:07.282297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435736133288783:2049] [node 2] ICP22 created new session: [5:7574435736133288869:2049]
2025-11-19T13:59:07.282302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435736133288869:2049] [node 2] ICS09 handshake done sender: [5:7574435736133288846:2109] self: [5:7574435736133288864:0] peer: [2:7574435733932075127:0] socket: 263 qp: -1
2025-11-19T13:59:07.282308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435736133288869:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:07.282315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:07.283243Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435734667630942:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.283259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435734667630942:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:07.283294Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435734142243511:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.283302Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435734142243511:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:07.283418Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435733932075141:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:07.284145Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435735722602278:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.284220Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435735722602277:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.284254Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435735722602252:2109] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:07.287823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435735722602260:2112] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.287840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:07.287846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435735722602260:2112] [node 6] ICP22 created new session: [3:7574435735722602282:2112]
2025-11-19T13:59:07.287851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435735722602282:2112] [node 6] ICS09 handshake done sender: [3:7574435735722602252:2109] self: [3:7574435735722602261:0] peer: [6:7574435736442975740:0] socket: 266 qp: -1
2025-11-19T13:59:07.287855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435735722602282:2112] [node 6] ICS10 traffic start
2025-11-19T13:59:07.287861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:07.287893Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435734667630942:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:07.287419Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435733932075151:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.287542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435733932075138:2115] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.287556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:07.287562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435733932075138:2115] [node 4] ICP22 created new session: [2:7574435733932075163:2115]
2025-11-19T13:59:07.287568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435733932075163:2115] [node 4] ICS09 handshake done sender: [2:7574435733932075141:2117] self: [2:7574435733932075140:0] peer: [4:7574435734142243532:0] socket: 298 qp: -1
2025-11-19T13:59:07.287574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435733932075163:2115] [node 4] ICS10 traffic start
2025-11-19T13:59:07.287582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:07.288354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435734667630936:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.288368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:07.288374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435734667630936:2111] [node 5] ICP22 created new session: [1:7574435734667630965:2111]
2025-11-19T13:59:07.288379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435734667630965:2111] [node 5] ICS09 handshake done sender: [1:7574435734667630942:2114] self: [1:7574435734667630941:0] peer: [5:7574435736133288867:0] socket: 264 qp: -1
2025-11-19T13:59:07.288385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435734667630965:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:07.288393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.288279Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435734142243530:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:07.288660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435734142243442:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.288668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:07.288674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435734142243442:2049] [node 2] ICP22 created new session: [4:7574435734142243537:2049]
2025-11-19T13:59:07.288678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435734142243537:2049] [node 2] ICS09 handshake done sender: [4:7574435734142243530:2119] self: [4:7574435734142243532:0] peer: [2:7574435733932075140:0] socket: 300 qp: -1
2025-11-19T13:59:07.288683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435734142243537:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:07.288688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:07.288815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435734142243535:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-19T13:59:07.303244Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435734142243511:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.303290Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435736133288857:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.303496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435736133288871:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43119
2025-11-19T13:59:07.303565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435736442975699:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.303586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:07.303592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435736442975699:2049] [node 2] ICP22 created new session: [6:7574435736442975794:2049]
2025-11-19T13:59:07.303598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435736442975794:2049] [node 2] ICS09 handshake done sender: [6:7574435736442975778:2116] self: [6:7574435736442975785:0] peer: [2:7574435733932075135:0] socket: 277 qp: -1
2025-11-19T13:59:07.303604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435736442975794:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:07.303611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:07.304112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435733932075121:2107] [node 5] ICP18 handshake (actor [2:7574435733932075151:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.304399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435736442975796:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37434
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.304563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435736133288872:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43119
2025-11-19T13:59:07.304787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435736133288784:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:07.304831Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435736133288865:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.304882Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435736133288858:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:07.304949Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435736133288845:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:07.305028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435736133288782:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.305041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:07.305046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435736133288782:2048] [node 1] ICP22 created new session: [5:7574435736133288875:2048]
2025-11-19T13:59:07.305050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435736133288875:2048] [node 1] ICS09 handshake done sender: [5:7574435736133288858:2117] self: [5:7574435736133288867:0] peer: [1:7574435734667630941:0] socket: 288 qp: -1
2025-11-19T13:59:07.305054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435736133288875:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:07.305059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:07.305173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435736133288843:2101] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.305178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:07.305181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435736133288843:2101] [node 4] ICP22 created new session: [5:7574435736133288877:2101]
2025-11-19T13:59:07.305186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435736133288877:2101] [node 4] ICS09 handshake done sender: [5:7574435736133288845:2108] self: [5:7574435736133288868:0] peer: [4:7574435734142243510:0] socket: 262 qp: -1
2025-11-19T13:59:07.305189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435736133288877:2101] [node 4] ICS10 traffic start
2025-11-19T13:59:07.305193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:07.305980Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435735722602265:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.305994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435735722602265:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.307907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435736133288880:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43119
2025-11-19T13:59:07.308147Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435735722602265:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:07.308263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435735722602263:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.308278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:07.308283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435735722602263:2113] [node 5] ICP22 created new session: [3:7574435735722602287:2113]
2025-11-19T13:59:07.308288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435735722602287:2113] [node 5] ICS09 handshake done sender: [3:7574435735722602265:2114] self: [3:7574435735722602264:0] peer: [5:7574435736133288874:0] socket: 274 qp: -1
2025-11-19T13:59:07.308293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435735722602287:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:07.308298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:07.308623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435734142243509:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.308639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:07.308644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435734142243509:2111] [node 5] ICP22 created new session: [4:7574435734142243539:2111]
2025-11-19T13:59:07.308649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435734142243539:2111] [node 5] ICS09 handshake done sender: [4:7574435734142243511:2112] self: [4:7574435734142243510:0] peer: [5:7574435736133288868:0] socket: 254 qp: -1
2025-11-19T13:59:07.308652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435734142243539:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:07.308658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:07.316176Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435734142243535:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.316208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435734142243443:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:07.316320Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435735722602275:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.316331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435735722602275:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.316944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435734142243542:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10828
2025-11-19T13:59:07.317166Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435736442975796:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.317198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435736442975756:2095] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:07.318305Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435735722602275:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:07.318489Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435734142243535:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:07.318554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435734142243443:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.318572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:07.318578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435734142243443:2050] [node 3] ICP22 created new session: [4:7574435734142243543:2050]
2025-11-19T13:59:07.318585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435734142243543:2050] [node 3] ICS09 handshake done sender: [4:7574435734142243535:2121] self: [4:7574435734142243541:0] peer: [3:7574435735722602274:0] socket: 305 qp: -1
2025-11-19T13:59:07.318590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435734142243543:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:07.318598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:07.320002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435735722602270:2116] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.320019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:07.320027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435735722602270:2116] [node 4] ICP22 created new session: [3:7574435735722602290:2116]
2025-11-19T13:59:07.320033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435735722602290:2116] [node 4] ICS09 handshake done sender: [3:7574435735722602275:2118] self: [3:7574435735722602274:0] peer: [4:7574435734142243541:0] socket: 304 qp: -1
2025-11-19T13:59:07.320038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435735722602290:2116] [node 4] ICS10 traffic start
2025-11-19T13:59:07.320045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:07.320581Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435736133288856:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:07.320592Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435736133288856:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:07.321208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435736133288784:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21986) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435736133288857:2116] held: no
2025-11-19T13:59:07.321243Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435736133288857:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:07.321438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435736133288784:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.321450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:07.321454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435736133288784:2050] [node 3] ICP22 created new session: [5:7574435736133288882:2050]
2025-11-19T13:59:07.321458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435736133288882:2050] [node 3] ICS09 handshake done sender: [5:7574435736133288857:2116] self: [5:7574435736133288874:0] peer: [3:7574435735722602264:0] socket: 283 qp: -1
2025-11-19T13:59:07.321463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435736133288882:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:07.321468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:07.321690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435736442975798:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37434
2025-11-19T13:59:07.321839Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435736133288856:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:07.322007Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435736442975796:2121] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:07.322066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435736442975756:2095] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:07.322078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:07.322082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435736442975756:2095] [node 5] ICP22 created new session: [6:7574435736442975799:2095]
2025-11-19T13:59:07.322087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435736442975799:2095] [node 5] ICS09 handshake done sender: [6:7574435736442975796:2121] self: [6:7574435736442975797:0] peer: [5:7574435736133288855:0] socket: 286 qp: -1
2025-11-19T13:59:07.322092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435736442975799:2095] [node 5] ICS10 traffic start
2025-11-19T13:59:07.322098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:07.323066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435736133288844:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:07.323081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:07.323086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435736133288844:2105] [node 6] ICP22 created new session: [5:7574435736133288884:2105]
2025-11-19T13:59:07.323090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435736133288884:2105] [node 6] ICS09 handshake done sender: [5:7574435736133288856:2115] self: [5:7574435736133288855:0] peer: [6:7574435736442975797:0] socket: 308 qp: -1
2025-11-19T13:59:07.323094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435736133288884:2105] [node 6] ICS10 traffic start
2025-11-19T13:59:07.323100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:07.325704Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:07.325778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435733932075155:2048] [node 1] ICS01 socket: 252 reason# EndOfStream
2025-11-19T13:59:07.325785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435729637107764:2048] [node 1] ICP30 unregister session Session# [2:7574435733932075155:2048] VirtualId# [2:7574435733932075143:0]
2025-11-19T13:59:07.325789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435733932075155:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:07.325814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:07.325832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:07.328063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:07.328122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435735722602272:2048] [node 1] ICS01 socket: 248 reason# EndOfStream
2025-11-19T13:59:07.328136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435731427634893:2048] [node 1] ICP30 unregister session Session# [3:7574435735722602272:2048] VirtualId# [3:7574435735722602256:0]
2025-11-19T13:59:07.328141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435735722602272:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:07.328165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:07.328181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:07.338572Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:07.338653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435734142243528:2048] [node 1] ICS01 socket: 249 reason# EndOfStream
2025-11-19T13:59:07.338661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435734142243441:2048] [node 1] ICP30 unregister session Session# [4:7574435734142243528:2048] VirtualId# [4:7574435734142243517:0]
2025-11-19T13:59:07.338667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435734142243528:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:07.338706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:07.338730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:07.338759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:07.338815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435736442975792:2048] [node 1] ICS01 socket: 281 reason# EndOfStream
2025-11-19T13:59:07.338819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435736442975698:2048] [node 1] ICP30 unregister session Session# [6:7574435736442975792:2048] VirtualId# [6:7574435736442975790:0]
2025-11-19T13:59:07.338823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435736442975792:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:07.338846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:07.338865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:07.338887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:07.338939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435736133288875:2048] [node 1] ICS01 socket: 288 reason# EndOfStream
2025-11-19T13:59:07.338943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435736133288782:2048] [node 1] ICP30 unregister session Session# [5:7574435736133288875:2048] VirtualId# [5:7574435736133288867:0]
2025-11-19T13:59:07.338946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435736133288875:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:07.338972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:07.338988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:07.344251Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:07.344354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435735722602279:2049] [node 2] ICS01 socket: 267 reason# EndOfStream
2025-11-19T13:59:07.344361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435731427634894:2049] [node 2] ICP30 unregister session Session# [3:7574435735722602279:2049] VirtualId# [3:7574435735722602267:0]
2025-11-19T13:59:07.344365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435735722602279:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:07.344394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:07.344410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:07.348210Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:07.348286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435736442975794:2049] [node 2] ICS01 socket: 277 reason# EndOfStream
2025-11-19T13:59:07.348293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435736442975699:2049] [node 2] ICP30 unregister session Session# [6:7574435736442975794:2049] VirtualId# [6:7574435736442975785:0]
2025-11-19T13:59:07.348299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435736442975794:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:07.348343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:07.348368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:07.348395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:07.348451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435734142243537:2049] [node 2] ICS01 socket: 300 reason# EndOfStream
2025-11-19T13:59:07.348455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435734142243442:2049] [node 2] ICP30 unregister session Session# [4:7574435734142243537:2049] VirtualId# [4:7574435734142243532:0]
2025-11-19T13:59:07.348459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435734142243537:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:07.348479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:07.348496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:07.350469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:07.350537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435736133288869:2049] [node 2] ICS01 socket: 263 reason# EndOfStream
2025-11-19T13:59:07.350542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435736133288783:2049] [node 2] ICP30 unregister session Session# [5:7574435736133288869:2049] VirtualId# [5:7574435736133288864:0]
2025-11-19T13:59:07.350546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435736133288869:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:07.350587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:07.350608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:07.360172Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:07.360222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435736133288882:2050] [node 3] ICS01 socket: 283 reason# EndOfStream
2025-11-19T13:59:07.360226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435736133288784:2050] [node 3] ICP30 unregister session Session# [5:7574435736133288882:2050] VirtualId# [5:7574435736133288874:0]
2025-11-19T13:59:07.360229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435736133288882:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:07.360248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:07.360267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:07.360002Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:07.360075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435736442975781:2050] [node 3] ICS01 socket: 256 reason# EndOfStream
2025-11-19T13:59:07.360082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435736442975700:2050] [node 3] ICP30 unregister session Session# [6:7574435736442975781:2050] VirtualId# [6:7574435736442975740:0]
2025-11-19T13:59:07.360087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435736442975781:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:07.360121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:07.360148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:07.370063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:07.370158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435736442975786:2086] [node 4] ICS01 socket: 253 reason# EndOfStream
2025-11-19T13:59:07.370167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435736442975744:2086] [node 4] ICP30 unregister session Session# [6:7574435736442975786:2086] VirtualId# [6:7574435736442975772:0]
2025-11-19T13:59:07.370173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435736442975786:2086] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:07.370215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:07.370237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:07.612093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435736582986203:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:07.612107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435736582986204:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:07.676266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435732960195573:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:07.676306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435732960195574:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:07.722518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435736161878934:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:07.722534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435736161878935:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:07.793186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435734999595951:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:07.793202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435734999595952:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:07.793207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435734999595953:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:07.891375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435735721575679:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:07.891396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435735721575680:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:07.891401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435735721575681:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:07.995888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435735956348896:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:07.995901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435735956348897:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:07.995905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435735956348898:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.058384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435739294563308:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19477
2025-11-19T13:59:08.057460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435740251316217:2071] [node 4] ICP01 ready to work
2025-11-19T13:59:08.059939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435740877953562:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:08.059949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435740877953563:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:08.060047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435740877953564:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:08.060139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435740251316232:2078] [node 5] ICP01 ready to work
2025-11-19T13:59:08.061468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435737255162932:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.062631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435740877953574:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23352
2025-11-19T13:59:08.060089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435740016543037:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.066675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435740016543039:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.083869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435740016543037:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(23703613ns) {Configure:104(23702876ns) {ProcessConfigured:135(23316437ns) {StartInitialHandshake:157(23315824ns) {DropHandshakes:469(613ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(261ns) PrepareNewSessionHandshake:476(1782ns) {SwitchToState:178(140ns)}}}}}
2025-11-19T13:59:08.084350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435737255162940:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:08.086052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435737255162939:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7225
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.111126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435740016543042:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13889
2025-11-19T13:59:08.109280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435740456846295:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:08.109810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435740456846298:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:08.110494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435740456846294:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4178
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.111854Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435736582986204:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(50051461ns) {Configure:104(50050916ns) {ProcessConfigured:135(49850478ns) {StartInitialHandshake:157(49850085ns) {DropHandshakes:469(201ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:476(1587ns) {SwitchToState:178(780ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.112303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435739294563318:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:08.112346Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435739294563308:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.113145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435739294563315:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19477
2025-11-19T13:59:08.113156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435739294563316:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19477
2025-11-19T13:59:08.113202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435739294563317:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19477
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.113452Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435732960195574:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(53558331ns) {Configure:104(53556595ns) {ProcessConfigured:135(51454668ns) {StartInitialHandshake:157(51454006ns) {DropHandshakes:469(334ns) {DropIncomingHandshake:441(32ns) DropOutgoingHandshake:456(30ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:476(849ns) {SwitchToState:178(435ns)}}}}}
2025-11-19T13:59:08.113571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435737255162945:2113] [node 5] ICP01 ready to work
2025-11-19T13:59:08.113643Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435737255162939:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.114039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435732960195573:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.114056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435737255162946:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7225
2025-11-19T13:59:08.114262Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435740016543039:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.114336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435740016543043:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13889
2025-11-19T13:59:08.114633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435735721575679:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.111381Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435740456846294:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.115837Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435740877953574:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.116918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435739294563316:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.120113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435740016543046:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:08.119007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435739294563313:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:08.123873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435740251316232:2078] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(63672938ns) {Configure:104(63672313ns) {ProcessConfigured:135(63428763ns) {StartInitialHandshake:157(63428287ns) {DropHandshakes:469(407ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(76ns)} GenerateSessionVirtualId:988(142ns) PrepareNewSessionHandshake:476(1213ns) {SwitchToState:178(899ns)}}}}}
2025-11-19T13:59:08.124417Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435740877953568:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.124437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435740877953568:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.124711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435740456846292:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.129425Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435740877953571:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.129437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435740877953571:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.130086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435740877953579:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23352
2025-11-19T13:59:08.130127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435740877953580:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23352
2025-11-19T13:59:08.130158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435740877953581:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23352
2025-11-19T13:59:08.130284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435740251316270:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:622
2025-11-19T13:59:08.130302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435740251316271:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:622
2025-11-19T13:59:08.131038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435734999595951:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.132048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435740016543050:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.132890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435737255162951:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7225
2025-11-19T13:59:08.133865Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435739294563315:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.133888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435734999595952:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.134040Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435740877953571:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:08.138700Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435740251316270:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.138739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435735956348897:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.148959Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435740877953566:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.148973Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435740877953566:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.149621Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435740877953579:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.149638Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435740877953580:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.149647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435740877953563:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.149661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:08.149669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435740877953563:2108] [node 5] ICP22 created new session: [1:7574435740877953584:2108]
2025-11-19T13:59:08.149674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435740877953584:2108] [node 5] ICS09 handshake done sender: [1:7574435740877953571:2112] self: [1:7574435740877953570:0] peer: [5:7574435740016543044:0] socket: 261 qp: -1
2025-11-19T13:59:08.149682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435740877953584:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:08.149692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.150205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435740456846311:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4178
2025-11-19T13:59:08.150706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435739294563332:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19477
2025-11-19T13:59:08.150949Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435737255162934:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.150962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435737255162934:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.151057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435737255162942:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.151062Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435737255162942:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.151376Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435740456846311:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.151484Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435740877953568:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:08.154282Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435740877953566:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:08.154426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435736582986203:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.154448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:08.154455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435736582986203:2048] [node 2] ICP22 created new session: [1:7574435740877953591:2048]
2025-11-19T13:59:08.154461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435740877953591:2048] [node 2] ICS09 handshake done sender: [1:7574435740877953568:2111] self: [1:7574435740877953567:0] peer: [2:7574435737255162950:0] socket: 260 qp: -1
2025-11-19T13:59:08.154468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435740877953591:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:08.154487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:08.154610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435740877953562:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.154619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:08.154624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435740877953562:2107] [node 4] ICP22 created new session: [1:7574435740877953593:2107]
2025-11-19T13:59:08.154628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435740877953593:2107] [node 4] ICS09 handshake done sender: [1:7574435740877953566:2110] self: [1:7574435740877953565:0] peer: [4:7574435739294563327:0] socket: 259 qp: -1
2025-11-19T13:59:08.154631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435740877953593:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:08.154635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:08.158671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435734999595951:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435739294563316:2113] held: no
2025-11-19T13:59:08.158717Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435739294563316:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:08.158808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435734999595951:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.158821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:08.158826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435734999595951:2048] [node 1] ICP22 created new session: [4:7574435739294563333:2048]
2025-11-19T13:59:08.158831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435739294563333:2048] [node 1] ICS09 handshake done sender: [4:7574435739294563316:2113] self: [4:7574435739294563327:0] peer: [1:7574435740877953565:0] socket: 273 qp: -1
2025-11-19T13:59:08.158838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435739294563333:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:08.158846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.159965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435740251316273:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:622
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.166254Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435732960195573:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435737255162939:2111] held: no
2025-11-19T13:59:08.166459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435737255162952:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7225
2025-11-19T13:59:08.170166Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435740016543039:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:08.170992Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435740877953581:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.173732Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435740251316273:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.173767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435735956348896:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.179845Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435740251316271:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.179890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435740251316232:2078] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.179604Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435739294563317:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.179647Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435739294563322:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.179652Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435739294563322:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.174652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435735721575681:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(41251461ns) {Configure:104(41250006ns) {ProcessConfigured:135(40992530ns) {StartInitialHandshake:157(40991921ns) {DropHandshakes:469(429ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(45ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:476(1157ns) {SwitchToState:178(1088ns)}}}}}
2025-11-19T13:59:08.174717Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435740016543042:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.174799Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435740016543043:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.174805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435735721575679:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.174816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:08.174833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435735721575679:2048] [node 1] ICP22 created new session: [5:7574435740016543056:2048]
2025-11-19T13:59:08.174838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435740016543056:2048] [node 1] ICS09 handshake done sender: [5:7574435740016543039:2109] self: [5:7574435740016543044:0] peer: [1:7574435740877953570:0] socket: 264 qp: -1
2025-11-19T13:59:08.174845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435740016543056:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:08.174852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:08.175096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435735721575681:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.175106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435740016543037:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.183961Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435740456846300:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.183981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435740456846300:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.184150Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435740877953573:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.184171Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435740877953573:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.185031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435739294563336:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19477
2025-11-19T13:59:08.184226Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435740016543048:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.184239Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435740016543048:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.185190Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435740016543048:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:08.185275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435740016543046:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.185288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:08.185296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435740016543046:2113] [node 6] ICP22 created new session: [5:7574435740016543061:2113]
2025-11-19T13:59:08.185302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435740016543061:2113] [node 6] ICS09 handshake done sender: [5:7574435740016543048:2114] self: [5:7574435740016543047:0] peer: [6:7574435740251316275:0] socket: 281 qp: -1
2025-11-19T13:59:08.185308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435740016543061:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:08.185316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.187942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435737255162953:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7225
2025-11-19T13:59:08.188042Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435737255162934:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:08.188766Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435737255162942:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:08.188891Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435737255162946:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.188924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435737255162932:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.188934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:08.188940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435737255162932:2107] [node 4] ICP22 created new session: [2:7574435737255162956:2107]
2025-11-19T13:59:08.188945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435737255162956:2107] [node 4] ICS09 handshake done sender: [2:7574435737255162934:2109] self: [2:7574435737255162933:0] peer: [4:7574435739294563331:0] socket: 256 qp: -1
2025-11-19T13:59:08.188950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435737255162956:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:08.188957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:08.189060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435737255162940:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.189065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:08.189070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435737255162940:2110] [node 6] ICP22 created new session: [2:7574435737255162958:2110]
2025-11-19T13:59:08.189073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435737255162958:2110] [node 6] ICS09 handshake done sender: [2:7574435737255162942:2112] self: [2:7574435737255162941:0] peer: [6:7574435740251316272:0] socket: 278 qp: -1
2025-11-19T13:59:08.189076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435737255162958:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:08.189080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:08.189143Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435737255162939:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:08.189383Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435737255162951:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.189390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435732960195573:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.189396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:08.189399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435732960195573:2048] [node 1] ICP22 created new session: [2:7574435737255162960:2048]
2025-11-19T13:59:08.189402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435737255162960:2048] [node 1] ICS09 handshake done sender: [2:7574435737255162939:2111] self: [2:7574435737255162950:0] peer: [1:7574435740877953567:0] socket: 265 qp: -1
2025-11-19T13:59:08.189405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435737255162960:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:08.189411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.189785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435740016543064:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.189913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435740251316276:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:622
2025-11-19T13:59:08.190027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435740251316277:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:622
2025-11-19T13:59:08.189950Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435739294563315:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:08.190004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435734999595952:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.190012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:08.190017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435734999595952:2049] [node 2] ICP22 created new session: [4:7574435739294563338:2049]
2025-11-19T13:59:08.190022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435739294563338:2049] [node 2] ICS09 handshake done sender: [4:7574435739294563315:2112] self: [4:7574435739294563331:0] peer: [2:7574435737255162933:0] socket: 257 qp: -1
2025-11-19T13:59:08.190027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435739294563338:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:08.190032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:08.190235Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435740251316271:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:08.190765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435740251316232:2078] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.190781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:08.190786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435740251316232:2078] [node 5] ICP22 created new session: [6:7574435740251316278:2078]
2025-11-19T13:59:08.190791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435740251316278:2078] [node 5] ICS09 handshake done sender: [6:7574435740251316271:2116] self: [6:7574435740251316275:0] peer: [5:7574435740016543047:0] socket: 286 qp: -1
2025-11-19T13:59:08.190796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435740251316278:2078] [node 5] ICS10 traffic start
2025-11-19T13:59:08.190802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:08.191346Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435740251316270:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:08.192806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435735956348897:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.192825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:08.192831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435735956348897:2049] [node 2] ICP22 created new session: [6:7574435740251316282:2049]
2025-11-19T13:59:08.192838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435740251316282:2049] [node 2] ICS09 handshake done sender: [6:7574435740251316270:2115] self: [6:7574435740251316272:0] peer: [2:7574435737255162941:0] socket: 280 qp: -1
2025-11-19T13:59:08.192843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435740251316282:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:08.192850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.191028Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435737255162953:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.193528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435740456846313:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4178
2025-11-19T13:59:08.193575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435740456846314:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4178
2025-11-19T13:59:08.193600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435740456846315:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4178
2025-11-19T13:59:08.193782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435737255162932:2107] [node 4] ICP18 handshake (actor [2:7574435737255162953:2119]) is too old
2025-11-19T13:59:08.194160Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435739294563322:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:08.195409Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435740877953573:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:08.195529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435740877953564:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.195545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:08.195551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435740877953564:2109] [node 6] ICP22 created new session: [1:7574435740877953598:2109]
2025-11-19T13:59:08.195556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435740877953598:2109] [node 6] ICS09 handshake done sender: [1:7574435740877953573:2113] self: [1:7574435740877953572:0] peer: [6:7574435740251316274:0] socket: 292 qp: -1
2025-11-19T13:59:08.195561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435740877953598:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:08.195568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.200031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435740016543065:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.200101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435737255162966:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7225
2025-11-19T13:59:08.200145Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435740456846300:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.200198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435740016543066:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13889
2025-11-19T13:59:08.200351Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435740016543042:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:08.200270Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435740456846313:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.200295Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435740456846314:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.200314Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435740456846315:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.200321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435740456846295:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.200329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:08.200341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435740456846295:2108] [node 5] ICP22 created new session: [3:7574435740456846317:2108]
2025-11-19T13:59:08.200346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435740456846317:2108] [node 5] ICS09 handshake done sender: [3:7574435740456846300:2112] self: [3:7574435740456846299:0] peer: [5:7574435740016543058:0] socket: 270 qp: -1
2025-11-19T13:59:08.200352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435740456846317:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:08.200359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:08.200412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435735721575681:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.200424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:08.200430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435735721575681:2050] [node 3] ICP22 created new session: [5:7574435740016543067:2050]
2025-11-19T13:59:08.200434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435740016543067:2050] [node 3] ICS09 handshake done sender: [5:7574435740016543042:2111] self: [5:7574435740016543058:0] peer: [3:7574435740456846299:0] socket: 272 qp: -1
2025-11-19T13:59:08.200449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435740016543067:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:08.200456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:08.200584Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435740016543043:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:08.201082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435736161878934:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.201099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435736161878935:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.201890Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435740877953578:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.201903Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435740877953578:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.202771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435739294563318:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.202791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:08.202799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435739294563318:2111] [node 5] ICP22 created new session: [4:7574435739294563342:2111]
2025-11-19T13:59:08.202805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435739294563342:2111] [node 5] ICS09 handshake done sender: [4:7574435739294563322:2116] self: [4:7574435739294563321:0] peer: [5:7574435740016543059:0] socket: 276 qp: -1
2025-11-19T13:59:08.202810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435739294563342:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:08.202817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:08.201842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435740251316281:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:622
2025-11-19T13:59:08.203876Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435740877953578:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:08.203536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435740016543037:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.203551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:08.203556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435740016543037:2107] [node 4] ICP22 created new session: [5:7574435740016543069:2107]
2025-11-19T13:59:08.203560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435740016543069:2107] [node 4] ICS09 handshake done sender: [5:7574435740016543043:2112] self: [5:7574435740016543059:0] peer: [4:7574435739294563321:0] socket: 279 qp: -1
2025-11-19T13:59:08.203565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435740016543069:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:08.203570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:08.203314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435740456846295:2108] [node 5] ICP18 handshake (actor [3:7574435740456846314:2118]) is too old
2025-11-19T13:59:08.207419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435736582986204:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.207437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:08.207441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435736582986204:2049] [node 3] ICP22 created new session: [1:7574435740877953601:2049]
2025-11-19T13:59:08.207447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435740877953601:2049] [node 3] ICS09 handshake done sender: [1:7574435740877953578:2115] self: [1:7574435740877953569:0] peer: [3:7574435740456846320:0] socket: 297 qp: -1
2025-11-19T13:59:08.207451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435740877953601:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:08.207456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:08.206286Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435737255162944:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.206300Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435737255162944:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.206997Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435737255162944:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:08.207064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435732960195574:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.207078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:08.207083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435732960195574:2049] [node 3] ICP22 created new session: [2:7574435737255162968:2049]
2025-11-19T13:59:08.207089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435737255162968:2049] [node 3] ICS09 handshake done sender: [2:7574435737255162944:2114] self: [2:7574435737255162937:0] peer: [3:7574435740456846321:0] socket: 302 qp: -1
2025-11-19T13:59:08.207093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435737255162968:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:08.207099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:08.208211Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435740016543064:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.208243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435735721575680:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.208360Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435737255162948:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.208368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435737255162948:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.209061Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435737255162948:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:08.209801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435740016543072:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13889
2025-11-19T13:59:08.209545Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435735956348896:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435740251316273:2117] held: no
2025-11-19T13:59:08.210013Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435740016543064:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.210150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435740251316284:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:622
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.210217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435740251316285:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:622
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.210074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435735721575680:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.210087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:08.210093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435735721575680:2049] [node 2] ICP22 created new session: [5:7574435740016543073:2049]
2025-11-19T13:59:08.210098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435740016543073:2049] [node 2] ICS09 handshake done sender: [5:7574435740016543064:2118] self: [5:7574435740016543071:0] peer: [2:7574435737255162947:0] socket: 304 qp: -1
2025-11-19T13:59:08.210103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435740016543073:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:08.210112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:08.209839Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435736161878935:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14620) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435740456846315:2119] held: no
2025-11-19T13:59:08.210313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435740456846322:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4178
2025-11-19T13:59:08.211523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435737255162945:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.211539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:08.211546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435737255162945:2113] [node 5] ICP22 created new session: [2:7574435737255162971:2113]
2025-11-19T13:59:08.211551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435737255162971:2113] [node 5] ICS09 handshake done sender: [2:7574435737255162948:2116] self: [2:7574435737255162947:0] peer: [5:7574435740016543071:0] socket: 303 qp: -1
2025-11-19T13:59:08.211556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435737255162971:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:08.211561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.212024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435740456846323:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4178
2025-11-19T13:59:08.212124Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435737255162940:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:08.212479Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435736161878934:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435740456846313:2117] held: no
2025-11-19T13:59:08.212508Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435740456846313:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:08.212700Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435739294563329:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.212709Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435739294563329:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.214419Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435739294563329:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:08.215431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435739294563313:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.215447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:08.215453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435739294563313:2110] [node 6] ICP22 created new session: [4:7574435739294563345:2110]
2025-11-19T13:59:08.215459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435739294563345:2110] [node 6] ICS09 handshake done sender: [4:7574435739294563329:2118] self: [4:7574435739294563328:0] peer: [6:7574435740251316288:0] socket: 258 qp: -1
2025-11-19T13:59:08.215463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435739294563345:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:08.215469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:08.211312Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435740251316273:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:08.211427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435735956348896:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.211436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:08.211442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435735956348896:2048] [node 1] ICP22 created new session: [6:7574435740251316286:2048]
2025-11-19T13:59:08.211447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435740251316286:2048] [node 1] ICS09 handshake done sender: [6:7574435740251316273:2117] self: [6:7574435740251316274:0] peer: [1:7574435740877953572:0] socket: 267 qp: -1
2025-11-19T13:59:08.211453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435740251316286:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:08.211459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:08.212564Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435740251316281:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.212580Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435740251316285:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.212601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435740251316217:2071] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.212609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435735956348898:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.214273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435740251316290:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:622
2025-11-19T13:59:08.214582Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435740251316281:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:08.214637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435740251316217:2071] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.214644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:08.214648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435740251316217:2071] [node 4] ICP22 created new session: [6:7574435740251316291:2071]
2025-11-19T13:59:08.214653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435740251316291:2071] [node 4] ICS09 handshake done sender: [6:7574435740251316281:2120] self: [6:7574435740251316288:0] peer: [4:7574435739294563328:0] socket: 307 qp: -1
2025-11-19T13:59:08.214657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435740251316291:2071] [node 4] ICS10 traffic start
2025-11-19T13:59:08.214664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:08.216101Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435740456846315:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:08.216125Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435740456846304:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.216128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435740456846304:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.216752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435736161878934:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.216767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:08.216780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435736161878934:2048] [node 1] ICP22 created new session: [3:7574435740456846324:2048]
2025-11-19T13:59:08.216785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435740456846324:2048] [node 1] ICS09 handshake done sender: [3:7574435740456846313:2117] self: [3:7574435740456846320:0] peer: [1:7574435740877953569:0] socket: 298 qp: -1
2025-11-19T13:59:08.216790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435740456846324:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:08.216796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:08.216877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435736161878935:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.216882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:08.216886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435736161878935:2049] [node 2] ICP22 created new session: [3:7574435740456846328:2049]
2025-11-19T13:59:08.216889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435740456846328:2049] [node 2] ICS09 handshake done sender: [3:7574435740456846315:2119] self: [3:7574435740456846321:0] peer: [2:7574435737255162937:0] socket: 310 qp: -1
2025-11-19T13:59:08.216893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435740456846328:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:08.216898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.217297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435740251316293:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:622
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.217427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435739294563347:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19477
2025-11-19T13:59:08.220078Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435740456846304:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:08.221743Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435740251316285:2122] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:08.221896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435740456846298:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.221911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:08.221917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435740456846298:2109] [node 6] ICP22 created new session: [3:7574435740456846330:2109]
2025-11-19T13:59:08.221921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435740456846330:2109] [node 6] ICS09 handshake done sender: [3:7574435740456846304:2114] self: [3:7574435740456846303:0] peer: [6:7574435740251316289:0] socket: 294 qp: -1
2025-11-19T13:59:08.221926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435740456846330:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:08.221932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:08.222165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435735956348898:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.222176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:08.222181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435735956348898:2050] [node 3] ICP22 created new session: [6:7574435740251316294:2050]
2025-11-19T13:59:08.222186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435740251316294:2050] [node 3] ICS09 handshake done sender: [6:7574435740251316285:2122] self: [6:7574435740251316289:0] peer: [3:7574435740456846303:0] socket: 274 qp: -1
2025-11-19T13:59:08.222190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435740251316294:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:08.222195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:08.227042Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435739294563347:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.227090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435734999595953:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.228127Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435740456846310:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.228137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435740456846310:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.230057Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435740456846310:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:08.230155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435740456846292:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.230173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:08.230179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435740456846292:2107] [node 4] ICP22 created new session: [3:7574435740456846333:2107]
2025-11-19T13:59:08.230184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435740456846333:2107] [node 4] ICS09 handshake done sender: [3:7574435740456846310:2115] self: [3:7574435740456846309:0] peer: [4:7574435739294563348:0] socket: 252 qp: -1
2025-11-19T13:59:08.230189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435740456846333:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:08.230196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:08.231632Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435734999595953:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21008) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435739294563347:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.234752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435739294563349:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19477
2025-11-19T13:59:08.234984Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435739294563347:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:08.235043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435734999595953:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.235057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:08.235063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435734999595953:2050] [node 3] ICP22 created new session: [4:7574435739294563350:2050]
2025-11-19T13:59:08.235068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435739294563350:2050] [node 3] ICS09 handshake done sender: [4:7574435739294563347:2121] self: [4:7574435739294563348:0] peer: [3:7574435740456846309:0] socket: 288 qp: -1
2025-11-19T13:59:08.235075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435739294563350:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:08.235084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:08.242019Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:08.242107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435737255162960:2048] [node 1] ICS01 socket: 265 reason# EndOfStream
2025-11-19T13:59:08.242115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435732960195573:2048] [node 1] ICP30 unregister session Session# [2:7574435737255162960:2048] VirtualId# [2:7574435737255162950:0]
2025-11-19T13:59:08.242123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435737255162960:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:08.242159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:08.242182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:08.242337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:08.242420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435740456846324:2048] [node 1] ICS01 socket: 298 reason# EndOfStream
2025-11-19T13:59:08.242428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435736161878934:2048] [node 1] ICP30 unregister session Session# [3:7574435740456846324:2048] VirtualId# [3:7574435740456846320:0]
2025-11-19T13:59:08.242434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435740456846324:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:08.242467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:08.242484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:08.247195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:08.247280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435740016543056:2048] [node 1] ICS01 socket: 264 reason# EndOfStream
2025-11-19T13:59:08.247288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435735721575679:2048] [node 1] ICP30 unregister session Session# [5:7574435740016543056:2048] VirtualId# [5:7574435740016543044:0]
2025-11-19T13:59:08.247295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435740016543056:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:08.247399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:08.247456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435740251316286:2048] [node 1] ICS01 socket: 267 reason# EndOfStream
2025-11-19T13:59:08.247461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435735956348896:2048] [node 1] ICP30 unregister session Session# [6:7574435740251316286:2048] VirtualId# [6:7574435740251316274:0]
2025-11-19T13:59:08.247465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435740251316286:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:08.247492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:08.247512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:08.247340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:08.247366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:08.247588Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:08.247644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435739294563333:2048] [node 1] ICS01 socket: 273 reason# EndOfStream
2025-11-19T13:59:08.247648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435734999595951:2048] [node 1] ICP30 unregister session Session# [4:7574435739294563333:2048] VirtualId# [4:7574435739294563327:0]
2025-11-19T13:59:08.247652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435739294563333:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:08.247675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:08.247693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:08.256438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:08.256557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435740456846328:2049] [node 2] ICS01 socket: 310 reason# EndOfStream
2025-11-19T13:59:08.256566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435736161878935:2049] [node 2] ICP30 unregister session Session# [3:7574435740456846328:2049] VirtualId# [3:7574435740456846321:0]
2025-11-19T13:59:08.256574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435740456846328:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:08.256622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:08.256642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:08.279844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:08.279946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435739294563338:2049] [node 2] ICS01 socket: 257 reason# EndOfStream
2025-11-19T13:59:08.279956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435734999595952:2049] [node 2] ICP30 unregister session Session# [4:7574435739294563338:2049] VirtualId# [4:7574435739294563331:0]
2025-11-19T13:59:08.279962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435739294563338:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:08.280004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:08.280026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:08.280059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:08.280117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435740251316282:2049] [node 2] ICS07 socket disconnect 280 reason# EndOfStream
2025-11-19T13:59:08.280121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435740251316282:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:08.280143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:08.280162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435740251316282:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:08.280896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435735956348897:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14620) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:08.280906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435740251316282:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:08.280910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435735956348897:2049] [node 2] ICP30 unregister session Session# [6:7574435740251316282:2049] VirtualId# [6:7574435740251316272:0]
2025-11-19T13:59:08.280921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:08.280926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435735956348897:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14620) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:08.280928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14620) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:08.280339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:08.280398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435740016543073:2049] [node 2] ICS01 socket: 304 reason# EndOfStream
2025-11-19T13:59:08.280403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435735721575680:2049] [node 2] ICP30 unregister session Session# [5:7574435740016543073:2049] VirtualId# [5:7574435740016543071:0]
2025-11-19T13:59:08.280406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435740016543073:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:08.280430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:08.280450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:08.290734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435735956348897:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:08.290854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:08.290930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435740016543067:2050] [node 3] ICS01 socket: 272 reason# EndOfStream
2025-11-19T13:59:08.290935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435735721575681:2050] [node 3] ICP30 unregister session Session# [5:7574435740016543067:2050] VirtualId# [5:7574435740016543058:0]
2025-11-19T13:59:08.290941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435740016543067:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:08.290983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:08.291003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:08.291070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:08.291121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435740251316294:2050] [node 3] ICS01 socket: 274 reason# EndOfStream
2025-11-19T13:59:08.291126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435735956348898:2050] [node 3] ICP30 unregister session Session# [6:7574435740251316294:2050] VirtualId# [6:7574435740251316289:0]
2025-11-19T13:59:08.291130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435740251316294:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:08.291152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:08.291170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:08.299909Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:08.300006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435740251316291:2071] [node 4] ICS01 socket: 307 reason# EndOfStream
2025-11-19T13:59:08.300014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435740251316217:2071] [node 4] ICP30 unregister session Session# [6:7574435740251316291:2071] VirtualId# [6:7574435740251316288:0]
2025-11-19T13:59:08.300021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435740251316291:2071] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:08.300065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:08.300085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:08.300258Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T13:59:08.300314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435740016543069:2107] [node 4] ICS01 socket: 279 reason# EndOfStream
2025-11-19T13:59:08.300318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435740016543037:2107] [node 4] ICP30 unregister session Session# [5:7574435740016543069:2107] VirtualId# [5:7574435740016543059:0]
2025-11-19T13:59:08.300322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435740016543069:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:08.300346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:08.300364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:08.666261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435741253902278:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:08.666274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435741253902279:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:08.730701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435740356383432:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:08.730715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435740356383433:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:08.791518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435738852024830:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:08.791529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435738852024831:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:08.808147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435738389601829:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:08.808158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435738389601830:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:08.808162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435738389601831:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:08.827823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435739555879701:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:08.827834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435739555879702:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:08.827839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435739555879703:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:08.844000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435738123290298:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:08.844017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435738123290299:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:08.844021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435738123290300:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:08.896404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435738123290317:2062] [node 5] ICP01 ready to work
2025-11-19T13:59:08.896457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435738123290319:2059] [node 4] ICP01 ready to work
2025-11-19T13:59:08.896136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435738389601874:2089] [node 6] ICP01 ready to work
2025-11-19T13:59:08.896255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435738389601875:2091] [node 5] ICP01 ready to work
2025-11-19T13:59:08.896697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435739555879742:2076] [node 4] ICP01 ready to work
2025-11-19T13:59:08.899714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435738852024891:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:08.901018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435741253902339:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:08.902024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435740356383493:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.902547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435740356383494:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26389
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.909011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435739555879767:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7007
2025-11-19T13:59:08.909273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435738389601894:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
2025-11-19T13:59:08.909291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435738389601895:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.914370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435738852024896:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:08.915029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435738389601897:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.915965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435741253902351:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14183
2025-11-19T13:59:08.915987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435741253902352:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14183
2025-11-19T13:59:08.915835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435738123290317:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19317970ns) {Configure:104(19317169ns) {ProcessConfigured:135(17949000ns) {StartInitialHandshake:157(17948571ns) {DropHandshakes:469(245ns) {DropIncomingHandshake:441(26ns) DropOutgoingHandshake:456(27ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(1034ns) {SwitchToState:178(494ns)}}}}}
2025-11-19T13:59:08.920122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435738123290299:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23669176ns) {Configure:104(23668443ns) {ProcessConfigured:135(22157086ns) {StartInitialHandshake:157(22156148ns) {DropHandshakes:469(144ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(22ns)} GenerateSessionVirtualId:988(98ns) PrepareNewSessionHandshake:476(971ns) {SwitchToState:178(190ns)}}}}}
2025-11-19T13:59:08.924010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435741253902354:2114] [node 6] ICP01 ready to work
2025-11-19T13:59:08.924077Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435741253902352:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.924429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435739555879744:2067] [node 6] ICP01 ready to work
2025-11-19T13:59:08.928231Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435738389601897:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.927884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435739555879771:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7007
2025-11-19T13:59:08.937244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435738389601874:2089] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(40932796ns) {Configure:104(40931714ns) {ProcessConfigured:135(40527573ns) {StartInitialHandshake:157(40526938ns) {DropHandshakes:469(692ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(129ns)} GenerateSessionVirtualId:988(271ns) PrepareNewSessionHandshake:476(885ns) {SwitchToState:178(488ns)}}}}}
2025-11-19T13:59:08.940036Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435738389601894:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.940409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435738389601829:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.940587Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435741253902343:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.940598Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435741253902343:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.941265Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435741253902343:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:08.941347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435741253902339:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.941361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:08.941369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435741253902339:2107] [node 4] ICP22 created new session: [1:7574435741253902357:2107]
2025-11-19T13:59:08.941374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435741253902357:2107] [node 4] ICS09 handshake done sender: [1:7574435741253902343:2109] self: [1:7574435741253902342:0] peer: [4:7574435738389601909:0] socket: 261 qp: -1
2025-11-19T13:59:08.941380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435741253902357:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:08.941388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.941839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435738389601910:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
2025-11-19T13:59:08.942157Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435738389601894:2106] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:08.942281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435738389601829:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.942295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:08.942300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435738389601829:2048] [node 1] ICP22 created new session: [4:7574435738389601911:2048]
2025-11-19T13:59:08.942305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435738389601911:2048] [node 1] ICS09 handshake done sender: [4:7574435738389601894:2106] self: [4:7574435738389601909:0] peer: [1:7574435741253902342:0] socket: 269 qp: -1
2025-11-19T13:59:08.942312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435738389601911:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:08.942332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:08.943861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435740356383432:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(39838220ns) {Configure:104(39837178ns) {ProcessConfigured:135(39551809ns) {StartInitialHandshake:157(39551197ns) {DropHandshakes:469(590ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(144ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:476(1155ns) {SwitchToState:178(492ns)}}}}}
2025-11-19T13:59:08.944032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435740356383501:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:08.944191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435740356383503:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:08.944831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435741253902347:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:08.948133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435741253902354:2114] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(24001425ns) {Configure:104(24000825ns) {ProcessConfigured:135(23687937ns) {StartInitialHandshake:157(23687551ns) {DropHandshakes:469(520ns) {DropIncomingHandshake:441(116ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(80ns) PrepareNewSessionHandshake:476(884ns) {SwitchToState:178(574ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.951378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435738123290376:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19809
2025-11-19T13:59:08.950453Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435739555879771:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.950491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435739555879742:2076] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.951954Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435738389601895:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.952070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435738852024830:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(37651268ns) {Configure:104(37650594ns) {ProcessConfigured:135(37338251ns) {StartInitialHandshake:157(37337506ns) {DropHandshakes:469(647ns) {DropIncomingHandshake:441(105ns) DropOutgoingHandshake:456(49ns)} GenerateSessionVirtualId:988(116ns) PrepareNewSessionHandshake:476(1179ns) {SwitchToState:178(1055ns)}}}}}
2025-11-19T13:59:08.952156Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435738852024896:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24401718ns) {Configure:104(24400186ns) {ProcessConfigured:135(24019502ns) {StartInitialHandshake:157(24019035ns) {DropHandshakes:469(699ns) {DropIncomingHandshake:441(153ns) DropOutgoingHandshake:456(94ns)} GenerateSessionVirtualId:988(290ns) PrepareNewSessionHandshake:476(1221ns) {SwitchToState:178(839ns)}}}}}
2025-11-19T13:59:08.952247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435738852024902:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:08.951866Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435739555879702:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(55337793ns) {Configure:104(55243160ns) {ProcessConfigured:135(54995961ns) {StartInitialHandshake:157(54995699ns) {DropHandshakes:469(284ns) {DropIncomingHandshake:441(81ns) DropOutgoingHandshake:456(33ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(1341ns) {SwitchToState:178(419ns)}}}}}
2025-11-19T13:59:08.956149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435738389601893:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.956164Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435738389601893:2105] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.955667Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435741253902351:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.957185Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435740356383493:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(55055389ns) {Configure:104(55054786ns) {ProcessConfigured:135(54711734ns) {StartInitialHandshake:157(54711283ns) {DropHandshakes:469(385ns) {DropIncomingHandshake:441(46ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(177ns) PrepareNewSessionHandshake:476(923ns) {SwitchToState:178(582ns)}}}}}
2025-11-19T13:59:08.957607Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435740356383494:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.957647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435740356383502:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26389
2025-11-19T13:59:08.957812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435740356383432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.958250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435738852024909:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19016
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.961105Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435741253902341:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.961122Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435741253902341:2108] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.963018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435738123290377:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19809
2025-11-19T13:59:08.960009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435738852024911:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19016
2025-11-19T13:59:08.960035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435738852024912:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19016
2025-11-19T13:59:08.963152Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435738852024912:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.963715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435740356383513:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26389
2025-11-19T13:59:08.964549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435738852024910:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19016
2025-11-19T13:59:08.964673Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435740356383502:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.964636Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435738852024909:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.966854Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435741253902341:2108] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.968554Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435740356383433:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(61355434ns) {Configure:104(61354776ns) {ProcessConfigured:135(61092315ns) {StartInitialHandshake:157(61091460ns) {DropHandshakes:469(518ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(95ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:476(928ns) {SwitchToState:178(519ns)}}}}}
2025-11-19T13:59:08.968590Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435740356383494:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:08.968789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435741253902366:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14183
2025-11-19T13:59:08.968807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435741253902367:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14183
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.974709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435738389601914:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
2025-11-19T13:59:08.974730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435738389601915:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
2025-11-19T13:59:08.976711Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435738389601893:2105] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:08.977199Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435738852024910:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.979157Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435738852024911:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.979261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435738852024830:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.980335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435741253902278:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.980363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:08.980372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435741253902278:2048] [node 2] ICP22 created new session: [1:7574435741253902368:2048]
2025-11-19T13:59:08.980380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435741253902368:2048] [node 2] ICS09 handshake done sender: [1:7574435741253902341:2108] self: [1:7574435741253902340:0] peer: [2:7574435740356383509:0] socket: 259 qp: -1
2025-11-19T13:59:08.980391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435741253902368:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:08.980400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:08.980593Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435741253902349:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.980599Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435741253902349:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.982056Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435741253902349:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:08.982168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435741253902279:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.982182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:08.982188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435741253902279:2049] [node 3] ICP22 created new session: [1:7574435741253902372:2049]
2025-11-19T13:59:08.982194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435741253902372:2049] [node 3] ICS09 handshake done sender: [1:7574435741253902349:2111] self: [1:7574435741253902348:0] peer: [3:7574435738852024915:0] socket: 264 qp: -1
2025-11-19T13:59:08.982199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435741253902372:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:08.982206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.983898Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435739555879767:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.983948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435739555879703:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.984006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435738852024917:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19016
2025-11-19T13:59:08.984250Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435738852024893:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.984259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435738852024893:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.984648Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435738852024910:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:08.986282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435740356383432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.986309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:08.986317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435740356383432:2048] [node 1] ICP22 created new session: [2:7574435740356383516:2048]
2025-11-19T13:59:08.986324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435740356383516:2048] [node 1] ICS09 handshake done sender: [2:7574435740356383494:2108] self: [2:7574435740356383509:0] peer: [1:7574435741253902340:0] socket: 262 qp: -1
2025-11-19T13:59:08.986331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435740356383516:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:08.986352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.987675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435739555879783:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7007
2025-11-19T13:59:08.987722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435739555879784:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7007
2025-11-19T13:59:08.987992Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435738123290376:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.988034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435738123290298:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.988200Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435741253902360:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.988210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435741253902360:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:08.989145Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435739555879784:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:08.989171Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435739555879771:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:08.989207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435739555879701:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:08.989349Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435741253902360:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:08.989437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435741253902354:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:08.989453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:08.989459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435741253902354:2114] [node 6] ICP22 created new session: [1:7574435741253902375:2114]
2025-11-19T13:59:08.989463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435741253902375:2114] [node 6] ICS09 handshake done sender: [1:7574435741253902360:2115] self: [1:7574435741253902355:0] peer: [6:7574435738123290378:0] socket: 283 qp: -1
2025-11-19T13:59:08.989468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435741253902375:2114] [node 6] ICS10 traffic start
2025-11-19T13:59:08.989474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:08.989776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435739555879742:2076] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.989789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:08.989794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435739555879742:2076] [node 4] ICP22 created new session: [5:7574435739555879786:2076]
2025-11-19T13:59:08.989798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435739555879786:2076] [node 4] ICS09 handshake done sender: [5:7574435739555879771:2108] self: [5:7574435739555879780:0] peer: [4:7574435738389601877:0] socket: 275 qp: -1
2025-11-19T13:59:08.989803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435739555879786:2076] [node 4] ICS10 traffic start
2025-11-19T13:59:08.989808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:08.992819Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435738123290298:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26423) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435738123290376:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.998546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435739555879789:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7007
2025-11-19T13:59:08.998605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435739555879790:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7007
2025-11-19T13:59:08.999315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435738852024830:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:08.999340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:08.999347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435738852024830:2048] [node 1] ICP22 created new session: [3:7574435738852024919:2048]
2025-11-19T13:59:08.999353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435738852024919:2048] [node 1] ICS09 handshake done sender: [3:7574435738852024910:2116] self: [3:7574435738852024915:0] peer: [1:7574435741253902348:0] socket: 292 qp: -1
2025-11-19T13:59:08.999360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435738852024919:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:08.999368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:08.999836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435739555879791:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7007
2025-11-19T13:59:09.000185Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435738123290377:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.000273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435738123290299:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.000398Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435739555879790:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.000622Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435741253902278:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:09.000892Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435738852024893:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:09.001122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435738852024891:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.001138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:09.001143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435738852024891:2107] [node 5] ICP22 created new session: [3:7574435743146992217:2107]
2025-11-19T13:59:09.001148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435743146992217:2107] [node 5] ICS09 handshake done sender: [3:7574435738852024893:2108] self: [3:7574435738852024892:0] peer: [5:7574435739555879782:0] socket: 258 qp: -1
2025-11-19T13:59:09.001152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435743146992217:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:09.001158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.002388Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435741253902363:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.002402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435741253902363:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.003433Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435741253902363:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:09.003213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435742418257677:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19809
2025-11-19T13:59:09.003230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435742418257678:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19809
2025-11-19T13:59:09.003332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435742418257679:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19809
2025-11-19T13:59:09.003505Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435740356383507:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.003519Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435740356383507:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.003575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435741253902347:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.003595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:09.003600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435741253902347:2110] [node 5] ICP22 created new session: [1:7574435745548869674:2110]
2025-11-19T13:59:09.003607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435745548869674:2110] [node 5] ICS09 handshake done sender: [1:7574435741253902363:2116] self: [1:7574435741253902359:0] peer: [5:7574435739555879785:0] socket: 305 qp: -1
2025-11-19T13:59:09.003613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435745548869674:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:09.003620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.004163Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435739555879789:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.004184Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435739555879703:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18506) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435739555879767:2107] held: no
2025-11-19T13:59:09.004215Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435739555879767:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:09.004082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435744651350815:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26389
2025-11-19T13:59:09.004583Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435742418257679:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.004604Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435738123290376:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:09.004647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435738123290317:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.004756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435738123290298:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.004769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:09.004774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435738123290298:2048] [node 1] ICP22 created new session: [6:7574435742418257681:2048]
2025-11-19T13:59:09.004778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435742418257681:2048] [node 1] ICS09 handshake done sender: [6:7574435738123290376:2115] self: [6:7574435738123290378:0] peer: [1:7574435741253902355:0] socket: 284 qp: -1
2025-11-19T13:59:09.004785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435742418257681:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:09.004792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.008094Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435741253902279:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:09.008439Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435744651350815:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.008580Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435742418257677:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.008600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435738123290319:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.008669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435742418257683:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19809
2025-11-19T13:59:09.008718Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435738389601915:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.008835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435738389601830:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.009182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435738389601875:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.009229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:09.009239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435738389601875:2091] [node 5] ICP22 created new session: [4:7574435742684569215:2091]
2025-11-19T13:59:09.009245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435742684569215:2091] [node 5] ICS09 handshake done sender: [4:7574435738389601893:2105] self: [4:7574435738389601877:0] peer: [5:7574435739555879780:0] socket: 271 qp: -1
2025-11-19T13:59:09.009250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435742684569215:2091] [node 5] ICS10 traffic start
2025-11-19T13:59:09.009260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:09.009360Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435738389601875:2091] [node 5] ICP35 event processing took too much time PendingConnection:278(30878514ns) {HandleHandshakeStatus:329(30877580ns) {DropIncomingHandshake:441(24994ns) DropOutgoingHandshake:456(92ns) SwitchToState:178(166ns) ProcessPendingSessionEvents:495(32721ns) {ForwardSessionEventToSession:612(23636ns) ForwardSessionEventToSession:612(782ns) ForwardSessionEventToSession:612(1171ns) ForwardSessionEventToSession:612(815ns) ForwardSessionEventToSession:612(465ns) ForwardSessionEventToSession:612(732ns) ForwardSessionEventToSession:612(622ns) ForwardSessionEventToSession:612(974ns) ForwardSessionEventToSession:612(414ns) ForwardSessionEventToSession:612(431ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.012901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435742418257685:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19809
2025-11-19T13:59:09.012981Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435740356383507:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.014080Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435739555879779:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.014091Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435739555879779:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.014674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435739555879703:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.014683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:09.014688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435739555879703:2050] [node 3] ICP22 created new session: [5:7574435743850847090:2050]
2025-11-19T13:59:09.014693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435743850847090:2050] [node 3] ICS09 handshake done sender: [5:7574435739555879767:2107] self: [5:7574435739555879782:0] peer: [3:7574435738852024892:0] socket: 265 qp: -1
2025-11-19T13:59:09.014699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435743850847090:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:09.014706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:09.013138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435740356383503:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.013152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:09.013158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435740356383503:2110] [node 6] ICP22 created new session: [2:7574435744651350818:2110]
2025-11-19T13:59:09.013164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435744651350818:2110] [node 6] ICS09 handshake done sender: [2:7574435740356383507:2114] self: [2:7574435740356383506:0] peer: [6:7574435738123290380:0] socket: 291 qp: -1
2025-11-19T13:59:09.013169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435744651350818:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:09.013176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:09.015401Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435740356383505:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.015406Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435740356383505:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.015347Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435738389601907:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.015478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435743146992221:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19016
2025-11-19T13:59:09.015358Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435738389601907:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.015987Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435738389601914:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.016013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435738389601831:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.017979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435743850847089:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7007
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.019708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435742418257686:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19809
2025-11-19T13:59:09.020096Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435742418257677:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:09.020149Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435738123290377:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:09.019892Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435738389601907:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:09.020205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435738123290299:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.020221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:09.020226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435738123290299:2049] [node 2] ICP22 created new session: [6:7574435742418257687:2049]
2025-11-19T13:59:09.020232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435742418257687:2049] [node 2] ICS09 handshake done sender: [6:7574435738123290377:2116] self: [6:7574435738123290380:0] peer: [2:7574435740356383506:0] socket: 296 qp: -1
2025-11-19T13:59:09.020237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435742418257687:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:09.020244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.020627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435739555879702:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.020648Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435739555879701:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26423) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435739555879784:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.021766Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435742418257683:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.021853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435738123290300:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.021869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435742418257689:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19809
2025-11-19T13:59:09.022439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435738123290319:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.022452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:09.022457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435738123290319:2059] [node 4] ICP22 created new session: [6:7574435742418257691:2059]
2025-11-19T13:59:09.022462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435742418257691:2059] [node 4] ICS09 handshake done sender: [6:7574435742418257677:2117] self: [6:7574435742418257684:0] peer: [4:7574435738389601876:0] socket: 303 qp: -1
2025-11-19T13:59:09.022467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435742418257691:2059] [node 4] ICS10 traffic start
2025-11-19T13:59:09.022472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.023627Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435740356383505:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:09.024953Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435740356383512:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.024962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435740356383512:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.025552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435740356383493:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.025562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:09.025567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435740356383493:2107] [node 4] ICP22 created new session: [2:7574435744651350823:2107]
2025-11-19T13:59:09.025573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435744651350823:2107] [node 4] ICS09 handshake done sender: [2:7574435740356383505:2113] self: [2:7574435740356383495:0] peer: [4:7574435742684569214:0] socket: 290 qp: -1
2025-11-19T13:59:09.025578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435744651350823:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:09.025585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.027277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435744651350822:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26389
2025-11-19T13:59:09.027523Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435739555879779:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:09.027709Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435742418257679:2119] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:09.020531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435742684569219:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
2025-11-19T13:59:09.022152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435738389601874:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.022165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:09.022170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435738389601874:2089] [node 6] ICP22 created new session: [4:7574435742684569221:2089]
2025-11-19T13:59:09.022175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435742684569221:2089] [node 6] ICS09 handshake done sender: [4:7574435738389601907:2116] self: [4:7574435738389601876:0] peer: [6:7574435742418257684:0] socket: 302 qp: -1
2025-11-19T13:59:09.022180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435742684569221:2089] [node 6] ICS10 traffic start
2025-11-19T13:59:09.022187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:09.023548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435742684569220:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34821
2025-11-19T13:59:09.026108Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435738389601914:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:09.026155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435738389601831:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.026164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:09.026168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435738389601831:2050] [node 3] ICP22 created new session: [4:7574435742684569223:2050]
2025-11-19T13:59:09.026174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435742684569223:2050] [node 3] ICS09 handshake done sender: [4:7574435738389601914:2118] self: [4:7574435742684569217:0] peer: [3:7574435738852024899:0] socket: 288 qp: -1
2025-11-19T13:59:09.026178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435742684569223:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:09.026184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:09.021015Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435738852024901:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.021031Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435738852024901:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.022051Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435738852024901:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:09.023784Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435738852024908:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.023789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435738852024908:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.024396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435738852024896:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.024410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:09.024416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435738852024896:2109] [node 4] ICP22 created new session: [3:7574435743146992223:2109]
2025-11-19T13:59:09.024421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435743146992223:2109] [node 4] ICS09 handshake done sender: [3:7574435738852024901:2112] self: [3:7574435738852024899:0] peer: [4:7574435742684569217:0] socket: 286 qp: -1
2025-11-19T13:59:09.024428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435743146992223:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:09.024437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:09.028012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435738123290317:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.028026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:09.028031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435738123290317:2062] [node 5] ICP22 created new session: [6:7574435742418257693:2062]
2025-11-19T13:59:09.028036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435742418257693:2062] [node 5] ICS09 handshake done sender: [6:7574435742418257679:2119] self: [6:7574435742418257680:0] peer: [5:7574435739555879778:0] socket: 313 qp: -1
2025-11-19T13:59:09.028041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435742418257693:2062] [node 5] ICS10 traffic start
2025-11-19T13:59:09.028047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:09.029069Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435738852024908:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:09.029154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435738852024902:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.029170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:09.029175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435738852024902:2110] [node 6] ICP22 created new session: [3:7574435743146992226:2110]
2025-11-19T13:59:09.029181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435743146992226:2110] [node 6] ICS09 handshake done sender: [3:7574435738852024908:2114] self: [3:7574435738852024907:0] peer: [6:7574435742418257690:0] socket: 277 qp: -1
2025-11-19T13:59:09.029185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435743146992226:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:09.029190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.029482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435742418257695:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19809
2025-11-19T13:59:09.030437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435739555879744:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.030455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:09.030461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435739555879744:2067] [node 6] ICP22 created new session: [5:7574435743850847096:2067]
2025-11-19T13:59:09.030466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435743850847096:2067] [node 6] ICS09 handshake done sender: [5:7574435739555879779:2115] self: [5:7574435739555879778:0] peer: [6:7574435742418257680:0] socket: 309 qp: -1
2025-11-19T13:59:09.030470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435743850847096:2067] [node 6] ICS10 traffic start
2025-11-19T13:59:09.030476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:09.027939Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435738389601915:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:09.030806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435738389601830:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.030816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:09.030820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435738389601830:2049] [node 2] ICP22 created new session: [4:7574435742684569225:2049]
2025-11-19T13:59:09.030825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435742684569225:2049] [node 2] ICS09 handshake done sender: [4:7574435738389601915:2119] self: [4:7574435742684569214:0] peer: [2:7574435740356383495:0] socket: 299 qp: -1
2025-11-19T13:59:09.030829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435742684569225:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:09.030834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:09.031823Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435740356383512:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.032008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435743850847095:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7007
2025-11-19T13:59:09.032171Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435739555879784:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:09.032281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435739555879701:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.032294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:09.032300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435739555879701:2048] [node 1] ICP22 created new session: [5:7574435743850847098:2048]
2025-11-19T13:59:09.032305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435743850847098:2048] [node 1] ICS09 handshake done sender: [5:7574435739555879784:2118] self: [5:7574435739555879785:0] peer: [1:7574435741253902359:0] socket: 307 qp: -1
2025-11-19T13:59:09.032311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435743850847098:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:09.032318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:09.032414Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435739555879789:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:09.032671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435738123290300:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18506) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435742418257683:2120] held: no
2025-11-19T13:59:09.032789Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435742418257683:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:09.033917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435740356383501:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.033933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:09.033939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435740356383501:2109] [node 5] ICP22 created new session: [2:7574435744651350826:2109]
2025-11-19T13:59:09.033945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435744651350826:2109] [node 5] ICS09 handshake done sender: [2:7574435740356383512:2115] self: [2:7574435740356383504:0] peer: [5:7574435743850847092:0] socket: 289 qp: -1
2025-11-19T13:59:09.033950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435744651350826:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:09.033956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:09.040043Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435743146992221:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.040081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435738852024831:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.039670Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435744651350822:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.039702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435740356383501:2109] [node 5] ICP18 handshake (actor [2:7574435744651350822:2119]) is too old
2025-11-19T13:59:09.040204Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435740356383515:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.040207Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435740356383515:2117] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.041380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435743146992229:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19016
2025-11-19T13:59:09.041549Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435740356383515:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:09.042431Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435743146992221:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:09.042537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435740356383433:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.042555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:09.042561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435740356383433:2049] [node 3] ICP22 created new session: [2:7574435744651350829:2049]
2025-11-19T13:59:09.042567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435744651350829:2049] [node 3] ICS09 handshake done sender: [2:7574435740356383515:2117] self: [2:7574435740356383497:0] peer: [3:7574435743146992228:0] socket: 316 qp: -1
2025-11-19T13:59:09.042573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435744651350829:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:09.042581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:09.042832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435738852024831:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.042844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:09.042849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435738852024831:2049] [node 2] ICP22 created new session: [3:7574435743146992230:2049]
2025-11-19T13:59:09.042853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435743146992230:2049] [node 2] ICS09 handshake done sender: [3:7574435743146992221:2120] self: [3:7574435743146992228:0] peer: [2:7574435740356383497:0] socket: 317 qp: -1
2025-11-19T13:59:09.042858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435743146992230:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:09.042865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:09.043887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435739555879702:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5479) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435739555879789:2119] held: no
2025-11-19T13:59:09.044574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435739555879702:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.044586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:09.044592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435739555879702:2049] [node 2] ICP22 created new session: [5:7574435743850847100:2049]
2025-11-19T13:59:09.044597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435743850847100:2049] [node 2] ICS09 handshake done sender: [5:7574435739555879789:2119] self: [5:7574435743850847092:0] peer: [2:7574435740356383504:0] socket: 278 qp: -1
2025-11-19T13:59:09.044602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435743850847100:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:09.044612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:09.045086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435738123290300:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.045102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:09.045107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435738123290300:2050] [node 3] ICP22 created new session: [6:7574435742418257696:2050]
2025-11-19T13:59:09.045112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435742418257696:2050] [node 3] ICS09 handshake done sender: [6:7574435742418257683:2120] self: [6:7574435742418257690:0] peer: [3:7574435738852024907:0] socket: 285 qp: -1
2025-11-19T13:59:09.045116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435742418257696:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:09.045127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:09.055175Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:09.055258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435740356383516:2048] [node 1] ICS01 socket: 262 reason# EndOfStream
2025-11-19T13:59:09.055266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435740356383432:2048] [node 1] ICP30 unregister session Session# [2:7574435740356383516:2048] VirtualId# [2:7574435740356383509:0]
2025-11-19T13:59:09.055273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435740356383516:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.055306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:09.055326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:09.055813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:09.055879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435738852024919:2048] [node 1] ICS01 socket: 292 reason# EndOfStream
2025-11-19T13:59:09.055885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435738852024830:2048] [node 1] ICP30 unregister session Session# [3:7574435738852024919:2048] VirtualId# [3:7574435738852024915:0]
2025-11-19T13:59:09.055889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435738852024919:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.055919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:09.055936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:09.059054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:09.059126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435738389601911:2048] [node 1] ICS07 socket disconnect 269 reason# EndOfStream
2025-11-19T13:59:09.059131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435738389601911:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.059174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:09.059192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435738389601911:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:09.059388Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:09.059449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435743850847098:2048] [node 1] ICS01 socket: 307 reason# EndOfStream
2025-11-19T13:59:09.059453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435739555879701:2048] [node 1] ICP30 unregister session Session# [5:7574435743850847098:2048] VirtualId# [5:7574435739555879785:0]
2025-11-19T13:59:09.059460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435743850847098:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.059487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:09.059506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:09.059536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:09.059588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435742418257681:2048] [node 1] ICS01 socket: 284 reason# EndOfStream
2025-11-19T13:59:09.059593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435738123290298:2048] [node 1] ICP30 unregister session Session# [6:7574435742418257681:2048] VirtualId# [6:7574435738123290378:0]
2025-11-19T13:59:09.059596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435742418257681:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.059619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:09.059636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:09.060282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435738389601829:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26423) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:09.060293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435738389601911:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:09.060299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435738389601829:2048] [node 1] ICP30 unregister session Session# [4:7574435738389601911:2048] VirtualId# [4:7574435738389601909:0]
2025-11-19T13:59:09.060310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:09.060315Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435738389601829:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26423) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:09.060318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26423) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:09.067474Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:09.067555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435743146992230:2049] [node 2] ICS01 socket: 317 reason# EndOfStream
2025-11-19T13:59:09.067562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435738852024831:2049] [node 2] ICP30 unregister session Session# [3:7574435743146992230:2049] VirtualId# [3:7574435743146992228:0]
2025-11-19T13:59:09.067568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435743146992230:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.067601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:09.067618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:09.071899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435738389601829:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:09.075957Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:09.076035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435742684569225:2049] [node 2] ICS01 socket: 299 reason# EndOfStream
2025-11-19T13:59:09.076044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435738389601830:2049] [node 2] ICP30 unregister session Session# [4:7574435742684569225:2049] VirtualId# [4:7574435742684569214:0]
2025-11-19T13:59:09.076050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435742684569225:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.076100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:09.076129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:09.076087Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:09.076163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435742418257687:2049] [node 2] ICS01 socket: 296 reason# EndOfStream
2025-11-19T13:59:09.076168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435738123290299:2049] [node 2] ICP30 unregister session Session# [6:7574435742418257687:2049] VirtualId# [6:7574435738123290380:0]
2025-11-19T13:59:09.076172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435742418257687:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.076208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:09.076227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:09.076392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:09.076467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435743850847100:2049] [node 2] ICS01 socket: 278 reason# EndOfStream
2025-11-19T13:59:09.076472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435739555879702:2049] [node 2] ICP30 unregister session Session# [5:7574435743850847100:2049] VirtualId# [5:7574435743850847092:0]
2025-11-19T13:59:09.076477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435743850847100:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.076516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:09.076554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:09.083791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:09.083867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435743850847090:2050] [node 3] ICS01 socket: 265 reason# EndOfStream
2025-11-19T13:59:09.083873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435739555879703:2050] [node 3] ICP30 unregister session Session# [5:7574435743850847090:2050] VirtualId# [5:7574435739555879782:0]
2025-11-19T13:59:09.083878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435743850847090:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.083922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:09.083944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:09.085819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:09.085898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435742418257696:2050] [node 3] ICS01 socket: 285 reason# EndOfStream
2025-11-19T13:59:09.085906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435738123290300:2050] [node 3] ICP30 unregister session Session# [6:7574435742418257696:2050] VirtualId# [6:7574435742418257690:0]
2025-11-19T13:59:09.085912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435742418257696:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.085960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:09.085988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:09.102964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:09.103059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435742418257691:2059] [node 4] ICS01 socket: 303 reason# EndOfStream
2025-11-19T13:59:09.103068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435738123290319:2059] [node 4] ICP30 unregister session Session# [6:7574435742418257691:2059] VirtualId# [6:7574435742418257684:0]
2025-11-19T13:59:09.103076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435742418257691:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.103133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:09.103158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:09.211872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435744261731068:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:09.211887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435744261731069:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:09.259960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435742375433182:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:09.259978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435742375433183:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:09.315874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435745383882098:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:09.315888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435745383882099:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:09.361084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435744165832205:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:09.361181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435744165832206:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:09.361186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435744165832207:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:09.377479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435743606308402:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:09.377493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435743606308403:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:09.377497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435743606308404:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:09.396030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435743776154268:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:09.396042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435743776154269:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:09.396046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435743776154270:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:09.432081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435744165832247:2088] [node 6] ICP01 ready to work
2025-11-19T13:59:09.432002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435743776154284:2059] [node 4] ICP01 ready to work
2025-11-19T13:59:09.432100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435743776154285:2061] [node 5] ICP01 ready to work
2025-11-19T13:59:09.432422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435744261731129:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.433288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435743776154290:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34593
2025-11-19T13:59:09.434760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435744261731137:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:09.434580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435745383882158:2106] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.435935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435744165832253:2091] [node 5] ICP01 ready to work
2025-11-19T13:59:09.436691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435742375433245:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53875
2025-11-19T13:59:09.439023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435743606308426:2065] [node 4] ICP01 ready to work
2025-11-19T13:59:09.441513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435742375433250:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.448105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435742375433253:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53875
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.451667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435744261731143:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3106
2025-11-19T13:59:09.451686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435744261731144:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3106
2025-11-19T13:59:09.452881Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435744261731144:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.451454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435744165832273:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22327
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.455893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435743606308449:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63297
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.456359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435745383882166:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:09.456436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435745383882167:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:09.458725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435743776154315:2087] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34593
2025-11-19T13:59:09.458767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435743776154316:2088] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34593
2025-11-19T13:59:09.458244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435744165832274:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22327
2025-11-19T13:59:09.459139Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435744165832273:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.461817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435743606308450:2086] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63297
2025-11-19T13:59:09.460161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435742375433256:2113] [node 4] ICP01 ready to work
2025-11-19T13:59:09.464106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435744261731141:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:09.466219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435742375433248:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:09.471402Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435743606308426:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(32321309ns) {Configure:104(32320572ns) {ProcessConfigured:135(32083713ns) {StartInitialHandshake:157(32083153ns) {DropHandshakes:469(408ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(79ns) PrepareNewSessionHandshake:476(918ns) {SwitchToState:178(499ns)}}}}}
2025-11-19T13:59:09.471509Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435743776154268:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(32286389ns) {Configure:104(32285531ns) {ProcessConfigured:135(32003668ns) {StartInitialHandshake:157(31996744ns) {DropHandshakes:469(392ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(49ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:476(675ns) {SwitchToState:178(510ns)}}}}}
2025-11-19T13:59:09.471581Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435743776154315:2087] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.471623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435743776154270:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.471884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435743776154269:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(32001758ns) {Configure:104(32001290ns) {ProcessConfigured:135(31760955ns) {StartInitialHandshake:157(31760770ns) {DropHandshakes:469(441ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(43ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:476(597ns) {SwitchToState:178(425ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.472154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435744261731148:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3106
2025-11-19T13:59:09.472038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435744165832206:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(32011552ns) {Configure:104(32010716ns) {ProcessConfigured:135(31756213ns) {StartInitialHandshake:157(31755759ns) {DropHandshakes:469(460ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(13ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:476(587ns) {SwitchToState:178(416ns)}}}}}
2025-11-19T13:59:09.478610Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435743776154290:2064] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.478633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435743776154284:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.478746Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435743606308449:2085] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.478778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435743606308426:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.478781Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435744165832249:2089] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.478789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435744165832249:2089] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.479324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435743776154347:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34593
2025-11-19T13:59:09.479430Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435744165832249:2089] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:09.479611Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435743776154290:2064] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:09.478244Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435744261731143:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.478447Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435744261731148:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.477446Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435745383882098:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(36624464ns) {Configure:104(36623241ns) {ProcessConfigured:135(36363740ns) {StartInitialHandshake:157(36362958ns) {DropHandshakes:469(423ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(159ns) PrepareNewSessionHandshake:476(1262ns) {SwitchToState:178(636ns)}}}}}
2025-11-19T13:59:09.479669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435743776154284:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.479694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:09.479700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435743776154284:2059] [node 4] ICP22 created new session: [6:7574435743776154348:2059]
2025-11-19T13:59:09.479705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435743776154348:2059] [node 4] ICS09 handshake done sender: [6:7574435743776154290:2064] self: [6:7574435743776154346:0] peer: [4:7574435744165832248:0] socket: 267 qp: -1
2025-11-19T13:59:09.479712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435743776154348:2059] [node 4] ICS10 traffic start
2025-11-19T13:59:09.479721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:09.479890Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435742375433253:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.480101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435742375433250:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(38552550ns) {Configure:104(38552121ns) {ProcessConfigured:135(38331734ns) {StartInitialHandshake:157(38331650ns) {DropHandshakes:469(383ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(38ns) PrepareNewSessionHandshake:476(632ns) {SwitchToState:178(116ns)}}}}}
2025-11-19T13:59:09.480003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435744261731069:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(47531066ns) {Configure:104(47530059ns) {ProcessConfigured:135(47156346ns) {StartInitialHandshake:157(47155912ns) {DropHandshakes:469(504ns) {DropIncomingHandshake:441(92ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(142ns) PrepareNewSessionHandshake:476(623ns) {SwitchToState:178(271ns)}}}}}
2025-11-19T13:59:09.484101Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435744165832263:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.484115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435744165832263:2099] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.483817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435743606308435:2074] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.493372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435743776154285:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(60693471ns) {Configure:104(60442351ns) {ProcessConfigured:135(60208273ns) {StartInitialHandshake:157(60207802ns) {DropHandshakes:469(331ns) {DropIncomingHandshake:441(44ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(166ns) PrepareNewSessionHandshake:476(794ns) {SwitchToState:178(155ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.494020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435744261731152:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3106
2025-11-19T13:59:09.494818Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435745383882160:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.494831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435745383882160:2107] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.495319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435744261731141:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(31151500ns) {Configure:104(31150798ns) {ProcessConfigured:135(30884313ns) {StartInitialHandshake:157(30883933ns) {DropHandshakes:469(420ns) {DropIncomingHandshake:441(34ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:476(538ns) {SwitchToState:178(223ns)}}}}}
2025-11-19T13:59:09.493607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435744165832287:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22327
2025-11-19T13:59:09.495531Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435742375433245:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.495588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435742375433182:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.493874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435743606308482:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63297
2025-11-19T13:59:09.495670Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435744261731133:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.495675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435744261731133:2108] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.496639Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435743776154316:2088] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.496721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435743776154268:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.496735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435743776154354:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34593
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.496856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435742375433264:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53875
2025-11-19T13:59:09.497181Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435745383882160:2107] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:09.497384Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435743776154315:2087] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:09.497445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435743776154270:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.497464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:09.497470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435743776154270:2050] [node 3] ICP22 created new session: [6:7574435743776154356:2050]
2025-11-19T13:59:09.497475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435743776154356:2050] [node 3] ICS09 handshake done sender: [6:7574435743776154315:2087] self: [6:7574435743776154343:0] peer: [3:7574435745383882159:0] socket: 276 qp: -1
2025-11-19T13:59:09.497484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435743776154356:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:09.497501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:09.498402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435745383882158:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.498413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:09.498428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435745383882158:2106] [node 6] ICP22 created new session: [3:7574435745383882177:2106]
2025-11-19T13:59:09.498433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435745383882177:2106] [node 6] ICS09 handshake done sender: [3:7574435745383882160:2107] self: [3:7574435745383882159:0] peer: [6:7574435743776154343:0] socket: 270 qp: -1
2025-11-19T13:59:09.498437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435745383882177:2106] [node 6] ICS10 traffic start
2025-11-19T13:59:09.498442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:09.496414Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435743606308450:2086] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.496448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435743606308402:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.500024Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435744261731133:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:09.500110Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435744261731139:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.500114Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435744261731139:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.501247Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435742375433245:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.501359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435743776154359:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34593
2025-11-19T13:59:09.501521Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435744261731139:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:09.501587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435744261731068:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.501601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:09.501606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435744261731068:2048] [node 2] ICP22 created new session: [1:7574435744261731156:2048]
2025-11-19T13:59:09.501611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435744261731156:2048] [node 2] ICS09 handshake done sender: [1:7574435744261731133:2108] self: [1:7574435744261731132:0] peer: [2:7574435742375433263:0] socket: 269 qp: -1
2025-11-19T13:59:09.501615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435744261731156:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:09.501631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:09.501905Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435743776154316:2088] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:09.501961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435742375433182:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.501974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:09.501978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435742375433182:2048] [node 1] ICP22 created new session: [2:7574435742375433265:2048]
2025-11-19T13:59:09.501983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435742375433265:2048] [node 1] ICS09 handshake done sender: [2:7574435742375433245:2108] self: [2:7574435742375433263:0] peer: [1:7574435744261731132:0] socket: 272 qp: -1
2025-11-19T13:59:09.501988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435742375433265:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:09.502001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:09.502848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435744261731137:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.502862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:09.502866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435744261731137:2110] [node 6] ICP22 created new session: [1:7574435744261731158:2110]
2025-11-19T13:59:09.502870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435744261731158:2110] [node 6] ICS09 handshake done sender: [1:7574435744261731139:2111] self: [1:7574435744261731138:0] peer: [6:7574435743776154355:0] socket: 271 qp: -1
2025-11-19T13:59:09.502874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435744261731158:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:09.502878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:09.503096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435743776154268:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.503109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:09.503113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435743776154268:2048] [node 1] ICP22 created new session: [6:7574435743776154360:2048]
2025-11-19T13:59:09.503117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435743776154360:2048] [node 1] ICS09 handshake done sender: [6:7574435743776154316:2088] self: [6:7574435743776154355:0] peer: [1:7574435744261731138:0] socket: 290 qp: -1
2025-11-19T13:59:09.503122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435743776154360:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:09.503127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:09.504100Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435743776154284:2059] [node 4] ICP35 event processing took too much time PendingConnection:278(24405624ns) {HandleHandshakeStatus:329(24404956ns) {DropIncomingHandshake:441(340ns) DropOutgoingHandshake:456(2172ns) SwitchToState:178(258ns) ProcessPendingSessionEvents:495(24249649ns) {ForwardSessionEventToSession:612(24238257ns) ForwardSessionEventToSession:612(1326ns) ForwardSessionEventToSession:612(1010ns) ForwardSessionEventToSession:612(540ns) ForwardSessionEventToSession:612(474ns) ForwardSessionEventToSession:612(812ns) ForwardSessionEventToSession:612(577ns) ForwardSessionEventToSession:612(791ns) ForwardSessionEventToSession:612(922ns) ForwardSessionEventToSession:612(1081ns)}}}
2025-11-19T13:59:09.503837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435742375433248:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(37548236ns) {Configure:104(37547721ns) {ProcessConfigured:135(37279267ns) {StartInitialHandshake:157(37278361ns) {DropHandshakes:469(828ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(130ns) PrepareNewSessionHandshake:476(823ns) {SwitchToState:178(227ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.504555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435743606308484:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63297
2025-11-19T13:59:09.504572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435743606308485:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63297
2025-11-19T13:59:09.505987Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435743606308484:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.506021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435743606308403:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.508145Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435744165832274:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.508312Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435743606308485:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.508325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435744165832247:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.508349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:09.508354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435744165832247:2088] [node 6] ICP22 created new session: [4:7574435744165832289:2088]
2025-11-19T13:59:09.508359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435744165832289:2088] [node 6] ICS09 handshake done sender: [4:7574435744165832249:2089] self: [4:7574435744165832248:0] peer: [6:7574435743776154346:0] socket: 266 qp: -1
2025-11-19T13:59:09.508364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435744165832289:2088] [node 6] ICS10 traffic start
2025-11-19T13:59:09.508381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:09.508240Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435744261731142:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.508251Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435744261731142:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.511962Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435742375433261:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.511976Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435742375433261:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.519921Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435745383882099:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(42409107ns) {Configure:104(42407750ns) {ProcessConfigured:135(42112623ns) {StartInitialHandshake:157(42111744ns) {DropHandshakes:469(1046ns) {DropIncomingHandshake:441(196ns) DropOutgoingHandshake:456(160ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:476(1676ns) {SwitchToState:178(681ns)}}}}}
2025-11-19T13:59:09.524907Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435743606308435:2074] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(41002029ns) {Configure:104(41000920ns) {ProcessConfigured:135(40655556ns) {StartInitialHandshake:157(40654835ns) {DropHandshakes:469(457ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(128ns) PrepareNewSessionHandshake:476(1298ns) {SwitchToState:178(677ns)}}}}}
2025-11-19T13:59:09.525135Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435744165832287:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.525264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435744165832207:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.525327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435744165832206:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.525770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435744261731162:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3106
2025-11-19T13:59:09.526690Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435744261731162:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.526839Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435744165832263:2099] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:09.526966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435744165832253:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.526978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:09.526987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435744165832253:2091] [node 5] ICP22 created new session: [4:7574435744165832294:2091]
2025-11-19T13:59:09.526992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435744165832294:2091] [node 5] ICS09 handshake done sender: [4:7574435744165832263:2099] self: [4:7574435744165832254:0] peer: [5:7574435743606308478:0] socket: 281 qp: -1
2025-11-19T13:59:09.526998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435744165832294:2091] [node 5] ICS10 traffic start
2025-11-19T13:59:09.527006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:09.527066Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435744261731152:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.529069Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435742375433258:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.529086Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435742375433258:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.529371Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435743606308482:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.529404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435743606308404:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.533570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435744165832296:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22327
2025-11-19T13:59:09.533865Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435744165832296:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.534875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435744165832205:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.535050Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435744261731153:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.535060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435744261731153:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.535626Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435744261731153:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:09.535702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435744261731141:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.535719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:09.535725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435744261731141:2109] [node 4] ICP22 created new session: [1:7574435744261731166:2109]
2025-11-19T13:59:09.535735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435744261731166:2109] [node 4] ICS09 handshake done sender: [1:7574435744261731153:2118] self: [1:7574435744261731147:0] peer: [4:7574435744165832298:0] socket: 311 qp: -1
2025-11-19T13:59:09.535741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435744261731166:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:09.535762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:09.536073Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435744261731142:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.536117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435742375433269:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53875
2025-11-19T13:59:09.536202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435742375433270:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53875
2025-11-19T13:59:09.536178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435744261731129:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.536193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:09.536197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435744261731129:2107] [node 5] ICP22 created new session: [1:7574435744261731169:2107]
2025-11-19T13:59:09.536202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435744261731169:2107] [node 5] ICS09 handshake done sender: [1:7574435744261731142:2112] self: [1:7574435744261731131:0] peer: [5:7574435743606308483:0] socket: 284 qp: -1
2025-11-19T13:59:09.536206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435744261731169:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:09.536210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:09.539846Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435742375433269:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.544063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435745383882181:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52051
2025-11-19T13:59:09.544127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435745383882182:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52051
2025-11-19T13:59:09.544160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435745383882183:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52051
2025-11-19T13:59:09.544196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435745383882184:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52051
2025-11-19T13:59:09.545118Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435745383882169:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.545135Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435745383882169:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.548999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435743606308490:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63297
2025-11-19T13:59:09.549687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435743606308491:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63297
2025-11-19T13:59:09.549825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435744165832247:2088] [node 6] ICP35 event processing took too much time PendingConnection:278(69855305ns) {HandleHandshakeStatus:329(69854712ns) {DropIncomingHandshake:441(5943ns) DropOutgoingHandshake:456(104ns) SwitchToState:178(206ns) ProcessPendingSessionEvents:495(20534508ns) {ForwardSessionEventToSession:612(20524771ns) ForwardSessionEventToSession:612(1289ns) ForwardSessionEventToSession:612(966ns) ForwardSessionEventToSession:612(470ns) ForwardSessionEventToSession:612(555ns) ForwardSessionEventToSession:612(877ns) ForwardSessionEventToSession:612(559ns) ForwardSessionEventToSession:612(496ns) ForwardSessionEventToSession:612(622ns) ForwardSessionEventToSession:612(1005ns)}}}
2025-11-19T13:59:09.549890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435744165832205:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8716) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435744165832296:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.550212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435744165832299:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22327
2025-11-19T13:59:09.550459Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435743606308450:2086] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:09.547785Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435745383882171:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.547800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435745383882171:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.551170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435743606308402:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8716) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435743606308450:2086] held: no
2025-11-19T13:59:09.551326Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435743606308449:2085] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:09.551411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435743606308402:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.551425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:09.551431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435743606308402:2048] [node 1] ICP22 created new session: [5:7574435743606308492:2048]
2025-11-19T13:59:09.551436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435743606308492:2048] [node 1] ICS09 handshake done sender: [5:7574435743606308450:2086] self: [5:7574435743606308483:0] peer: [1:7574435744261731131:0] socket: 286 qp: -1
2025-11-19T13:59:09.551442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435743606308492:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:09.551457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:09.549537Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435742375433270:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.551874Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435742375433258:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:09.551773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435743606308426:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.551787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:09.551792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435743606308426:2065] [node 4] ICP22 created new session: [5:7574435743606308494:2065]
2025-11-19T13:59:09.551797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435743606308494:2065] [node 4] ICS09 handshake done sender: [5:7574435743606308449:2085] self: [5:7574435743606308478:0] peer: [4:7574435744165832254:0] socket: 285 qp: -1
2025-11-19T13:59:09.551804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435743606308494:2065] [node 4] ICS10 traffic start
2025-11-19T13:59:09.551809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.556243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435745383882185:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52051
2025-11-19T13:59:09.556300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435745383882158:2106] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:09.556327Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435745383882181:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.556347Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435745383882182:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.556363Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435745383882183:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.556391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435745383882099:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.559849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435744165832300:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22327
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.560132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435743606308497:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63297
2025-11-19T13:59:09.560213Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435742375433261:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:09.560221Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435742375433243:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.560227Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435742375433243:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.560424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435742375433250:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.560440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:09.560446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435742375433250:2110] [node 5] ICP22 created new session: [2:7574435742375433275:2110]
2025-11-19T13:59:09.560451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435742375433275:2110] [node 5] ICS09 handshake done sender: [2:7574435742375433261:2115] self: [2:7574435742375433251:0] peer: [5:7574435743606308486:0] socket: 283 qp: -1
2025-11-19T13:59:09.560456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435742375433275:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:09.560464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:09.560557Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435743606308484:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:09.561840Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435745383882185:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.561939Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435745383882169:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:09.562214Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435744165832296:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.562315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435745383882098:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.562339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435745383882190:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52051
2025-11-19T13:59:09.562511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435745383882166:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.562529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:09.562536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435745383882166:2109] [node 4] ICP22 created new session: [3:7574435745383882192:2109]
2025-11-19T13:59:09.562542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435745383882192:2109] [node 4] ICS09 handshake done sender: [3:7574435745383882169:2111] self: [3:7574435745383882168:0] peer: [4:7574435744165832292:0] socket: 287 qp: -1
2025-11-19T13:59:09.562548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435745383882192:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:09.562554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:09.563056Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435744165832287:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:09.563129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435744165832205:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.563143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:09.563148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435744165832205:2048] [node 1] ICP22 created new session: [4:7574435744165832302:2048]
2025-11-19T13:59:09.563154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435744165832302:2048] [node 1] ICS09 handshake done sender: [4:7574435744165832296:2118] self: [4:7574435744165832298:0] peer: [1:7574435744261731147:0] socket: 280 qp: -1
2025-11-19T13:59:09.563160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435744165832302:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:09.563165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:09.563278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435744165832206:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.563285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:09.563289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435744165832206:2049] [node 2] ICP22 created new session: [4:7574435744165832304:2049]
2025-11-19T13:59:09.563294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435744165832304:2049] [node 2] ICS09 handshake done sender: [4:7574435744165832287:2117] self: [4:7574435744165832293:0] peer: [2:7574435742375433257:0] socket: 293 qp: -1
2025-11-19T13:59:09.563297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435744165832304:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:09.563303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:09.560900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435742375433256:2113] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.560921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:09.560928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435742375433256:2113] [node 4] ICP22 created new session: [2:7574435742375433278:2113]
2025-11-19T13:59:09.560934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435742375433278:2113] [node 4] ICS09 handshake done sender: [2:7574435742375433258:2114] self: [2:7574435742375433257:0] peer: [4:7574435744165832293:0] socket: 292 qp: -1
2025-11-19T13:59:09.560939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435742375433278:2113] [node 4] ICS10 traffic start
2025-11-19T13:59:09.560946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.563929Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435744261731151:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.563940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435744261731151:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.564601Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435744261731151:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:09.564693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435744261731069:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.564712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:09.564719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435744261731069:2049] [node 3] ICP22 created new session: [1:7574435744261731172:2049]
2025-11-19T13:59:09.564723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435744261731172:2049] [node 3] ICS09 handshake done sender: [1:7574435744261731151:2116] self: [1:7574435744261731130:0] peer: [3:7574435745383882191:0] socket: 310 qp: -1
2025-11-19T13:59:09.564727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435744261731172:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:09.564732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:09.565040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435743606308403:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.565057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:09.565063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435743606308403:2049] [node 2] ICP22 created new session: [5:7574435743606308498:2049]
2025-11-19T13:59:09.565070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435743606308498:2049] [node 2] ICS09 handshake done sender: [5:7574435743606308484:2117] self: [5:7574435743606308486:0] peer: [2:7574435742375433251:0] socket: 296 qp: -1
2025-11-19T13:59:09.565076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435743606308498:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:09.565084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:09.566042Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435743606308403:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:09.564356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435744165832253:2091] [node 5] ICP35 event processing took too much time PendingConnection:278(37356728ns) {HandleHandshakeStatus:329(37355319ns) {DropIncomingHandshake:441(21ns) DropOutgoingHandshake:456(361ns) SwitchToState:178(61ns) ProcessPendingSessionEvents:495(19110624ns) {ForwardSessionEventToSession:612(19097982ns) ForwardSessionEventToSession:612(1594ns) ForwardSessionEventToSession:612(892ns) ForwardSessionEventToSession:612(583ns) ForwardSessionEventToSession:612(663ns) ForwardSessionEventToSession:612(1027ns) ForwardSessionEventToSession:612(875ns) ForwardSessionEventToSession:612(583ns) ForwardSessionEventToSession:612(473ns) ForwardSessionEventToSession:612(1076ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.571611Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435745383882171:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:09.568567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435744165832207:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21451) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435744165832274:2106] held: no
2025-11-19T13:59:09.569111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435743776154363:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34593
2025-11-19T13:59:09.571820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435744165832301:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22327
2025-11-19T13:59:09.575889Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435744165832274:2106] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:09.573882Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435743776154363:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.573923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435743776154285:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.576730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435744165832207:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.576745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:09.576751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435744165832207:2050] [node 3] ICP22 created new session: [4:7574435744165832306:2050]
2025-11-19T13:59:09.576757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435744165832306:2050] [node 3] ICS09 handshake done sender: [4:7574435744165832274:2106] self: [4:7574435744165832292:0] peer: [3:7574435745383882168:0] socket: 289 qp: -1
2025-11-19T13:59:09.576763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435744165832306:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:09.576770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:09.577122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435745383882167:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.577141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:09.577145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435745383882167:2110] [node 5] ICP22 created new session: [3:7574435745383882196:2110]
2025-11-19T13:59:09.577150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435745383882196:2110] [node 5] ICS09 handshake done sender: [3:7574435745383882171:2112] self: [3:7574435745383882170:0] peer: [5:7574435743606308489:0] socket: 288 qp: -1
2025-11-19T13:59:09.577151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435745383882098:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8716) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435745383882185:2119] held: no
2025-11-19T13:59:09.577154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435745383882196:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:09.577159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:09.577473Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435742375433243:2107] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.577622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435742375433183:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.577634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:09.577640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435742375433183:2049] [node 3] ICP22 created new session: [2:7574435742375433282:2049]
2025-11-19T13:59:09.577641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435742375433281:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53875
2025-11-19T13:59:09.577645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435742375433282:2049] [node 3] ICS09 handshake done sender: [2:7574435742375433243:2107] self: [2:7574435742375433242:0] peer: [3:7574435745383882186:0] socket: 268 qp: -1
2025-11-19T13:59:09.577651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435742375433282:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:09.577656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.578697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435743776154365:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34593
2025-11-19T13:59:09.578712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435743776154366:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34593
2025-11-19T13:59:09.579063Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435742375433281:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.579085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435742375433183:2049] [node 3] ICP18 handshake (actor [2:7574435742375433281:2120]) is too old
2025-11-19T13:59:09.579211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435742375433250:2110] [node 5] ICP35 event processing took too much time PendingConnection:278(18764190ns) {HandleHandshakeStatus:329(18763619ns) {DropIncomingHandshake:441(3636ns) DropOutgoingHandshake:456(205ns) SwitchToState:178(73ns) ProcessPendingSessionEvents:495(9732871ns) {ForwardSessionEventToSession:612(9723925ns) ForwardSessionEventToSession:612(1287ns) ForwardSessionEventToSession:612(1123ns) ForwardSessionEventToSession:612(466ns) ForwardSessionEventToSession:612(538ns) ForwardSessionEventToSession:612(835ns) ForwardSessionEventToSession:612(559ns) ForwardSessionEventToSession:612(517ns) ForwardSessionEventToSession:612(781ns) ForwardSessionEventToSession:612(677ns)}}}
2025-11-19T13:59:09.577974Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435743606308487:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.577986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435743606308487:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.578862Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435745383882181:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:09.579020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435745383882099:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.579030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:09.579036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435745383882099:2049] [node 2] ICP22 created new session: [3:7574435745383882198:2049]
2025-11-19T13:59:09.579040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435745383882198:2049] [node 2] ICS09 handshake done sender: [3:7574435745383882181:2115] self: [3:7574435745383882186:0] peer: [2:7574435742375433242:0] socket: 273 qp: -1
2025-11-19T13:59:09.579045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435745383882198:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:09.579050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:09.581353Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435743776154363:2121] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:09.581583Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435742375433267:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.581590Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435742375433267:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:09.581455Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435743776154365:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:09.581483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435743776154269:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:09.581193Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435743606308487:2119] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.583061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435743776154368:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34593
2025-11-19T13:59:09.583742Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435742375433267:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:09.583189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435743776154285:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.583204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:09.583210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435743776154285:2061] [node 5] ICP22 created new session: [6:7574435743776154369:2061]
2025-11-19T13:59:09.583215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435743776154369:2061] [node 5] ICS09 handshake done sender: [6:7574435743776154363:2121] self: [6:7574435743776154364:0] peer: [5:7574435743606308481:0] socket: 323 qp: -1
2025-11-19T13:59:09.583220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435743776154369:2061] [node 5] ICS10 traffic start
2025-11-19T13:59:09.583227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:09.583945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435742375433248:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.583964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:09.583971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435742375433248:2109] [node 6] ICP22 created new session: [2:7574435742375433285:2109]
2025-11-19T13:59:09.583975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435742375433285:2109] [node 6] ICS09 handshake done sender: [2:7574435742375433267:2117] self: [2:7574435742375433260:0] peer: [6:7574435743776154367:0] socket: 330 qp: -1
2025-11-19T13:59:09.583982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435742375433285:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:09.583997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:09.584040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435743606308435:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:09.584055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:09.584059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435743606308435:2074] [node 6] ICP22 created new session: [5:7574435743606308502:2074]
2025-11-19T13:59:09.584063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435743606308502:2074] [node 6] ICS09 handshake done sender: [5:7574435743606308487:2119] self: [5:7574435743606308481:0] peer: [6:7574435743776154364:0] socket: 274 qp: -1
2025-11-19T13:59:09.584068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435743606308502:2074] [node 6] ICS10 traffic start
2025-11-19T13:59:09.584074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:09.584210Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435743606308404:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21451) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435743606308482:2116] held: no
2025-11-19T13:59:09.584327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435743606308500:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63297
2025-11-19T13:59:09.584530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435743776154269:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29650) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435743776154365:2122] held: no
2025-11-19T13:59:09.584556Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435743776154365:2122] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:09.585108Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435743606308482:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:09.588397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435743776154269:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.588411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:09.588416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435743776154269:2049] [node 2] ICP22 created new session: [6:7574435743776154371:2049]
2025-11-19T13:59:09.588421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435743776154371:2049] [node 2] ICS09 handshake done sender: [6:7574435743776154365:2122] self: [6:7574435743776154367:0] peer: [2:7574435742375433260:0] socket: 331 qp: -1
2025-11-19T13:59:09.588426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435743776154371:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:09.588445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:09.588685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435743606308404:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.588696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:09.588701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435743606308404:2050] [node 3] ICP22 created new session: [5:7574435743606308504:2050]
2025-11-19T13:59:09.588705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435743606308504:2050] [node 3] ICS09 handshake done sender: [5:7574435743606308482:2116] self: [5:7574435743606308489:0] peer: [3:7574435745383882170:0] socket: 294 qp: -1
2025-11-19T13:59:09.588710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435743606308504:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:09.588715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:09.588011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435745383882200:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52051
2025-11-19T13:59:09.589176Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435745383882185:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:09.589308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435745383882098:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:09.589319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:09.589323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435745383882098:2048] [node 1] ICP22 created new session: [3:7574435745383882201:2048]
2025-11-19T13:59:09.589328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435745383882201:2048] [node 1] ICS09 handshake done sender: [3:7574435745383882185:2119] self: [3:7574435745383882191:0] peer: [1:7574435744261731130:0] socket: 320 qp: -1
2025-11-19T13:59:09.589333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435745383882201:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:09.589340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:09.597006Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:09.597073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435742375433265:2048] [node 1] ICS01 socket: 272 reason# EndOfStream
2025-11-19T13:59:09.597080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435742375433182:2048] [node 1] ICP30 unregister session Session# [2:7574435742375433265:2048] VirtualId# [2:7574435742375433263:0]
2025-11-19T13:59:09.597085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435742375433265:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.597111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:09.597130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:09.597442Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:09.597500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435745383882201:2048] [node 1] ICS01 socket: 320 reason# EndOfStream
2025-11-19T13:59:09.597504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435745383882098:2048] [node 1] ICP30 unregister session Session# [3:7574435745383882201:2048] VirtualId# [3:7574435745383882191:0]
2025-11-19T13:59:09.597507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435745383882201:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.597529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:09.597543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:09.605195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:09.605272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435743606308492:2048] [node 1] ICS01 socket: 286 reason# EndOfStream
2025-11-19T13:59:09.605279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435743606308402:2048] [node 1] ICP30 unregister session Session# [5:7574435743606308492:2048] VirtualId# [5:7574435743606308483:0]
2025-11-19T13:59:09.605284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435743606308492:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.605321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:09.605340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:09.605363Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:09.605416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435743776154360:2048] [node 1] ICS01 socket: 290 reason# EndOfStream
2025-11-19T13:59:09.605420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435743776154268:2048] [node 1] ICP30 unregister session Session# [6:7574435743776154360:2048] VirtualId# [6:7574435743776154355:0]
2025-11-19T13:59:09.605424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435743776154360:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.605445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:09.605461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:09.609921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:09.609997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435744165832302:2048] [node 1] ICS01 socket: 280 reason# EndOfStream
2025-11-19T13:59:09.610002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435744165832205:2048] [node 1] ICP30 unregister session Session# [4:7574435744165832302:2048] VirtualId# [4:7574435744165832298:0]
2025-11-19T13:59:09.610007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435744165832302:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.610050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:09.610072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:09.614443Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:09.614522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435745383882198:2049] [node 2] ICS01 socket: 273 reason# EndOfStream
2025-11-19T13:59:09.614528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435745383882099:2049] [node 2] ICP30 unregister session Session# [3:7574435745383882198:2049] VirtualId# [3:7574435745383882186:0]
2025-11-19T13:59:09.614533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435745383882198:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.614559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:09.614577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:09.619844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:09.619922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435744165832304:2049] [node 2] ICS01 socket: 293 reason# EndOfStream
2025-11-19T13:59:09.619929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435744165832206:2049] [node 2] ICP30 unregister session Session# [4:7574435744165832304:2049] VirtualId# [4:7574435744165832293:0]
2025-11-19T13:59:09.619934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435744165832304:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.619980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:09.620006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:09.622702Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:09.622769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435743606308498:2049] [node 2] ICS01 socket: 296 reason# EndOfStream
2025-11-19T13:59:09.622774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435743606308403:2049] [node 2] ICP30 unregister session Session# [5:7574435743606308498:2049] VirtualId# [5:7574435743606308486:0]
2025-11-19T13:59:09.622778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435743606308498:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.622819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:09.622839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:09.622901Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:09.622961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435743776154371:2049] [node 2] ICS01 socket: 331 reason# EndOfStream
2025-11-19T13:59:09.622965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435743776154269:2049] [node 2] ICP30 unregister session Session# [6:7574435743776154371:2049] VirtualId# [6:7574435743776154367:0]
2025-11-19T13:59:09.622968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435743776154371:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.622996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:09.623015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:09.690217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:09.690328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435743606308504:2050] [node 3] ICS01 socket: 294 reason# EndOfStream
2025-11-19T13:59:09.690339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435743606308404:2050] [node 3] ICP30 unregister session Session# [5:7574435743606308504:2050] VirtualId# [5:7574435743606308489:0]
2025-11-19T13:59:09.690347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435743606308504:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.690414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:09.690452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:09.690643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:09.690704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435743776154356:2050] [node 3] ICS01 socket: 276 reason# EndOfStream
2025-11-19T13:59:09.690710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435743776154270:2050] [node 3] ICP30 unregister session Session# [6:7574435743776154356:2050] VirtualId# [6:7574435743776154343:0]
2025-11-19T13:59:09.690714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435743776154356:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.690744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:09.690762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:09.712313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:09.712408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435743776154348:2059] [node 4] ICS01 socket: 267 reason# EndOfStream
2025-11-19T13:59:09.712417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435743776154284:2059] [node 4] ICP30 unregister session Session# [6:7574435743776154348:2059] VirtualId# [6:7574435743776154346:0]
2025-11-19T13:59:09.712425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435743776154348:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.712468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:09.712490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:09.729302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:59:09.729405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435743776154369:2061] [node 5] ICS01 socket: 323 reason# EndOfStream
2025-11-19T13:59:09.729415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435743776154285:2061] [node 5] ICP30 unregister session Session# [6:7574435743776154369:2061] VirtualId# [6:7574435743776154364:0]
2025-11-19T13:59:09.729423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435743776154369:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:09.729469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:59:09.729492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:09.814628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435745697869881:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:09.814648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435745697869882:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:09.856074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435741969860264:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:09.856086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435741969860265:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:09.919916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435742037722989:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:09.919935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435742037722990:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:09.952387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435742020009080:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:09.952403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435742020009081:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:09.952436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435742020009082:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:09.995838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435745022010991:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:09.995849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435745022010992:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:09.995853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435745022010993:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:10.027828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435746584254663:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:10.027841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435746584254664:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:10.027845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435746584254665:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.071464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435746332690345:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10826
2025-11-19T13:59:10.071987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435749992837238:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:10.076644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435746332690347:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:10.078173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435749992837242:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:10.077417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435746584254690:2069] [node 5] ICP01 ready to work
2025-11-19T13:59:10.077859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435746584254683:2064] [node 4] ICP01 ready to work
2025-11-19T13:59:10.077371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435746264827625:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:10.082253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435749316978313:2067] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.080069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435749992837240:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:10.085068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435746264827630:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:10.086191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435746332690350:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:10.088989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435746314976446:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:10.090249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435746264827628:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10604
2025-11-19T13:59:10.098932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435745697869882:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(26204217ns) {Configure:104(26203441ns) {ProcessConfigured:135(21879601ns) {StartInitialHandshake:157(21879265ns) {DropHandshakes:469(114ns) {DropIncomingHandshake:441(22ns) DropOutgoingHandshake:456(20ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:476(1057ns) {SwitchToState:178(501ns)}}}}}
2025-11-19T13:59:10.099319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435746264827625:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21895939ns) {Configure:104(21895380ns) {ProcessConfigured:135(21651543ns) {StartInitialHandshake:157(21651186ns) {DropHandshakes:469(473ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(43ns)} GenerateSessionVirtualId:988(126ns) PrepareNewSessionHandshake:476(598ns) {SwitchToState:178(353ns)}}}}}
2025-11-19T13:59:10.099372Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435746264827628:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.099590Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435746584254690:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(22135943ns) {Configure:104(22135696ns) {ProcessConfigured:135(21886828ns) {StartInitialHandshake:157(21886691ns) {DropHandshakes:469(276ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(102ns) PrepareNewSessionHandshake:476(955ns) {SwitchToState:178(226ns)}}}}}
2025-11-19T13:59:10.100152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435746584254663:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23497441ns) {Configure:104(23496887ns) {ProcessConfigured:135(23237124ns) {StartInitialHandshake:157(23236900ns) {DropHandshakes:469(470ns) {DropIncomingHandshake:441(86ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:476(452ns) {SwitchToState:178(34ns)}}}}}
2025-11-19T13:59:10.101259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435746584254683:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(23358153ns) {Configure:104(23357756ns) {ProcessConfigured:135(23150867ns) {StartInitialHandshake:157(23150695ns) {DropHandshakes:469(257ns) {DropIncomingHandshake:441(24ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(50ns) PrepareNewSessionHandshake:476(575ns) {SwitchToState:178(244ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.104375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435749992837255:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22878
2025-11-19T13:59:10.104436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435749992837256:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22878
2025-11-19T13:59:10.104781Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435749992837255:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.111763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435749992837254:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22878
2025-11-19T13:59:10.112047Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435749992837254:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.111572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435746314976439:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:10.111971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435749316978327:2077] [node 6] ICP01 ready to work
2025-11-19T13:59:10.113003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435746332690347:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(36306314ns) {Configure:104(36305901ns) {ProcessConfigured:135(35939530ns) {StartInitialHandshake:157(35939165ns) {DropHandshakes:469(461ns) {DropIncomingHandshake:441(104ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(69ns) PrepareNewSessionHandshake:476(620ns) {SwitchToState:178(375ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.113170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435741969860265:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(37242700ns) {Configure:104(37242240ns) {ProcessConfigured:135(36964256ns) {StartInitialHandshake:157(36964029ns) {DropHandshakes:469(812ns) {DropIncomingHandshake:441(209ns) DropOutgoingHandshake:456(178ns)} GenerateSessionVirtualId:988(294ns) PrepareNewSessionHandshake:476(476ns) {SwitchToState:178(181ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.115899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435746264827630:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(30789723ns) {Configure:104(30789131ns) {ProcessConfigured:135(30542707ns) {StartInitialHandshake:157(30542514ns) {DropHandshakes:469(481ns) {DropIncomingHandshake:441(90ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(113ns) PrepareNewSessionHandshake:476(730ns) {SwitchToState:178(414ns)}}}}}
2025-11-19T13:59:10.116053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435746264827639:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10604
2025-11-19T13:59:10.116091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435746264827640:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10604
2025-11-19T13:59:10.115433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435746332690362:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10826
2025-11-19T13:59:10.115479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435746332690363:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10826
2025-11-19T13:59:10.115493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435746332690364:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10826
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.115611Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435746314976446:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26586743ns) {Configure:104(26586219ns) {ProcessConfigured:135(26316255ns) {StartInitialHandshake:157(26315999ns) {DropHandshakes:469(412ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(83ns) PrepareNewSessionHandshake:476(459ns) {SwitchToState:178(147ns)}}}}}
2025-11-19T13:59:10.115823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435742020009081:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(26787142ns) {Configure:104(26786725ns) {ProcessConfigured:135(26314347ns) {StartInitialHandshake:157(26313945ns) {DropHandshakes:469(847ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(18ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:476(485ns) {SwitchToState:178(239ns)}}}}}
2025-11-19T13:59:10.116275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746314976453:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61504
2025-11-19T13:59:10.116335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746314976454:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61504
2025-11-19T13:59:10.116365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746314976455:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61504
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.117541Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435746264827639:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.117636Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435746332690345:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.117885Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435746332690364:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.117905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435742037722990:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.117991Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435746264827636:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.117996Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435746264827636:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.118465Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435746264827636:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:10.118544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435741969860265:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.118560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:10.118567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435741969860265:2049] [node 3] ICP22 created new session: [2:7574435746264827643:2049]
2025-11-19T13:59:10.118573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435746264827643:2049] [node 3] ICS09 handshake done sender: [2:7574435746264827636:2113] self: [2:7574435746264827623:0] peer: [3:7574435746332690365:0] socket: 295 qp: -1
2025-11-19T13:59:10.118579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435746264827643:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:10.118587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.118890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435746332690366:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10826
2025-11-19T13:59:10.119042Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435746332690364:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:10.119092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435742037722990:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.119104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:10.119108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435742037722990:2049] [node 2] ICP22 created new session: [3:7574435746332690367:2049]
2025-11-19T13:59:10.119112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435746332690367:2049] [node 2] ICS09 handshake done sender: [3:7574435746332690364:2116] self: [3:7574435746332690365:0] peer: [2:7574435746264827623:0] socket: 297 qp: -1
2025-11-19T13:59:10.119117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435746332690367:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:10.119120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:10.117417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746584254741:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19498
2025-11-19T13:59:10.119857Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435746314976454:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.121058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435742020009082:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.123527Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435746314976453:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.123955Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435742020009082:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18986) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435746314976454:2115] held: no
2025-11-19T13:59:10.124026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435746264827632:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:10.124448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435746264827641:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10604
2025-11-19T13:59:10.124554Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435746264827640:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.124565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435741969860265:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:10.124579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435741969860264:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.121225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435742037722989:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(28994295ns) {Configure:104(28993652ns) {ProcessConfigured:135(28730006ns) {StartInitialHandshake:157(28729433ns) {DropHandshakes:469(416ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(47ns)} GenerateSessionVirtualId:988(136ns) PrepareNewSessionHandshake:476(728ns) {SwitchToState:178(37ns)}}}}}
2025-11-19T13:59:10.121301Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435746332690362:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.121458Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435746332690352:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.121461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435746332690352:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.122537Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435746332690352:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:10.123008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435746332690350:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.123019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:10.123024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435746332690350:2109] [node 4] ICP22 created new session: [3:7574435746332690373:2109]
2025-11-19T13:59:10.123028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435746332690373:2109] [node 4] ICS09 handshake done sender: [3:7574435746332690352:2110] self: [3:7574435746332690351:0] peer: [4:7574435746314976456:0] socket: 282 qp: -1
2025-11-19T13:59:10.123033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435746332690373:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:10.123039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:10.127882Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435746584254741:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.127935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435746584254683:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.128258Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435746314976450:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.128266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435746314976450:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.128779Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435746314976450:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.128858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435746314976446:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.128871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:10.128891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435746314976446:2111] [node 6] ICP22 created new session: [4:7574435746314976458:2111]
2025-11-19T13:59:10.128896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435746314976458:2111] [node 6] ICS09 handshake done sender: [4:7574435746314976450:2112] self: [4:7574435746314976447:0] peer: [6:7574435746584254742:0] socket: 298 qp: -1
2025-11-19T13:59:10.128902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435746314976458:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:10.128908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:10.129186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435746264827648:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10604
2025-11-19T13:59:10.130659Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435746264827648:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.127355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435746332690355:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:10.128227Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435749992837238:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(56174972ns) {Configure:104(56174370ns) {ProcessConfigured:135(51043701ns) {StartInitialHandshake:157(51043387ns) {DropHandshakes:469(108ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(26ns)} GenerateSessionVirtualId:988(134ns) PrepareNewSessionHandshake:476(586ns) {SwitchToState:178(266ns)}}}}}
2025-11-19T13:59:10.128282Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435749992837247:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.128285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435749992837247:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.135820Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435746314976439:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(24179802ns) {Configure:104(24179013ns) {ProcessConfigured:135(23842477ns) {StartInitialHandshake:157(23842293ns) {DropHandshakes:469(1158ns) {DropIncomingHandshake:441(300ns) DropOutgoingHandshake:456(64ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(633ns) {SwitchToState:178(268ns)}}}}}
2025-11-19T13:59:10.137578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435749316978364:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36181
2025-11-19T13:59:10.137833Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435749316978364:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.137862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435745022010992:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.139952Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435746314976455:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.140011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435742020009080:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.143881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435746264827632:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19782572ns) {Configure:104(19781862ns) {ProcessConfigured:135(19499118ns) {StartInitialHandshake:157(19483871ns) {DropHandshakes:469(340ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(29ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:476(1189ns) {SwitchToState:178(39ns)}} IncomingHandshake:261(7246ns) {DropIncomingHandshake:441(14ns) PrepareNewSessionHandshake:476(142ns) {SwitchToState:178(78ns)} GenerateSessionVirtualId:988(150ns) IssueIncomingHandshakeReply:193(145ns)}}}}
2025-11-19T13:59:10.144047Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435749992837256:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.144080Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435749992837249:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.144083Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435749992837249:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.144796Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435749992837249:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:10.144864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435749992837240:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.144877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:10.144883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435749992837240:2108] [node 4] ICP22 created new session: [1:7574435749992837261:2108]
2025-11-19T13:59:10.144889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435749992837261:2108] [node 4] ICS09 handshake done sender: [1:7574435749992837249:2111] self: [1:7574435749992837248:0] peer: [4:7574435746314976463:0] socket: 284 qp: -1
2025-11-19T13:59:10.144895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435749992837261:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:10.144903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.147867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435749316978327:2077] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(35785766ns) {Configure:104(23599525ns) {ProcessConfigured:135(23305798ns) {StartInitialHandshake:157(23305189ns) {DropHandshakes:469(731ns) {DropIncomingHandshake:441(250ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:476(1285ns) {SwitchToState:178(701ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.148087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746314976464:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61504
2025-11-19T13:59:10.148150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746314976465:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61504
2025-11-19T13:59:10.148239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746314976466:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61504
2025-11-19T13:59:10.152140Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435746314976464:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.152157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435742020009080:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24153) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435746314976455:2116] held: no
2025-11-19T13:59:10.152199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435742020009081:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.152368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435749992837242:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(72244282ns) {Configure:104(72243939ns) {ProcessConfigured:135(71998974ns) {StartInitialHandshake:157(71998075ns) {DropHandshakes:469(379ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(419ns) PrepareNewSessionHandshake:476(482ns) {SwitchToState:178(138ns)}}}}}
2025-11-19T13:59:10.154103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435746332690355:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26668448ns) {Configure:104(26668035ns) {ProcessConfigured:135(26395861ns) {StartInitialHandshake:157(26380619ns) {DropHandshakes:469(531ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(49ns)} GenerateSessionVirtualId:988(1078ns) PrepareNewSessionHandshake:476(1143ns) {SwitchToState:178(929ns)}} IncomingHandshake:261(5834ns) {DropIncomingHandshake:441(89ns) PrepareNewSessionHandshake:476(128ns) {SwitchToState:178(85ns)} GenerateSessionVirtualId:988(257ns) IssueIncomingHandshakeReply:193(223ns)}}}}
2025-11-19T13:59:10.155401Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435749992837247:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-19T13:59:10.155581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746584254745:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19498
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.155861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435749316978369:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36181
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.155915Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435746584254745:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.155917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746584254744:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19498
2025-11-19T13:59:10.155959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435746584254665:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.157151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746584254747:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19498
2025-11-19T13:59:10.157349Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435746584254745:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:10.157919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435745697869881:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.157941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:10.157947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435745697869881:2048] [node 2] ICP22 created new session: [1:7574435749992837267:2048]
2025-11-19T13:59:10.157953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435749992837267:2048] [node 2] ICS09 handshake done sender: [1:7574435749992837247:2110] self: [1:7574435749992837243:0] peer: [2:7574435746264827647:0] socket: 283 qp: -1
2025-11-19T13:59:10.157959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435749992837267:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:10.157965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:10.155534Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435746332690363:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.155576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435742037722989:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.156118Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435746332690376:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.156122Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435746332690376:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.156666Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435746332690376:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:10.156734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435746332690355:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.156749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:10.156757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435746332690355:2111] [node 6] ICP22 created new session: [3:7574435746332690381:2111]
2025-11-19T13:59:10.156763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435746332690381:2111] [node 6] ICS09 handshake done sender: [3:7574435746332690376:2119] self: [3:7574435746332690375:0] peer: [6:7574435746584254746:0] socket: 316 qp: -1
2025-11-19T13:59:10.156769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435746332690381:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:10.156779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:10.159910Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435746314976454:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.159648Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435749992837258:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.159655Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435749992837258:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.160223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435749316978367:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36181
2025-11-19T13:59:10.157466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435746584254665:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.157481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:10.157486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435746584254665:2050] [node 3] ICP22 created new session: [6:7574435746584254748:2050]
2025-11-19T13:59:10.157491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435746584254748:2050] [node 3] ICS09 handshake done sender: [6:7574435746584254745:2118] self: [6:7574435746584254746:0] peer: [3:7574435746332690375:0] socket: 319 qp: -1
2025-11-19T13:59:10.157495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435746584254748:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:10.157503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:10.159489Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435746584254744:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.159527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435746584254663:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.160331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746584254743:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19498
2025-11-19T13:59:10.160767Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435746584254741:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:10.163348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435749316978368:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36181
2025-11-19T13:59:10.163831Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435749992837252:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.163840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435749992837252:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.163854Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435749316978367:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.160070Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435746264827633:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.160084Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435746264827633:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.163786Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435746264827638:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.163795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435746264827638:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.167814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435741969860264:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24153) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435746264827640:2116] held: no
2025-11-19T13:59:10.164199Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435749992837258:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:10.167763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435746584254683:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.167787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:10.167793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435746584254683:2064] [node 4] ICP22 created new session: [6:7574435746584254752:2064]
2025-11-19T13:59:10.167800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435746584254752:2064] [node 4] ICS09 handshake done sender: [6:7574435746584254741:2115] self: [6:7574435746584254742:0] peer: [4:7574435746314976447:0] socket: 303 qp: -1
2025-11-19T13:59:10.167806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435746584254752:2064] [node 4] ICS10 traffic start
2025-11-19T13:59:10.167813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:10.167891Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435749316978368:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.167921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435745022010993:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.168259Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435746314976455:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:10.168423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435742020009082:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.168434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:10.168439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435742020009082:2050] [node 3] ICP22 created new session: [4:7574435746314976469:2050]
2025-11-19T13:59:10.168444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435746314976469:2050] [node 3] ICS09 handshake done sender: [4:7574435746314976454:2115] self: [4:7574435746314976456:0] peer: [3:7574435746332690351:0] socket: 301 qp: -1
2025-11-19T13:59:10.168449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435746314976469:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:10.168454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:10.168579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435742020009080:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.168585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:10.168588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435742020009080:2048] [node 1] ICP22 created new session: [4:7574435746314976471:2048]
2025-11-19T13:59:10.168591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435746314976471:2048] [node 1] ICS09 handshake done sender: [4:7574435746314976455:2116] self: [4:7574435746314976463:0] peer: [1:7574435749992837248:0] socket: 302 qp: -1
2025-11-19T13:59:10.168594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435746314976471:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:10.168598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:10.173346Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435749316978369:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.173377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435749316978313:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.173542Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435746314976461:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.173548Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435746314976461:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.174049Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435746314976461:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:10.174117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435746314976439:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.174131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:10.174135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435746314976439:2108] [node 5] ICP22 created new session: [4:7574435746314976474:2108]
2025-11-19T13:59:10.174139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435746314976474:2108] [node 5] ICS09 handshake done sender: [4:7574435746314976461:2117] self: [4:7574435746314976449:0] peer: [5:7574435749316978372:0] socket: 277 qp: -1
2025-11-19T13:59:10.174143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435746314976474:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:10.174148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:10.174985Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435749316978313:2067] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16624) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435749316978369:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.175369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435746332690383:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10826
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.175905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435745697869881:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(17959316ns) {HandleHandshakeStatus:329(17958612ns) {DropIncomingHandshake:441(3855ns) DropOutgoingHandshake:456(122ns) SwitchToState:178(60ns) ProcessPendingSessionEvents:495(17865961ns) {ForwardSessionEventToSession:612(17854561ns) ForwardSessionEventToSession:612(1138ns) ForwardSessionEventToSession:612(546ns) ForwardSessionEventToSession:612(1236ns) ForwardSessionEventToSession:612(705ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(724ns) ForwardSessionEventToSession:612(677ns) ForwardSessionEventToSession:612(561ns) ForwardSessionEventToSession:612(1034ns)}}}
2025-11-19T13:59:10.175922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435749992837238:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.175930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:10.175935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435749992837238:2107] [node 6] ICP22 created new session: [1:7574435749992837273:2107]
2025-11-19T13:59:10.175939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435749992837273:2107] [node 6] ICS09 handshake done sender: [1:7574435749992837258:2116] self: [1:7574435749992837245:0] peer: [6:7574435746584254751:0] socket: 292 qp: -1
2025-11-19T13:59:10.175945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435749992837273:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:10.175951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:10.176312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435749992837271:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22878
2025-11-19T13:59:10.176511Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435749992837252:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:10.176576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435745697869882:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.176586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:10.176591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435745697869882:2049] [node 3] ICP22 created new session: [1:7574435749992837275:2049]
2025-11-19T13:59:10.176596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435749992837275:2049] [node 3] ICS09 handshake done sender: [1:7574435749992837252:2112] self: [1:7574435749992837244:0] peer: [3:7574435746332690379:0] socket: 287 qp: -1
2025-11-19T13:59:10.176601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435749992837275:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:10.176487Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435746332690360:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.176502Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435746332690360:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.176614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:10.178832Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435749992837271:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.187906Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435746264827633:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.188402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746584254754:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19498
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.188743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435746264827653:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10604
2025-11-19T13:59:10.188942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435746264827625:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.188961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:10.188966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435746264827625:2108] [node 5] ICP22 created new session: [2:7574435746264827654:2108]
2025-11-19T13:59:10.188972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435746264827654:2108] [node 5] ICS09 handshake done sender: [2:7574435746264827633:2112] self: [2:7574435746264827626:0] peer: [5:7574435749316978365:0] socket: 288 qp: -1
2025-11-19T13:59:10.188978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435746264827654:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:10.189000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:10.189406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435749316978374:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36181
2025-11-19T13:59:10.190215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435745022010992:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27689) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435749316978364:2114] held: no
2025-11-19T13:59:10.191803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435746332690384:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10826
2025-11-19T13:59:10.191929Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435746584254744:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:10.192023Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435746332690363:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:10.195018Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435746264827640:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.199865Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435749316978374:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.199920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435741969860264:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.199938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:10.199946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435741969860264:2048] [node 1] ICP22 created new session: [2:7574435746264827658:2048]
2025-11-19T13:59:10.199953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435746264827658:2048] [node 1] ICS09 handshake done sender: [2:7574435746264827640:2116] self: [2:7574435746264827647:0] peer: [1:7574435749992837243:0] socket: 299 qp: -1
2025-11-19T13:59:10.199963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435746264827658:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:10.199975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:10.200025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435745022010991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.201525Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435746332690360:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.201692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746584254755:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19498
2025-11-19T13:59:10.201816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435746332690347:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.201827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:10.201832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435746332690347:2108] [node 5] ICP22 created new session: [3:7574435746332690386:2108]
2025-11-19T13:59:10.201836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435746332690386:2108] [node 5] ICS09 handshake done sender: [3:7574435746332690360:2113] self: [3:7574435746332690348:0] peer: [5:7574435749316978371:0] socket: 294 qp: -1
2025-11-19T13:59:10.201840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435746332690386:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:10.201846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.201406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435749992837277:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22878
2025-11-19T13:59:10.203852Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435746264827638:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:10.204036Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435749992837264:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.204044Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435749992837264:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.204131Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435749992837277:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.204239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435749992837238:2107] [node 6] ICP18 handshake (actor [1:7574435749992837277:2119]) is too old
2025-11-19T13:59:10.203918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435749316978376:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36181
2025-11-19T13:59:10.204874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435749316978375:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36181
2025-11-19T13:59:10.205448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435746584254663:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.205470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:10.205475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435746584254663:2048] [node 1] ICP22 created new session: [6:7574435746584254756:2048]
2025-11-19T13:59:10.205481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435746584254756:2048] [node 1] ICS09 handshake done sender: [6:7574435746584254744:2117] self: [6:7574435746584254751:0] peer: [1:7574435749992837245:0] socket: 318 qp: -1
2025-11-19T13:59:10.205485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435746584254756:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:10.205503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:10.205293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435749316978378:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36181
2025-11-19T13:59:10.205650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435746264827630:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.205664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:10.205668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435746264827630:2111] [node 4] ICP22 created new session: [2:7574435746264827660:2111]
2025-11-19T13:59:10.205672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435746264827660:2111] [node 4] ICS09 handshake done sender: [2:7574435746264827638:2114] self: [2:7574435746264827631:0] peer: [4:7574435746314976467:0] socket: 305 qp: -1
2025-11-19T13:59:10.205675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435746264827660:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:10.205680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:10.205091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435745697869882:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(28486293ns) {HandleHandshakeStatus:329(28485692ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(145ns) SwitchToState:178(54ns) ProcessPendingSessionEvents:495(10226133ns) {ForwardSessionEventToSession:612(10214398ns) ForwardSessionEventToSession:612(1262ns) ForwardSessionEventToSession:612(998ns) ForwardSessionEventToSession:612(520ns) ForwardSessionEventToSession:612(830ns) ForwardSessionEventToSession:612(1080ns) ForwardSessionEventToSession:612(560ns) ForwardSessionEventToSession:612(469ns) ForwardSessionEventToSession:612(522ns) ForwardSessionEventToSession:612(911ns)}}}
2025-11-19T13:59:10.205126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435745697869882:2049] [node 3] ICP18 handshake (actor [1:7574435749992837271:2118]) is too old
2025-11-19T13:59:10.207772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435742037722989:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.207800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:10.207808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435742037722989:2048] [node 1] ICP22 created new session: [3:7574435746332690388:2048]
2025-11-19T13:59:10.207814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435746332690388:2048] [node 1] ICS09 handshake done sender: [3:7574435746332690363:2115] self: [3:7574435746332690379:0] peer: [1:7574435749992837244:0] socket: 296 qp: -1
2025-11-19T13:59:10.207822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435746332690388:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:10.207834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.208056Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435746332690383:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.208077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435746332690347:2108] [node 5] ICP18 handshake (actor [3:7574435746332690383:2120]) is too old
2025-11-19T13:59:10.208171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746314976476:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61504
2025-11-19T13:59:10.209792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435746264827625:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(20828788ns) {HandleHandshakeStatus:329(20827597ns) {DropIncomingHandshake:441(4367ns) DropOutgoingHandshake:456(180ns) SwitchToState:178(210ns) ProcessPendingSessionEvents:495(14890593ns) {ForwardSessionEventToSession:612(14880670ns) ForwardSessionEventToSession:612(1316ns) ForwardSessionEventToSession:612(860ns) ForwardSessionEventToSession:612(535ns) ForwardSessionEventToSession:612(460ns) ForwardSessionEventToSession:612(1027ns) ForwardSessionEventToSession:612(584ns) ForwardSessionEventToSession:612(495ns) ForwardSessionEventToSession:612(531ns) ForwardSessionEventToSession:612(1183ns)}}}
2025-11-19T13:59:10.208362Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435746314976464:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:10.208433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435742020009081:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.208445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:10.208450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435742020009081:2049] [node 2] ICP22 created new session: [4:7574435746314976477:2049]
2025-11-19T13:59:10.208455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435746314976477:2049] [node 2] ICS09 handshake done sender: [4:7574435746314976464:2118] self: [4:7574435746314976467:0] peer: [2:7574435746264827631:0] socket: 307 qp: -1
2025-11-19T13:59:10.208458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435746314976477:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:10.208476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:10.210280Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435746584254755:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.210310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435746584254664:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.211552Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435749992837264:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:10.211685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435749992837242:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.211696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:10.211703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435749992837242:2109] [node 5] ICP22 created new session: [1:7574435749992837279:2109]
2025-11-19T13:59:10.211707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435749992837279:2109] [node 5] ICS09 handshake done sender: [1:7574435749992837264:2117] self: [1:7574435749992837246:0] peer: [5:7574435749316978377:0] socket: 325 qp: -1
2025-11-19T13:59:10.211718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435749992837279:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:10.211724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:10.207813Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435749316978364:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:10.208217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435745022010992:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.208226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:10.208230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435745022010992:2049] [node 2] ICP22 created new session: [5:7574435749316978379:2049]
2025-11-19T13:59:10.208236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435749316978379:2049] [node 2] ICS09 handshake done sender: [5:7574435749316978364:2114] self: [5:7574435749316978365:0] peer: [2:7574435746264827626:0] socket: 290 qp: -1
2025-11-19T13:59:10.208241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435749316978379:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:10.208246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:10.208308Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435749316978369:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:10.212046Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435749316978368:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.212680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746584254759:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19498
2025-11-19T13:59:10.212229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435749316978313:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.212241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:10.212247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435749316978313:2067] [node 4] ICP22 created new session: [5:7574435749316978381:2067]
2025-11-19T13:59:10.212252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435749316978381:2067] [node 4] ICS09 handshake done sender: [5:7574435749316978369:2118] self: [5:7574435749316978372:0] peer: [4:7574435746314976449:0] socket: 321 qp: -1
2025-11-19T13:59:10.212256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435749316978381:2067] [node 4] ICS10 traffic start
2025-11-19T13:59:10.212269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:10.212464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435749316978383:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36181
2025-11-19T13:59:10.214814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435745022010993:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18986) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435749316978368:2117] held: no
2025-11-19T13:59:10.214829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435745022010993:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.214836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:10.214840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435745022010993:2050] [node 3] ICP22 created new session: [5:7574435749316978385:2050]
2025-11-19T13:59:10.214846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435749316978385:2050] [node 3] ICS09 handshake done sender: [5:7574435749316978368:2117] self: [5:7574435749316978371:0] peer: [3:7574435746332690348:0] socket: 320 qp: -1
2025-11-19T13:59:10.214850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435749316978385:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:10.214855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:10.215615Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435742037722989:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(23500505ns) {HandleHandshakeStatus:329(23500188ns) {DropIncomingHandshake:441(657ns) DropOutgoingHandshake:456(7280ns) SwitchToState:178(221ns) ProcessPendingSessionEvents:495(7715681ns) {ForwardSessionEventToSession:612(7706223ns) ForwardSessionEventToSession:612(809ns) ForwardSessionEventToSession:612(790ns) ForwardSessionEventToSession:612(588ns) ForwardSessionEventToSession:612(504ns) ForwardSessionEventToSession:612(692ns) ForwardSessionEventToSession:612(942ns) ForwardSessionEventToSession:612(548ns) ForwardSessionEventToSession:612(495ns) ForwardSessionEventToSession:612(1040ns)}}}
2025-11-19T13:59:10.216177Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435749316978374:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:10.216231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435745022010991:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.216240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:10.216245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435745022010991:2048] [node 1] ICP22 created new session: [5:7574435749316978387:2048]
2025-11-19T13:59:10.216249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435749316978387:2048] [node 1] ICS09 handshake done sender: [5:7574435749316978374:2119] self: [5:7574435749316978377:0] peer: [1:7574435749992837246:0] socket: 326 qp: -1
2025-11-19T13:59:10.216254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435749316978387:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:10.216258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:10.220246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435746584254663:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(28231353ns) {HandleHandshakeStatus:329(28230890ns) {DropIncomingHandshake:441(339ns) DropOutgoingHandshake:456(6832ns) SwitchToState:178(211ns) ProcessPendingSessionEvents:495(10289318ns) {ForwardSessionEventToSession:612(10279615ns) ForwardSessionEventToSession:612(1504ns) ForwardSessionEventToSession:612(695ns) ForwardSessionEventToSession:612(530ns) ForwardSessionEventToSession:612(557ns) ForwardSessionEventToSession:612(891ns) ForwardSessionEventToSession:612(604ns) ForwardSessionEventToSession:612(597ns) ForwardSessionEventToSession:612(551ns) ForwardSessionEventToSession:612(709ns)}}}
2025-11-19T13:59:10.220729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435746264827650:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.220739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435746264827650:2119] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.221493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746584254760:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19498
2025-11-19T13:59:10.221671Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435746264827650:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:10.221726Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435746584254759:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.221783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435746584254690:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.221939Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435746584254755:2121] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.223045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746584254762:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19498
2025-11-19T13:59:10.223123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435746264827632:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.223138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:10.223144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435746264827632:2109] [node 6] ICP22 created new session: [2:7574435746264827663:2109]
2025-11-19T13:59:10.223149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435746264827663:2109] [node 6] ICS09 handshake done sender: [2:7574435746264827650:2119] self: [2:7574435746264827646:0] peer: [6:7574435746584254758:0] socket: 334 qp: -1
2025-11-19T13:59:10.223153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435746264827663:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:10.223158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:10.223403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435746584254664:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.223415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:10.223419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435746584254664:2049] [node 2] ICP22 created new session: [6:7574435746584254763:2049]
2025-11-19T13:59:10.223423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435746584254763:2049] [node 2] ICS09 handshake done sender: [6:7574435746584254755:2121] self: [6:7574435746584254758:0] peer: [2:7574435746264827646:0] socket: 335 qp: -1
2025-11-19T13:59:10.223428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435746584254763:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:10.223432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:10.224023Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435746584254759:2122] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:10.221997Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435749316978366:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.222006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435749316978366:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.222617Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435749316978366:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:10.222686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435749316978327:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.222698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:10.222703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435749316978327:2077] [node 6] ICP22 created new session: [5:7574435749316978390:2077]
2025-11-19T13:59:10.222708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435749316978390:2077] [node 6] ICS09 handshake done sender: [5:7574435749316978366:2115] self: [5:7574435749316978363:0] peer: [6:7574435746584254761:0] socket: 279 qp: -1
2025-11-19T13:59:10.222712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435749316978390:2077] [node 6] ICS10 traffic start
2025-11-19T13:59:10.222717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:10.224096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435746584254690:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.224105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:10.224109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435746584254690:2069] [node 5] ICP22 created new session: [6:7574435746584254765:2069]
2025-11-19T13:59:10.224115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435746584254765:2069] [node 5] ICS09 handshake done sender: [6:7574435746584254759:2122] self: [6:7574435746584254761:0] peer: [5:7574435749316978363:0] socket: 328 qp: -1
2025-11-19T13:59:10.224120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435746584254765:2069] [node 5] ICS10 traffic start
2025-11-19T13:59:10.224125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:10.227846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435749992837242:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(16124837ns) {HandleHandshakeStatus:329(16124092ns) {DropIncomingHandshake:441(2565ns) DropOutgoingHandshake:456(85ns) SwitchToState:178(70ns) ProcessPendingSessionEvents:495(16064102ns) {ForwardSessionEventToSession:612(16053403ns) ForwardSessionEventToSession:612(1057ns) ForwardSessionEventToSession:612(1830ns) ForwardSessionEventToSession:612(595ns) ForwardSessionEventToSession:612(754ns) ForwardSessionEventToSession:612(648ns) ForwardSessionEventToSession:612(600ns) ForwardSessionEventToSession:612(544ns) ForwardSessionEventToSession:612(633ns) ForwardSessionEventToSession:612(499ns)}}}
2025-11-19T13:59:10.248096Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:10.248181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435746332690388:2048] [node 1] ICS07 socket disconnect 296 reason# EndOfStream
2025-11-19T13:59:10.248188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435746332690388:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:10.248217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:10.248231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435746332690388:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:10.250496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435742037722989:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24153) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:10.250507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435746332690388:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:10.250790Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435742037722989:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24153) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:10.250801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435746332690388:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:10.250807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435742037722989:2048] [node 1] ICP30 unregister session Session# [3:7574435746332690388:2048] VirtualId# [3:7574435746332690379:0]
2025-11-19T13:59:10.250823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:10.250828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435742037722989:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24153) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:10.250831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24153) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:10.251783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:10.251861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435746264827658:2048] [node 1] ICS01 socket: 299 reason# EndOfStream
2025-11-19T13:59:10.251867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435741969860264:2048] [node 1] ICP30 unregister session Session# [2:7574435746264827658:2048] VirtualId# [2:7574435746264827647:0]
2025-11-19T13:59:10.251874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435746264827658:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:10.251910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:10.251934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:10.259921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:10.259997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435749316978387:2048] [node 1] ICS01 socket: 326 reason# EndOfStream
2025-11-19T13:59:10.260005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435745022010991:2048] [node 1] ICP30 unregister session Session# [5:7574435749316978387:2048] VirtualId# [5:7574435749316978377:0]
2025-11-19T13:59:10.260012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435749316978387:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:10.260049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:10.260072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:10.260052Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:10.260121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435746584254756:2048] [node 1] ICS07 socket disconnect 318 reason# EndOfStream
2025-11-19T13:59:10.260125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435746584254756:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:10.260158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:10.260177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435746584254756:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:10.261693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:10.261763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435746314976471:2048] [node 1] ICS01 socket: 302 reason# EndOfStream
2025-11-19T13:59:10.261769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435742020009080:2048] [node 1] ICP30 unregister session Session# [4:7574435746314976471:2048] VirtualId# [4:7574435746314976463:0]
2025-11-19T13:59:10.261774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435746314976471:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:10.261819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:10.261840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:10.263227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574435742037722989:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:10.268280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435746584254663:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24153) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:10.268296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435746584254756:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:10.268300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435746584254663:2048] [node 1] ICP30 unregister session Session# [6:7574435746584254756:2048] VirtualId# [6:7574435746584254751:0]
2025-11-19T13:59:10.268316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:10.268322Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435746584254663:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24153) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:10.268324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24153) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:10.272347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:10.272413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435746332690367:2049] [node 2] ICS01 socket: 297 reason# EndOfStream
2025-11-19T13:59:10.272419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435742037722990:2049] [node 2] ICP30 unregister session Session# [3:7574435746332690367:2049] VirtualId# [3:7574435746332690365:0]
2025-11-19T13:59:10.272424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435746332690367:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:10.272451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:10.272469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:10.279652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435746584254663:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:10.294752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:10.294855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435746584254763:2049] [node 2] ICS01 socket: 335 reason# EndOfStream
2025-11-19T13:59:10.294865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435746584254664:2049] [node 2] ICP30 unregister session Session# [6:7574435746584254763:2049] VirtualId# [6:7574435746584254758:0]
2025-11-19T13:59:10.294873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435746584254763:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:10.294930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:10.294952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:10.295830Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:10.295908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435749316978379:2049] [node 2] ICS01 socket: 290 reason# EndOfStream
2025-11-19T13:59:10.295916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435745022010992:2049] [node 2] ICP30 unregister session Session# [5:7574435749316978379:2049] VirtualId# [5:7574435749316978365:0]
2025-11-19T13:59:10.295922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435749316978379:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:10.295970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:10.295992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:10.296395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:10.296470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435746314976477:2049] [node 2] ICS01 socket: 307 reason# EndOfStream
2025-11-19T13:59:10.296477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435742020009081:2049] [node 2] ICP30 unregister session Session# [4:7574435746314976477:2049] VirtualId# [4:7574435746314976467:0]
2025-11-19T13:59:10.296482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435746314976477:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:10.296529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:10.296552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:10.335733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:10.336407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435746584254748:2050] [node 3] ICS01 socket: 319 reason# EndOfStream
2025-11-19T13:59:10.336418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435746584254665:2050] [node 3] ICP30 unregister session Session# [6:7574435746584254748:2050] VirtualId# [6:7574435746584254746:0]
2025-11-19T13:59:10.336422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435746584254748:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:10.336452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:10.336474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:10.336106Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:10.336211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435749316978385:2050] [node 3] ICS01 socket: 320 reason# EndOfStream
2025-11-19T13:59:10.336219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435745022010993:2050] [node 3] ICP30 unregister session Session# [5:7574435749316978385:2050] VirtualId# [5:7574435749316978371:0]
2025-11-19T13:59:10.336228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435749316978385:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:10.336275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:10.336302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:10.348504Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:10.348602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435746584254752:2064] [node 4] ICS01 socket: 303 reason# EndOfStream
2025-11-19T13:59:10.348611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435746584254683:2064] [node 4] ICP30 unregister session Session# [6:7574435746584254752:2064] VirtualId# [6:7574435746584254742:0]
2025-11-19T13:59:10.348618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435746584254752:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:10.348662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:10.348685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:10.617275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435747239417063:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:10.617286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435747239417064:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:10.655737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435747967617648:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:10.655765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435747967617649:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:10.687887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435749340665321:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:10.687902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435749340665322:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:10.718425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435746730043817:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:10.718452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435746730043818:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:10.718456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435746730043819:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:10.755910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435748985715607:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:10.755922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435748985715608:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:10.755925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435748985715609:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:10.847924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435746578813583:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:10.847961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435746578813584:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:10.847967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435746578813585:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:10.862753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435747239417124:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.863802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435749340665368:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49931
2025-11-19T13:59:10.863904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746730043862:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T13:59:10.864456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435748985715648:2087] [node 4] ICP01 ready to work
2025-11-19T13:59:10.864933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435748985715654:2089] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.864742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435746578813612:2065] [node 5] ICP01 ready to work
2025-11-19T13:59:10.864820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435746578813613:2067] [node 4] ICP01 ready to work
2025-11-19T13:59:10.865427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435747967617712:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:10.867015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435747967617716:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:10.868604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435747967617720:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:10.869558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435746730043876:2096] [node 6] ICP01 ready to work
2025-11-19T13:59:10.869634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435746730043877:2100] [node 5] ICP01 ready to work
2025-11-19T13:59:10.871429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435749340665369:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49931
2025-11-19T13:59:10.871581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435749340665384:2097] [node 6] ICP01 ready to work
2025-11-19T13:59:10.871649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435749340665385:2100] [node 5] ICP01 ready to work
2025-11-19T13:59:10.871844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435747239417133:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:10.871925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435747239417134:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.878590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435747967617726:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6689
2025-11-19T13:59:10.878628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435747967617727:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6689
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.892296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435749340665389:2106] [node 4] ICP01 ready to work
2025-11-19T13:59:10.893043Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435749340665369:2095] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.893704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435749340665388:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49931
2025-11-19T13:59:10.893724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435749340665322:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.895004Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435747967617709:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.895016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435747967617709:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.899874Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435747967617709:2107] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.917437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435747967617649:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.917456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:10.917462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435747967617649:2049] [node 3] ICP22 created new session: [2:7574435747967617730:2049]
2025-11-19T13:59:10.917470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435747967617730:2049] [node 3] ICS09 handshake done sender: [2:7574435747967617709:2107] self: [2:7574435747967617708:0] peer: [3:7574435749340665400:0] socket: 280 qp: -1
2025-11-19T13:59:10.917477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435747967617730:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:10.917486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:10.917642Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435747967617726:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.917203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435748985715676:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46160
2025-11-19T13:59:10.917263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435748985715677:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46160
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.918415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435747239417134:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(24288636ns) {Configure:104(24287612ns) {ProcessConfigured:135(19506170ns) {StartInitialHandshake:157(19505529ns) {DropHandshakes:469(1079ns) {DropIncomingHandshake:441(137ns) DropOutgoingHandshake:456(354ns)} GenerateSessionVirtualId:988(183ns) PrepareNewSessionHandshake:476(601ns) {SwitchToState:178(365ns)}}}}}
2025-11-19T13:59:10.920370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435747239417135:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13873
2025-11-19T13:59:10.920431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435747239417136:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13873
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.926977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746578813653:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40315
2025-11-19T13:59:10.927027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746578813662:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40315
2025-11-19T13:59:10.927037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746578813663:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40315
2025-11-19T13:59:10.927048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746578813664:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40315
2025-11-19T13:59:10.924572Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435748985715676:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.925668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435748985715608:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.926347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746730043896:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T13:59:10.926388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746730043897:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T13:59:10.926399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746730043898:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.930980Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435747967617723:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.930990Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435747967617723:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.930823Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435746578813664:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.930875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435746578813584:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.930401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435749340665404:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49931
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.931910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746578813666:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40315
2025-11-19T13:59:10.932032Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435746730043896:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.932085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435746730043818:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.932282Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435746578813653:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.932305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435746578813613:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.932424Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435747967617723:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:10.932474Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435746730043885:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.932481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435746730043885:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.936234Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435747239417135:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.936299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435747239417143:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13873
2025-11-19T13:59:10.936847Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435746578813664:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:10.937644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435746578813584:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.937660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:10.937666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435746578813584:2049] [node 2] ICP22 created new session: [6:7574435746578813669:2049]
2025-11-19T13:59:10.937672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435746578813669:2049] [node 2] ICS09 handshake done sender: [6:7574435746578813664:2118] self: [6:7574435746578813665:0] peer: [2:7574435747967617722:0] socket: 313 qp: -1
2025-11-19T13:59:10.937679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435746578813669:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:10.937688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:10.937769Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435747967617727:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.937805Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435747967617719:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.937809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435747967617719:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.938378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435747967617720:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.938393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:10.938408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435747967617720:2111] [node 6] ICP22 created new session: [2:7574435747967617735:2111]
2025-11-19T13:59:10.938413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435747967617735:2111] [node 6] ICS09 handshake done sender: [2:7574435747967617723:2113] self: [2:7574435747967617722:0] peer: [6:7574435746578813665:0] socket: 307 qp: -1
2025-11-19T13:59:10.938419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435747967617735:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:10.938428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:10.937807Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435746730043885:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.940362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435747239417146:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13873
2025-11-19T13:59:10.940616Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435747967617719:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:10.940763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435747967617716:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.940780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:10.940786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435747967617716:2110] [node 4] ICP22 created new session: [2:7574435747967617740:2110]
2025-11-19T13:59:10.940791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435747967617740:2110] [node 4] ICS09 handshake done sender: [2:7574435747967617719:2112] self: [2:7574435747967617718:0] peer: [4:7574435746730043899:0] socket: 295 qp: -1
2025-11-19T13:59:10.940796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435747967617740:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:10.940804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.946294Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435746730043862:2092] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.946316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435746730043876:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.946326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:10.946332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435746730043876:2096] [node 6] ICP22 created new session: [4:7574435746730043901:2096]
2025-11-19T13:59:10.946337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435746730043901:2096] [node 6] ICS09 handshake done sender: [4:7574435746730043885:2111] self: [4:7574435746730043884:0] peer: [6:7574435746578813667:0] socket: 287 qp: -1
2025-11-19T13:59:10.946343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435746730043901:2096] [node 6] ICS10 traffic start
2025-11-19T13:59:10.946351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:10.946526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746578813668:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40315
2025-11-19T13:59:10.946642Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435749340665388:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.946770Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435749340665368:2094] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.946787Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435749340665369:2095] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:10.948153Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435747239417136:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.948701Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435747239417143:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.948352Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435747967617714:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.948363Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435747967617714:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.946122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435748985715688:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46160
2025-11-19T13:59:10.948557Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435748985715688:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.948583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435748985715607:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.949129Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435747239417138:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.949139Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435747239417138:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.949698Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435747239417146:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.949932Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435747239417138:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:10.950012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435747239417134:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.950027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:10.950032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435747239417134:2111] [node 5] ICP22 created new session: [1:7574435747239417151:2111]
2025-11-19T13:59:10.950037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435747239417151:2111] [node 5] ICS09 handshake done sender: [1:7574435747239417138:2114] self: [1:7574435747239417137:0] peer: [5:7574435748985715689:0] socket: 317 qp: -1
2025-11-19T13:59:10.950042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435747239417151:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:10.950047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:10.950342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435749340665322:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.950357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:10.950362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435749340665322:2049] [node 2] ICP22 created new session: [3:7574435749340665405:2049]
2025-11-19T13:59:10.950366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435749340665405:2049] [node 2] ICS09 handshake done sender: [3:7574435749340665369:2095] self: [3:7574435749340665400:0] peer: [2:7574435747967617708:0] socket: 281 qp: -1
2025-11-19T13:59:10.950371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435749340665405:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:10.950376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:10.950507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435749340665321:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.953209Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435746730043898:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.953257Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435746730043897:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.953286Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435746730043818:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8474) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435746730043896:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.955970Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435746578813663:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.955996Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435746578813662:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.956059Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435746578813653:2107] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:10.954609Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435748985715677:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.955667Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435748985715607:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12598) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435748985715688:2117] held: no
2025-11-19T13:59:10.955758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435748985715690:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46160
2025-11-19T13:59:10.956200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435746578813613:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.956217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:10.956223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435746578813613:2067] [node 4] ICP22 created new session: [6:7574435746578813671:2067]
2025-11-19T13:59:10.956229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435746578813671:2067] [node 4] ICS09 handshake done sender: [6:7574435746578813653:2107] self: [6:7574435746578813667:0] peer: [4:7574435746730043884:0] socket: 294 qp: -1
2025-11-19T13:59:10.956235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435746578813671:2067] [node 4] ICS10 traffic start
2025-11-19T13:59:10.956242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:10.956415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435746578813612:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.956427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435746578813585:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.957124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435746730043817:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.957239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746730043903:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T13:59:10.957419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435748985715609:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.957521Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435748985715670:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.957524Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435748985715670:2106] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.957550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435747967617743:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6689
2025-11-19T13:59:10.958910Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435747967617714:2109] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.958152Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435749340665393:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.958163Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435749340665393:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.958719Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435749340665391:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.958722Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435749340665391:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.959333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435749340665410:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49931
2025-11-19T13:59:10.959658Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435747239417126:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.959673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435747239417126:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.958074Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435748985715688:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:10.958781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435748985715607:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.958792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:10.958797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435748985715607:2048] [node 1] ICP22 created new session: [5:7574435748985715697:2048]
2025-11-19T13:59:10.958803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435748985715697:2048] [node 1] ICS09 handshake done sender: [5:7574435748985715688:2117] self: [5:7574435748985715689:0] peer: [1:7574435747239417137:0] socket: 290 qp: -1
2025-11-19T13:59:10.958808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435748985715697:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:10.958813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.964184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435746730043819:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.964308Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435746730043896:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:10.964598Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435749340665393:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.964665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435747967617712:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.964681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:10.964687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435747967617712:2108] [node 5] ICP22 created new session: [2:7574435747967617744:2108]
2025-11-19T13:59:10.964694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435747967617744:2108] [node 5] ICS09 handshake done sender: [2:7574435747967617714:2109] self: [2:7574435747967617713:0] peer: [5:7574435748985715687:0] socket: 285 qp: -1
2025-11-19T13:59:10.964699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435747967617744:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:10.964706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:10.965172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746578813676:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40315
2025-11-19T13:59:10.965188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746578813677:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40315
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.967087Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435749340665391:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:10.967214Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435747967617743:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.967243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435747967617648:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.965512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435748985715693:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46160
2025-11-19T13:59:10.965569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435748985715694:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46160
2025-11-19T13:59:10.965645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435748985715695:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46160
2025-11-19T13:59:10.966114Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435747239417128:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.966126Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435747239417128:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.966872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435747239417154:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13873
2025-11-19T13:59:10.967451Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435749340665395:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.967457Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435749340665395:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.967999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435749340665384:2097] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.968014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:10.968028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435749340665384:2097] [node 6] ICP22 created new session: [3:7574435749340665415:2097]
2025-11-19T13:59:10.968034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435749340665415:2097] [node 6] ICS09 handshake done sender: [3:7574435749340665393:2114] self: [3:7574435749340665392:0] peer: [6:7574435746578813675:0] socket: 297 qp: -1
2025-11-19T13:59:10.968039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435749340665415:2097] [node 6] ICS10 traffic start
2025-11-19T13:59:10.968045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:10.968174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435749340665385:2100] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.968179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:10.968183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435749340665385:2100] [node 5] ICP22 created new session: [3:7574435749340665417:2100]
2025-11-19T13:59:10.968186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435749340665417:2100] [node 5] ICS09 handshake done sender: [3:7574435749340665391:2113] self: [3:7574435749340665390:0] peer: [5:7574435748985715691:0] socket: 296 qp: -1
2025-11-19T13:59:10.968190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435749340665417:2100] [node 5] ICS10 traffic start
2025-11-19T13:59:10.968195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:10.968645Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435746578813662:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.968945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435746730043818:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.968958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:10.968964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435746730043818:2049] [node 2] ICP22 created new session: [4:7574435746730043910:2049]
2025-11-19T13:59:10.968968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435746730043910:2049] [node 2] ICS09 handshake done sender: [4:7574435746730043896:2116] self: [4:7574435746730043899:0] peer: [2:7574435747967617718:0] socket: 302 qp: -1
2025-11-19T13:59:10.968972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435746730043910:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:10.968978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:10.970142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746730043909:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T13:59:10.968824Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435748985715670:2106] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:10.971585Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435748985715693:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.971620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435748985715654:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.971628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:10.971632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435748985715654:2089] [node 6] ICP22 created new session: [5:7574435748985715700:2089]
2025-11-19T13:59:10.971636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435748985715700:2089] [node 6] ICS09 handshake done sender: [5:7574435748985715670:2106] self: [5:7574435748985715656:0] peer: [6:7574435746578813674:0] socket: 305 qp: -1
2025-11-19T13:59:10.971641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435748985715700:2089] [node 6] ICS10 traffic start
2025-11-19T13:59:10.971647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:10.971770Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435748985715676:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:10.971994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435746578813585:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.972009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:10.972014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435746578813585:2050] [node 3] ICP22 created new session: [6:7574435746578813678:2050]
2025-11-19T13:59:10.972019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435746578813678:2050] [node 3] ICS09 handshake done sender: [6:7574435746578813662:2116] self: [6:7574435746578813675:0] peer: [3:7574435749340665392:0] socket: 301 qp: -1
2025-11-19T13:59:10.972023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435746578813678:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:10.972030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:10.972737Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435746578813663:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.973056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435749340665413:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49931
2025-11-19T13:59:10.973074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435749340665414:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49931
2025-11-19T13:59:10.978339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435746578813612:2065] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.978357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:10.978364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435746578813612:2065] [node 5] ICP22 created new session: [6:7574435746578813680:2065]
2025-11-19T13:59:10.978369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435746578813680:2065] [node 5] ICS09 handshake done sender: [6:7574435746578813663:2117] self: [6:7574435746578813674:0] peer: [5:7574435748985715656:0] socket: 310 qp: -1
2025-11-19T13:59:10.978375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435746578813680:2065] [node 5] ICS10 traffic start
2025-11-19T13:59:10.978383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:10.979436Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435747239417126:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:10.979619Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435747239417128:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:10.979704Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435749340665395:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:10.980444Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435747239417140:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.980461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435747239417140:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.981362Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435747239417154:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.981371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435747239417064:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.981383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:10.981389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435747239417064:2049] [node 3] ICP22 created new session: [1:7574435747239417157:2049]
2025-11-19T13:59:10.981395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435747239417157:2049] [node 3] ICS09 handshake done sender: [1:7574435747239417126:2108] self: [1:7574435747239417125:0] peer: [3:7574435749340665408:0] socket: 276 qp: -1
2025-11-19T13:59:10.981401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435747239417157:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:10.981409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:10.981529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435747239417124:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.981538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:10.981542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435747239417124:2107] [node 4] ICP22 created new session: [1:7574435747239417159:2107]
2025-11-19T13:59:10.981546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435747239417159:2107] [node 4] ICS09 handshake done sender: [1:7574435747239417128:2109] self: [1:7574435747239417127:0] peer: [4:7574435746730043905:0] socket: 277 qp: -1
2025-11-19T13:59:10.981550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435747239417159:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:10.981556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:10.982046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435749340665389:2106] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:10.982060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:10.982066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435749340665389:2106] [node 4] ICP22 created new session: [3:7574435749340665420:2106]
2025-11-19T13:59:10.982070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435749340665420:2106] [node 4] ICS09 handshake done sender: [3:7574435749340665395:2115] self: [3:7574435749340665394:0] peer: [4:7574435746730043908:0] socket: 298 qp: -1
2025-11-19T13:59:10.982075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435749340665420:2106] [node 4] ICS10 traffic start
2025-11-19T13:59:10.982081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:10.982190Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435749340665368:2094] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:10.983472Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435746730043862:2092] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.983612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435748985715699:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46160
2025-11-19T13:59:10.983905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435749340665321:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.983918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:10.983922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435749340665321:2048] [node 1] ICP22 created new session: [3:7574435749340665422:2048]
2025-11-19T13:59:10.983927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435749340665422:2048] [node 1] ICS09 handshake done sender: [3:7574435749340665368:2094] self: [3:7574435749340665408:0] peer: [1:7574435747239417125:0] socket: 278 qp: -1
2025-11-19T13:59:10.983931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435749340665422:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:10.983938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:10.984755Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435748985715694:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.984765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435748985715608:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.984774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:10.984778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435748985715608:2049] [node 2] ICP22 created new session: [5:7574435748985715702:2049]
2025-11-19T13:59:10.984782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435748985715702:2049] [node 2] ICS09 handshake done sender: [5:7574435748985715676:2112] self: [5:7574435748985715687:0] peer: [2:7574435747967617713:0] socket: 291 qp: -1
2025-11-19T13:59:10.984786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435748985715702:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:10.984793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:10.984907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435748985715654:2089] [node 6] ICP18 handshake (actor [5:7574435748985715693:2119]) is too old
2025-11-19T13:59:10.985016Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435748985715677:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:10.993024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435749340665410:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.993068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435749340665385:2100] [node 5] ICP18 handshake (actor [3:7574435749340665410:2119]) is too old
2025-11-19T13:59:10.993315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435748985715648:2087] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:10.993332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435748985715609:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:10.993343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:10.993348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435748985715609:2050] [node 3] ICP22 created new session: [5:7574435748985715706:2050]
2025-11-19T13:59:10.993353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435748985715706:2050] [node 3] ICS09 handshake done sender: [5:7574435748985715677:2113] self: [5:7574435748985715691:0] peer: [3:7574435749340665390:0] socket: 303 qp: -1
2025-11-19T13:59:10.993359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435748985715706:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:10.993365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:10.994818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435747239417064:2049] [node 3] ICP18 handshake (actor [1:7574435747239417154:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.995128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435747967617747:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6689
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:10.995250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435746730043912:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T13:59:10.995519Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435746730043889:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.995526Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435746730043889:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:10.996678Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435749340665413:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:10.996701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435749340665389:2106] [node 4] ICP18 handshake (actor [3:7574435749340665413:2120]) is too old
2025-11-19T13:59:11.002431Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435747239417140:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:11.002676Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435747967617743:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:11.002749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435747967617648:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.002770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:11.002775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435747967617648:2048] [node 1] ICP22 created new session: [2:7574435752262585044:2048]
2025-11-19T13:59:11.002781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435752262585044:2048] [node 1] ICS09 handshake done sender: [2:7574435747967617743:2117] self: [2:7574435747967617746:0] peer: [1:7574435747239417139:0] socket: 324 qp: -1
2025-11-19T13:59:11.002787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435752262585044:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:11.002796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.003686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435746578813682:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40315
2025-11-19T13:59:11.003941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435747239417063:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.003960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:11.003965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435747239417063:2048] [node 2] ICP22 created new session: [1:7574435751534384459:2048]
2025-11-19T13:59:11.003970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435751534384459:2048] [node 2] ICS09 handshake done sender: [1:7574435747239417140:2115] self: [1:7574435747239417139:0] peer: [2:7574435747967617746:0] socket: 282 qp: -1
2025-11-19T13:59:11.003974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435751534384459:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:11.003981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:11.006761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435746730043817:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.006780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:11.006786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435746730043817:2048] [node 1] ICP22 created new session: [4:7574435751025011209:2048]
2025-11-19T13:59:11.006791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435751025011209:2048] [node 1] ICS09 handshake done sender: [4:7574435746730043862:2092] self: [4:7574435746730043905:0] peer: [1:7574435747239417127:0] socket: 279 qp: -1
2025-11-19T13:59:11.006797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435751025011209:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:11.006805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:11.006863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435746730043817:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:11.012200Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435746578813682:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.012239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435746578813583:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.012446Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435746730043897:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:11.012607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435746730043819:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3011) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435746730043897:2117] held: no
2025-11-19T13:59:11.012627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435746730043819:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.012637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:11.012641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435746730043819:2050] [node 3] ICP22 created new session: [4:7574435751025011212:2050]
2025-11-19T13:59:11.012645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435751025011212:2050] [node 3] ICS09 handshake done sender: [4:7574435746730043897:2117] self: [4:7574435746730043908:0] peer: [3:7574435749340665394:0] socket: 311 qp: -1
2025-11-19T13:59:11.012651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435751025011212:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:11.012655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.013042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435753280683004:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46160
2025-11-19T13:59:11.014284Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435747239417142:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.014297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435747239417142:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.015164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435750873780980:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40315
2025-11-19T13:59:11.015287Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435746730043889:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:11.015497Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435748985715694:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:11.015580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435748985715648:2087] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.015594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:11.015599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435748985715648:2087] [node 4] ICP22 created new session: [5:7574435753280683005:2087]
2025-11-19T13:59:11.015605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435753280683005:2087] [node 4] ICS09 handshake done sender: [5:7574435748985715694:2120] self: [5:7574435748985715705:0] peer: [4:7574435746730043888:0] socket: 327 qp: -1
2025-11-19T13:59:11.015610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435753280683005:2087] [node 4] ICS10 traffic start
2025-11-19T13:59:11.015620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:11.016655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435746730043877:2100] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.016671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:11.016676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435746730043877:2100] [node 5] ICP22 created new session: [4:7574435751025011214:2100]
2025-11-19T13:59:11.016680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435751025011214:2100] [node 5] ICS09 handshake done sender: [4:7574435746730043889:2113] self: [4:7574435746730043888:0] peer: [5:7574435748985715705:0] socket: 323 qp: -1
2025-11-19T13:59:11.016684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435751025011214:2100] [node 5] ICS10 traffic start
2025-11-19T13:59:11.016691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:11.020026Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435747239417142:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:11.020220Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435746578813682:2123] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:11.020288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435746578813583:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.020301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:11.020307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435746578813583:2048] [node 1] ICP22 created new session: [6:7574435750873780981:2048]
2025-11-19T13:59:11.020312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435750873780981:2048] [node 1] ICS09 handshake done sender: [6:7574435746578813682:2123] self: [6:7574435750873780979:0] peer: [1:7574435747239417141:0] socket: 325 qp: -1
2025-11-19T13:59:11.020317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435750873780981:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:11.020325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:11.027983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435747239417133:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.028008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:11.028015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435747239417133:2110] [node 6] ICP22 created new session: [1:7574435751534384462:2110]
2025-11-19T13:59:11.028021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435751534384462:2110] [node 6] ICS09 handshake done sender: [1:7574435747239417142:2116] self: [1:7574435747239417141:0] peer: [6:7574435750873780979:0] socket: 309 qp: -1
2025-11-19T13:59:11.028027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435751534384462:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:11.028036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:11.034470Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:11.034573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435752262585044:2048] [node 1] ICS01 socket: 324 reason# EndOfStream
2025-11-19T13:59:11.034581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435747967617648:2048] [node 1] ICP30 unregister session Session# [2:7574435752262585044:2048] VirtualId# [2:7574435747967617746:0]
2025-11-19T13:59:11.034588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435752262585044:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.034633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.034654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.035220Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:11.035293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435749340665422:2048] [node 1] ICS01 socket: 278 reason# EndOfStream
2025-11-19T13:59:11.035299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435749340665321:2048] [node 1] ICP30 unregister session Session# [3:7574435749340665422:2048] VirtualId# [3:7574435749340665408:0]
2025-11-19T13:59:11.035304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435749340665422:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.035326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.035342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.039914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:11.039990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435751025011209:2048] [node 1] ICS01 socket: 279 reason# EndOfStream
2025-11-19T13:59:11.039997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435746730043817:2048] [node 1] ICP30 unregister session Session# [4:7574435751025011209:2048] VirtualId# [4:7574435746730043905:0]
2025-11-19T13:59:11.040003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435751025011209:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.040043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.040064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.040094Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:11.040154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435750873780981:2048] [node 1] ICS01 socket: 325 reason# EndOfStream
2025-11-19T13:59:11.040158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435746578813583:2048] [node 1] ICP30 unregister session Session# [6:7574435750873780981:2048] VirtualId# [6:7574435750873780979:0]
2025-11-19T13:59:11.040162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435750873780981:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.040184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.040200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.040225Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:11.040282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435748985715697:2048] [node 1] ICS01 socket: 290 reason# EndOfStream
2025-11-19T13:59:11.040286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435748985715607:2048] [node 1] ICP30 unregister session Session# [5:7574435748985715697:2048] VirtualId# [5:7574435748985715689:0]
2025-11-19T13:59:11.040290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435748985715697:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.040316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.040335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.044509Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:11.044583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435749340665405:2049] [node 2] ICS01 socket: 281 reason# EndOfStream
2025-11-19T13:59:11.044589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435749340665322:2049] [node 2] ICP30 unregister session Session# [3:7574435749340665405:2049] VirtualId# [3:7574435749340665400:0]
2025-11-19T13:59:11.044595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435749340665405:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.044628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:11.044648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:11.047817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:11.047892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435748985715702:2049] [node 2] ICS01 socket: 291 reason# EndOfStream
2025-11-19T13:59:11.047899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435748985715608:2049] [node 2] ICP30 unregister session Session# [5:7574435748985715702:2049] VirtualId# [5:7574435748985715687:0]
2025-11-19T13:59:11.047903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435748985715702:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.047943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:11.047962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:11.047989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:11.048048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435746730043910:2049] [node 2] ICS01 socket: 302 reason# EndOfStream
2025-11-19T13:59:11.048052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435746730043818:2049] [node 2] ICP30 unregister session Session# [4:7574435746730043910:2049] VirtualId# [4:7574435746730043899:0]
2025-11-19T13:59:11.048056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435746730043910:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.048077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:11.048098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:11.048120Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:11.048192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435746578813669:2049] [node 2] ICS01 socket: 313 reason# EndOfStream
2025-11-19T13:59:11.048197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435746578813584:2049] [node 2] ICP30 unregister session Session# [6:7574435746578813669:2049] VirtualId# [6:7574435746578813665:0]
2025-11-19T13:59:11.048201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435746578813669:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.048225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:11.048244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:11.060051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:11.060139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435746578813678:2050] [node 3] ICS01 socket: 301 reason# EndOfStream
2025-11-19T13:59:11.060148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435746578813585:2050] [node 3] ICP30 unregister session Session# [6:7574435746578813678:2050] VirtualId# [6:7574435746578813675:0]
2025-11-19T13:59:11.060155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435746578813678:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.060201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:11.060224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:11.062248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:11.062341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435748985715706:2050] [node 3] ICS01 socket: 303 reason# EndOfStream
2025-11-19T13:59:11.062347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435748985715609:2050] [node 3] ICP30 unregister session Session# [5:7574435748985715706:2050] VirtualId# [5:7574435748985715691:0]
2025-11-19T13:59:11.062352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435748985715706:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.062389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:11.062409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:11.074226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:11.074317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435746578813671:2067] [node 4] ICS01 socket: 294 reason# EndOfStream
2025-11-19T13:59:11.074328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435746578813613:2067] [node 4] ICP30 unregister session Session# [6:7574435746578813671:2067] VirtualId# [6:7574435746578813667:0]
2025-11-19T13:59:11.074336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435746578813671:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.074414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:11.074441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:11.136089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435753305119070:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:11.136107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435753305119071:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:11.156075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435752193059028:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:11.156089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435752193059029:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:11.175972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435752636127917:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:11.175991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435752636127918:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:11.214029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435750927694100:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:11.214044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435750927694101:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:11.214048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435750927694102:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:11.243272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435753056064702:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:11.243284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435753056064703:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:11.243289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435753056064704:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:11.279936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435752372042062:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:11.279947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435752372042063:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:11.279951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435752372042064:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:11.315990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435753056064718:2062] [node 6] ICP01 ready to work
2025-11-19T13:59:11.316252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435750927694149:2093] [node 5] ICP01 ready to work
2025-11-19T13:59:11.316360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435750927694150:2095] [node 6] ICP01 ready to work
2025-11-19T13:59:11.316456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435752372042100:2075] [node 4] ICP01 ready to work
2025-11-19T13:59:11.316557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435752372042103:2077] [node 5] ICP01 ready to work
2025-11-19T13:59:11.317307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435753305119132:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:11.317382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435753305119133:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:11.320125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435753056064732:2066] [node 4] ICP01 ready to work
2025-11-19T13:59:11.333025Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435752372042100:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16432001ns) {Configure:104(16431736ns) {ProcessConfigured:135(16222798ns) {StartInitialHandshake:157(16222414ns) {DropHandshakes:469(313ns) {DropIncomingHandshake:441(33ns) DropOutgoingHandshake:456(33ns)} GenerateSessionVirtualId:988(64ns) PrepareNewSessionHandshake:476(779ns) {SwitchToState:178(29ns)}}}}}
2025-11-19T13:59:11.335533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435750927694150:2095] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(18642978ns) {Configure:104(18641491ns) {ProcessConfigured:135(18425206ns) {StartInitialHandshake:157(18424712ns) {DropHandshakes:469(189ns) {DropIncomingHandshake:441(32ns) DropOutgoingHandshake:456(28ns)} GenerateSessionVirtualId:988(181ns) PrepareNewSessionHandshake:476(747ns) {SwitchToState:178(575ns)}}}}}
2025-11-19T13:59:11.339989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435753305119132:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(22570583ns) {Configure:104(22570178ns) {ProcessConfigured:135(22242381ns) {StartInitialHandshake:157(22242110ns) {DropHandshakes:469(160ns) {DropIncomingHandshake:441(32ns) DropOutgoingHandshake:456(37ns)} GenerateSessionVirtualId:988(50ns) PrepareNewSessionHandshake:476(813ns) {SwitchToState:178(466ns)}}}}}
2025-11-19T13:59:11.340178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435753305119138:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.344208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435752372042103:2077] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(24008820ns) {Configure:104(24007578ns) {ProcessConfigured:135(23753233ns) {StartInitialHandshake:157(23752700ns) {DropHandshakes:469(643ns) {DropIncomingHandshake:441(44ns) DropOutgoingHandshake:456(47ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:476(596ns) {SwitchToState:178(484ns)}}}}}
2025-11-19T13:59:11.345532Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435753056064704:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(25340041ns) {Configure:104(25339547ns) {ProcessConfigured:135(25081825ns) {StartInitialHandshake:157(25081515ns) {DropHandshakes:469(425ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:476(2016ns) {SwitchToState:178(986ns)}}}}}
2025-11-19T13:59:11.346315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435753056064773:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23306
2025-11-19T13:59:11.347250Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435753056064773:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.348019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435752372042140:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63613
2025-11-19T13:59:11.347857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435753305119133:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23955916ns) {Configure:104(23954458ns) {ProcessConfigured:135(23651029ns) {StartInitialHandshake:157(23650377ns) {DropHandshakes:469(384ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(49ns)} GenerateSessionVirtualId:988(180ns) PrepareNewSessionHandshake:476(835ns) {SwitchToState:178(738ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.359205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435753305119149:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65054
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.359966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435750927694178:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62501
2025-11-19T13:59:11.360481Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435753305119149:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.360790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435752636127980:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:11.360999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435752636127981:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:11.361080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435752636127982:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:11.367891Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435752372042140:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.367932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435752372042103:2077] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.368488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435752193059098:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:11.368665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435752193059099:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:11.368735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435752193059100:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.371890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435752193059028:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(54054559ns) {Configure:104(54053782ns) {ProcessConfigured:135(53722754ns) {StartInitialHandshake:157(53722415ns) {DropHandshakes:469(209ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(20ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(814ns) {SwitchToState:178(39ns)}}}}}
2025-11-19T13:59:11.372665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435752193059092:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48679
2025-11-19T13:59:11.372719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435752193059093:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48679
2025-11-19T13:59:11.373368Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435750927694178:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.373397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435750927694100:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.373498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435752193059094:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48679
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.379903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435752636127980:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18761118ns) {Configure:104(18760133ns) {ProcessConfigured:135(18357542ns) {StartInitialHandshake:157(18356435ns) {DropHandshakes:469(1316ns) {DropIncomingHandshake:441(113ns) DropOutgoingHandshake:456(635ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:476(1585ns) {SwitchToState:178(536ns)}}}}}
2025-11-19T13:59:11.381079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752636127984:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49443
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.384185Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752636127984:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.385622Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435753056064730:2073] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.385637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435753056064730:2073] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.386314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435752193059098:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(17488326ns) {Configure:104(17487502ns) {ProcessConfigured:135(17175407ns) {StartInitialHandshake:157(17174914ns) {DropHandshakes:469(700ns) {DropIncomingHandshake:441(97ns) DropOutgoingHandshake:456(135ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:476(1004ns) {SwitchToState:178(713ns)}}}}}
2025-11-19T13:59:11.386385Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435752193059093:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.386405Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435752193059092:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.386702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435752193059108:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48679
2025-11-19T13:59:11.388021Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435753056064730:2073] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:11.388225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435753056064718:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.388239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:11.388246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435753056064718:2062] [node 6] ICP22 created new session: [5:7574435753056064783:2062]
2025-11-19T13:59:11.388252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435753056064783:2062] [node 6] ICS09 handshake done sender: [5:7574435753056064730:2073] self: [5:7574435753056064725:0] peer: [6:7574435752372042141:0] socket: 292 qp: -1
2025-11-19T13:59:11.388259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435753056064783:2062] [node 6] ICS10 traffic start
2025-11-19T13:59:11.388268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.391138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435753305119151:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65054
2025-11-19T13:59:11.391159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435753305119152:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65054
2025-11-19T13:59:11.391499Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435753305119137:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.391510Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435753305119137:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.391319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435752193059095:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48679
2025-11-19T13:59:11.391798Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435752193059095:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.391830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435752193059028:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.392045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435750927694180:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62501
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.396619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752636127998:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49443
2025-11-19T13:59:11.396681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752636127999:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49443
2025-11-19T13:59:11.396714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752636128000:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49443
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.396785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435753056064732:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(59873020ns) {Configure:104(59872123ns) {ProcessConfigured:135(59612123ns) {StartInitialHandshake:157(59611673ns) {DropHandshakes:469(458ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(400ns) PrepareNewSessionHandshake:476(2100ns) {SwitchToState:178(2040ns)}}}}}
2025-11-19T13:59:11.396963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435753056064732:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.397054Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435753305119141:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.397067Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435753305119141:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.397206Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435752193059094:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.403882Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752636127998:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.408030Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435753305119137:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:11.408189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435753305119132:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.408206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:11.408213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435753305119132:2107] [node 4] ICP22 created new session: [1:7574435753305119154:2107]
2025-11-19T13:59:11.408219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435753305119154:2107] [node 4] ICS09 handshake done sender: [1:7574435753305119137:2110] self: [1:7574435753305119134:0] peer: [4:7574435750927694179:0] socket: 296 qp: -1
2025-11-19T13:59:11.408225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435753305119154:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:11.408233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:11.407859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435753056064718:2062] [node 6] ICP35 event processing took too much time PendingConnection:278(19599312ns) {HandleHandshakeStatus:329(19598426ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(339ns) SwitchToState:178(67ns) ProcessPendingSessionEvents:495(15698128ns) {ForwardSessionEventToSession:612(15685774ns) ForwardSessionEventToSession:612(2420ns) ForwardSessionEventToSession:612(797ns) ForwardSessionEventToSession:612(600ns) ForwardSessionEventToSession:612(752ns) ForwardSessionEventToSession:612(1334ns) ForwardSessionEventToSession:612(509ns) ForwardSessionEventToSession:612(557ns) ForwardSessionEventToSession:612(703ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.411953Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435750927694161:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.411968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435750927694161:2104] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.412547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435750927694181:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62501
2025-11-19T13:59:11.412856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752636127999:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.412739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435753056064782:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23306
2025-11-19T13:59:11.413188Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435752193059108:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.413283Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752636128000:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.413303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435752636127918:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.415816Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435753056064782:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.415847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435753056064703:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.415952Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435752193059104:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.415961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435752193059104:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.415877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435752372042142:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63613
2025-11-19T13:59:11.417178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752636128005:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49443
2025-11-19T13:59:11.417302Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435750927694161:2104] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:11.417478Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435750927694180:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.417496Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435750927694178:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:11.417605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752636128004:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49443
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.416476Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435752193059097:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.416484Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435752193059097:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.417233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435753056064788:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23306
2025-11-19T13:59:11.418926Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435753056064773:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:11.419018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435753056064732:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.419035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:11.419042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435753056064732:2066] [node 4] ICP22 created new session: [5:7574435753056064789:2066]
2025-11-19T13:59:11.419048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435753056064789:2066] [node 4] ICS09 handshake done sender: [5:7574435753056064773:2111] self: [5:7574435753056064785:0] peer: [4:7574435750927694152:0] socket: 291 qp: -1
2025-11-19T13:59:11.419057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435753056064789:2066] [node 4] ICS10 traffic start
2025-11-19T13:59:11.419065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:11.419565Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435752193059097:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:11.417355Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435753305119151:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.418538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435753305119157:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65054
2025-11-19T13:59:11.419898Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435753305119157:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.420040Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435753305119139:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.420043Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435753305119139:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.420995Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435753305119141:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:11.421077Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435753305119139:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:11.419882Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435752372042142:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.419916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435752372042062:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.419998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435750927694149:2093] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.420016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:11.420023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435750927694149:2093] [node 5] ICP22 created new session: [4:7574435750927694183:2093]
2025-11-19T13:59:11.420029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435750927694183:2093] [node 5] ICS09 handshake done sender: [4:7574435750927694161:2104] self: [4:7574435750927694152:0] peer: [5:7574435753056064785:0] socket: 287 qp: -1
2025-11-19T13:59:11.420035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435750927694183:2093] [node 5] ICS10 traffic start
2025-11-19T13:59:11.420067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:11.423967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435750927694102:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.423997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435750927694100:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.424006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:11.424010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435750927694100:2048] [node 1] ICP22 created new session: [4:7574435750927694186:2048]
2025-11-19T13:59:11.424014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435750927694186:2048] [node 1] ICS09 handshake done sender: [4:7574435750927694178:2115] self: [4:7574435750927694179:0] peer: [1:7574435753305119134:0] socket: 302 qp: -1
2025-11-19T13:59:11.424018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435750927694186:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:11.424023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:11.424512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435752372042144:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63613
2025-11-19T13:59:11.424730Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435752636128000:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:11.424831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435752636127918:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.424847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:11.424853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435752636127918:2049] [node 2] ICP22 created new session: [3:7574435752636128006:2049]
2025-11-19T13:59:11.424858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435752636128006:2049] [node 2] ICS09 handshake done sender: [3:7574435752636128000:2118] self: [3:7574435752636128003:0] peer: [2:7574435752193059096:0] socket: 318 qp: -1
2025-11-19T13:59:11.424864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435752636128006:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:11.424870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.424797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435752193059029:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.424822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:11.424828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435752193059029:2049] [node 3] ICP22 created new session: [2:7574435752193059120:2049]
2025-11-19T13:59:11.424834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435752193059120:2049] [node 3] ICS09 handshake done sender: [2:7574435752193059097:2114] self: [2:7574435752193059096:0] peer: [3:7574435752636128003:0] socket: 311 qp: -1
2025-11-19T13:59:11.424839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435752193059120:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:11.424845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:11.425150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435752193059119:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48679
2025-11-19T13:59:11.425287Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435753305119152:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.425302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435753305119070:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.425310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:11.425315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435753305119070:2048] [node 2] ICP22 created new session: [1:7574435753305119162:2048]
2025-11-19T13:59:11.425319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435753305119162:2048] [node 2] ICS09 handshake done sender: [1:7574435753305119141:2112] self: [1:7574435753305119140:0] peer: [2:7574435752193059114:0] socket: 298 qp: -1
2025-11-19T13:59:11.425325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435753305119162:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:11.425331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:11.425411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435753305119133:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.425419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:11.425422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435753305119133:2108] [node 6] ICP22 created new session: [1:7574435753305119164:2108]
2025-11-19T13:59:11.425426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435753305119164:2108] [node 6] ICS09 handshake done sender: [1:7574435753305119139:2111] self: [1:7574435753305119136:0] peer: [6:7574435752372042143:0] socket: 297 qp: -1
2025-11-19T13:59:11.425430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435753305119164:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:11.425433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.425835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435753056064791:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23306
2025-11-19T13:59:11.426026Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435752193059104:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:11.425975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435753305119070:2048] [node 2] ICP18 handshake (actor [1:7574435753305119152:2117]) is too old
2025-11-19T13:59:11.426194Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435753056064782:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.426500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435753056064792:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23306
2025-11-19T13:59:11.426583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435753056064703:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.426594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:11.426599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435753056064703:2049] [node 2] ICP22 created new session: [5:7574435753056064793:2049]
2025-11-19T13:59:11.426604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435753056064793:2049] [node 2] ICS09 handshake done sender: [5:7574435753056064782:2116] self: [5:7574435753056064787:0] peer: [2:7574435752193059103:0] socket: 314 qp: -1
2025-11-19T13:59:11.426608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435753056064793:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:11.426613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:11.429550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435752193059099:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.429569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:11.429573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435752193059099:2108] [node 5] ICP22 created new session: [2:7574435752193059123:2108]
2025-11-19T13:59:11.429578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435752193059123:2108] [node 5] ICS09 handshake done sender: [2:7574435752193059104:2116] self: [2:7574435752193059103:0] peer: [5:7574435753056064787:0] socket: 313 qp: -1
2025-11-19T13:59:11.429583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435752193059123:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:11.429589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:11.429739Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435752193059095:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:11.429976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435752193059028:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7934) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435752193059095:2113] held: no
2025-11-19T13:59:11.429996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435752193059028:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.430002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:11.430006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435752193059028:2048] [node 1] ICP22 created new session: [2:7574435752193059125:2048]
2025-11-19T13:59:11.430009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435752193059125:2048] [node 1] ICS09 handshake done sender: [2:7574435752193059095:2113] self: [2:7574435752193059114:0] peer: [1:7574435753305119140:0] socket: 301 qp: -1
2025-11-19T13:59:11.430013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435752193059125:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:11.430019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:11.430335Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435753056064792:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.430361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435753056064702:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.430508Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435753305119145:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.430515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435753305119145:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.431115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435753056064796:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23306
2025-11-19T13:59:11.431986Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752636127985:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.432001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435752636127985:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.432492Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435753305119145:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:11.432592Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435752372042140:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:11.432698Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435753056064792:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:11.432595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435753305119138:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.432625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:11.432632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435753305119138:2109] [node 5] ICP22 created new session: [1:7574435753305119168:2109]
2025-11-19T13:59:11.432637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435753305119168:2109] [node 5] ICS09 handshake done sender: [1:7574435753305119145:2114] self: [1:7574435753305119144:0] peer: [5:7574435753056064795:0] socket: 310 qp: -1
2025-11-19T13:59:11.432643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435753305119168:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:11.432649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:11.434048Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752636128004:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.434167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435752636127917:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.434409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435750927694190:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62501
2025-11-19T13:59:11.434419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435750927694191:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62501
2025-11-19T13:59:11.435860Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435752636127985:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:11.435980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435752636127980:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.435997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:11.436002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435752636127980:2107] [node 4] ICP22 created new session: [3:7574435752636128011:2107]
2025-11-19T13:59:11.436007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435752636128011:2107] [node 4] ICS09 handshake done sender: [3:7574435752636127985:2111] self: [3:7574435752636127983:0] peer: [4:7574435750927694185:0] socket: 305 qp: -1
2025-11-19T13:59:11.436013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435752636128011:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:11.436018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:11.436512Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435753305119143:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.436525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435753305119143:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.437151Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435753305119143:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:11.437304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435753305119071:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.437318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:11.437324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435753305119071:2049] [node 3] ICP22 created new session: [1:7574435753305119171:2049]
2025-11-19T13:59:11.437329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435753305119171:2049] [node 3] ICS09 handshake done sender: [1:7574435753305119143:2113] self: [1:7574435753305119142:0] peer: [3:7574435752636128009:0] socket: 322 qp: -1
2025-11-19T13:59:11.437334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435753305119171:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:11.437340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.437568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752636128014:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49443
2025-11-19T13:59:11.437795Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435752636128004:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:11.438970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435752636127917:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.438988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:11.438994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435752636127917:2048] [node 1] ICP22 created new session: [3:7574435752636128015:2048]
2025-11-19T13:59:11.439000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435752636128015:2048] [node 1] ICS09 handshake done sender: [3:7574435752636128004:2119] self: [3:7574435752636128009:0] peer: [1:7574435753305119142:0] socket: 326 qp: -1
2025-11-19T13:59:11.439005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435752636128015:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:11.439012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:11.439840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435752372042103:2077] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.439857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:11.439863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435752372042103:2077] [node 5] ICP22 created new session: [6:7574435752372042148:2077]
2025-11-19T13:59:11.439869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435752372042148:2077] [node 5] ICS09 handshake done sender: [6:7574435752372042140:2115] self: [6:7574435752372042141:0] peer: [5:7574435753056064725:0] socket: 295 qp: -1
2025-11-19T13:59:11.439875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435752372042148:2077] [node 5] ICS10 traffic start
2025-11-19T13:59:11.439881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:11.440782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435753056064702:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.440800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:11.440806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435753056064702:2048] [node 1] ICP22 created new session: [5:7574435753056064797:2048]
2025-11-19T13:59:11.440811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435753056064797:2048] [node 1] ICS09 handshake done sender: [5:7574435753056064792:2119] self: [5:7574435753056064795:0] peer: [1:7574435753305119144:0] socket: 316 qp: -1
2025-11-19T13:59:11.440817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435753056064797:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:11.440824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.442227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435753056064702:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:11.442353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435753056064799:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23306
2025-11-19T13:59:11.444627Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435750927694102:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9153) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435750927694180:2116] held: no
2025-11-19T13:59:11.444677Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435750927694190:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.444686Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435750927694180:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:11.444783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435750927694102:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.444798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:11.444803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435750927694102:2050] [node 3] ICP22 created new session: [4:7574435750927694192:2050]
2025-11-19T13:59:11.444808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435750927694192:2050] [node 3] ICS09 handshake done sender: [4:7574435750927694180:2116] self: [4:7574435750927694185:0] peer: [3:7574435752636127983:0] socket: 308 qp: -1
2025-11-19T13:59:11.444813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435750927694192:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:11.444820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:11.444958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435750927694101:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.446121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435750927694195:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62501
2025-11-19T13:59:11.446482Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435750927694190:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:11.446564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435750927694101:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.446577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:11.446581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435750927694101:2049] [node 2] ICP22 created new session: [4:7574435750927694196:2049]
2025-11-19T13:59:11.446585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435750927694196:2049] [node 2] ICS09 handshake done sender: [4:7574435750927694190:2118] self: [4:7574435750927694194:0] peer: [2:7574435752193059101:0] socket: 342 qp: -1
2025-11-19T13:59:11.446590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435750927694196:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:11.446595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:11.444544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435752372042145:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63613
2025-11-19T13:59:11.445084Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435752193059107:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.445095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435752193059107:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.446331Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435752193059107:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:11.447439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435752193059098:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.447450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:11.447455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435752193059098:2107] [node 4] ICP22 created new session: [2:7574435752193059129:2107]
2025-11-19T13:59:11.447460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435752193059129:2107] [node 4] ICS09 handshake done sender: [2:7574435752193059107:2118] self: [2:7574435752193059101:0] peer: [4:7574435750927694194:0] socket: 300 qp: -1
2025-11-19T13:59:11.447465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435752193059129:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:11.447471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.448133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435752372042146:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63613
2025-11-19T13:59:11.448198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435752372042147:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63613
2025-11-19T13:59:11.448348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435752372042062:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7934) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435752372042142:2116] held: no
2025-11-19T13:59:11.448490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435752372042150:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63613
2025-11-19T13:59:11.450150Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435752372042150:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.450189Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435752372042146:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.450207Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435752372042142:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:11.449146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435750927694198:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62501
2025-11-19T13:59:11.452074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435753056064799:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.452108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435753056064704:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.451643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435752372042062:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.451660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:11.451666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435752372042062:2048] [node 1] ICP22 created new session: [6:7574435752372042152:2048]
2025-11-19T13:59:11.451672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435752372042152:2048] [node 1] ICS09 handshake done sender: [6:7574435752372042142:2116] self: [6:7574435752372042143:0] peer: [1:7574435753305119136:0] socket: 303 qp: -1
2025-11-19T13:59:11.451679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435752372042152:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:11.451699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:11.451818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435752372042064:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.451833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435752372042100:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.452243Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435750927694171:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.452251Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435750927694171:2108] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.452809Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435750927694198:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.452781Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435752193059106:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.452789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435752193059106:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.453313Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435752193059106:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:11.453466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435752193059100:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.453482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:11.453488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435752193059100:2109] [node 6] ICP22 created new session: [2:7574435752193059132:2109]
2025-11-19T13:59:11.453493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435752193059132:2109] [node 6] ICS09 handshake done sender: [2:7574435752193059106:2117] self: [2:7574435752193059105:0] peer: [6:7574435752372042156:0] socket: 337 qp: -1
2025-11-19T13:59:11.453497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435752193059132:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:11.453504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:11.452589Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435752372042147:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.452680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435752372042063:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.454092Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435750927694171:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:11.452373Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752636127993:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.452393Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435752636127993:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.459904Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752636127991:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.459917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435752636127991:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.461247Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435752636127991:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:11.462502Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435752636127993:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:11.462801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435752636127981:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.462814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:11.462818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435752636127981:2108] [node 5] ICP22 created new session: [3:7574435752636128019:2108]
2025-11-19T13:59:11.462823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435752636128019:2108] [node 5] ICS09 handshake done sender: [3:7574435752636127991:2114] self: [3:7574435752636127990:0] peer: [5:7574435753056064800:0] socket: 341 qp: -1
2025-11-19T13:59:11.462826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435752636128019:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:11.462831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:11.462933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435752636127982:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.462939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:11.462942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435752636127982:2109] [node 6] ICP22 created new session: [3:7574435752636128021:2109]
2025-11-19T13:59:11.462946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435752636128021:2109] [node 6] ICS09 handshake done sender: [3:7574435752636127993:2115] self: [3:7574435752636127992:0] peer: [6:7574435752372042154:0] socket: 307 qp: -1
2025-11-19T13:59:11.462949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435752636128021:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:11.462953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:11.461110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435753056064801:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23306
2025-11-19T13:59:11.461410Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435753056064799:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:11.461478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435753056064704:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.461488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:11.461493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435753056064704:2050] [node 3] ICP22 created new session: [5:7574435753056064802:2050]
2025-11-19T13:59:11.461499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435753056064802:2050] [node 3] ICS09 handshake done sender: [5:7574435753056064799:2121] self: [5:7574435753056064800:0] peer: [3:7574435752636127990:0] socket: 323 qp: -1
2025-11-19T13:59:11.461504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435753056064802:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:11.461511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:11.462548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435750927694150:2095] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.462568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:11.462576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435750927694150:2095] [node 6] ICP22 created new session: [4:7574435750927694201:2095]
2025-11-19T13:59:11.462581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435750927694201:2095] [node 6] ICS09 handshake done sender: [4:7574435750927694171:2108] self: [4:7574435750927694153:0] peer: [6:7574435752372042155:0] socket: 334 qp: -1
2025-11-19T13:59:11.462586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435750927694201:2095] [node 6] ICS10 traffic start
2025-11-19T13:59:11.462592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:11.464929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435752372042063:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10174) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435752372042147:2120] held: no
2025-11-19T13:59:11.465768Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435752372042064:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9153) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435752372042150:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.466478Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435752372042100:2075] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9716) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435752372042146:2119] held: no
2025-11-19T13:59:11.466508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435752372042157:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63613
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.466647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435752372042158:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63613
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.466773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435752372042159:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63613
2025-11-19T13:59:11.466912Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435752372042146:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:11.466972Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435752372042147:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:11.467021Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435752372042150:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:11.467071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435752372042100:2075] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.467083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:11.467089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435752372042100:2075] [node 4] ICP22 created new session: [6:7574435752372042160:2075]
2025-11-19T13:59:11.467095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435752372042160:2075] [node 4] ICS09 handshake done sender: [6:7574435752372042146:2119] self: [6:7574435752372042155:0] peer: [4:7574435750927694153:0] socket: 338 qp: -1
2025-11-19T13:59:11.467100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435752372042160:2075] [node 4] ICS10 traffic start
2025-11-19T13:59:11.467106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:11.467190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435752372042063:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.467195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:11.467199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435752372042063:2049] [node 2] ICP22 created new session: [6:7574435752372042162:2049]
2025-11-19T13:59:11.467202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435752372042162:2049] [node 2] ICS09 handshake done sender: [6:7574435752372042147:2120] self: [6:7574435752372042156:0] peer: [2:7574435752193059105:0] socket: 339 qp: -1
2025-11-19T13:59:11.467206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435752372042162:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:11.467210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:11.467266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435752372042064:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.467271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:11.467274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435752372042064:2050] [node 3] ICP22 created new session: [6:7574435752372042164:2050]
2025-11-19T13:59:11.467278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435752372042164:2050] [node 3] ICS09 handshake done sender: [6:7574435752372042150:2121] self: [6:7574435752372042154:0] peer: [3:7574435752636127992:0] socket: 288 qp: -1
2025-11-19T13:59:11.467281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435752372042164:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:11.467285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:11.475820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:11.475904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435752193059125:2048] [node 1] ICS01 socket: 301 reason# EndOfStream
2025-11-19T13:59:11.475912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435752193059028:2048] [node 1] ICP30 unregister session Session# [2:7574435752193059125:2048] VirtualId# [2:7574435752193059114:0]
2025-11-19T13:59:11.475918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435752193059125:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.475969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.475989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.476744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:11.476812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435752636128015:2048] [node 1] ICS01 socket: 326 reason# EndOfStream
2025-11-19T13:59:11.476817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435752636127917:2048] [node 1] ICP30 unregister session Session# [3:7574435752636128015:2048] VirtualId# [3:7574435752636128009:0]
2025-11-19T13:59:11.476822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435752636128015:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.476851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.476873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.479891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:11.479956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435752372042152:2048] [node 1] ICS01 socket: 303 reason# EndOfStream
2025-11-19T13:59:11.479962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435752372042062:2048] [node 1] ICP30 unregister session Session# [6:7574435752372042152:2048] VirtualId# [6:7574435752372042143:0]
2025-11-19T13:59:11.479967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435752372042152:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.480011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.480033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.480154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:11.480211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435750927694186:2048] [node 1] ICS01 socket: 302 reason# EndOfStream
2025-11-19T13:59:11.480215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435750927694100:2048] [node 1] ICP30 unregister session Session# [4:7574435750927694186:2048] VirtualId# [4:7574435750927694179:0]
2025-11-19T13:59:11.480219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435750927694186:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.480246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.480266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.481749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:11.481819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435753056064797:2048] [node 1] ICS01 socket: 316 reason# EndOfStream
2025-11-19T13:59:11.481825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435753056064702:2048] [node 1] ICP30 unregister session Session# [5:7574435753056064797:2048] VirtualId# [5:7574435753056064795:0]
2025-11-19T13:59:11.481830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435753056064797:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.481874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.481895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.492388Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:11.492469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435752636128006:2049] [node 2] ICS07 socket disconnect 318 reason# EndOfStream
2025-11-19T13:59:11.492474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435752636128006:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.492504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:11.492518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435752636128006:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:11.494494Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435752636127918:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10174) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:11.494505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435752636128006:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:11.494757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435752636127918:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10174) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:11.494767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435752636128006:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:11.494772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435752636127918:2049] [node 2] ICP30 unregister session Session# [3:7574435752636128006:2049] VirtualId# [3:7574435752636128003:0]
2025-11-19T13:59:11.494786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:11.494793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435752636127918:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10174) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:11.494795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10174) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:11.495720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:11.496094Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:11.496156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435752372042162:2049] [node 2] ICS01 socket: 339 reason# EndOfStream
2025-11-19T13:59:11.496160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435752372042063:2049] [node 2] ICP30 unregister session Session# [6:7574435752372042162:2049] VirtualId# [6:7574435752372042156:0]
2025-11-19T13:59:11.496165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435752372042162:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.496187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:11.496206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:11.495960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435750927694196:2049] [node 2] ICS01 socket: 342 reason# EndOfStream
2025-11-19T13:59:11.495968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435750927694101:2049] [node 2] ICP30 unregister session Session# [4:7574435750927694196:2049] VirtualId# [4:7574435750927694194:0]
2025-11-19T13:59:11.495974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435750927694196:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.496019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:11.496040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:11.499251Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:11.499327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435753056064793:2049] [node 2] ICS01 socket: 314 reason# EndOfStream
2025-11-19T13:59:11.499334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435753056064703:2049] [node 2] ICP30 unregister session Session# [5:7574435753056064793:2049] VirtualId# [5:7574435753056064787:0]
2025-11-19T13:59:11.499339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435753056064793:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.499381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:11.499400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:11.515841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:11.515927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435753056064802:2050] [node 3] ICS01 socket: 323 reason# EndOfStream
2025-11-19T13:59:11.515936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435753056064704:2050] [node 3] ICP30 unregister session Session# [5:7574435753056064802:2050] VirtualId# [5:7574435753056064800:0]
2025-11-19T13:59:11.515943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435753056064802:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.515998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:11.516022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:11.516068Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:11.516143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435752372042164:2050] [node 3] ICS01 socket: 288 reason# EndOfStream
2025-11-19T13:59:11.516149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435752372042064:2050] [node 3] ICP30 unregister session Session# [6:7574435752372042164:2050] VirtualId# [6:7574435752372042154:0]
2025-11-19T13:59:11.516156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435752372042164:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.516215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:11.516234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:11.527878Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:11.527973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435752372042160:2075] [node 4] ICS01 socket: 338 reason# EndOfStream
2025-11-19T13:59:11.527982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435752372042100:2075] [node 4] ICP30 unregister session Session# [6:7574435752372042160:2075] VirtualId# [6:7574435752372042155:0]
2025-11-19T13:59:11.527989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435752372042160:2075] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.528046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:11.528069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:11.535896Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:59:11.535982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435752372042148:2077] [node 5] ICS01 socket: 295 reason# EndOfStream
2025-11-19T13:59:11.535992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435752372042103:2077] [node 5] ICP30 unregister session Session# [6:7574435752372042148:2077] VirtualId# [6:7574435752372042141:0]
2025-11-19T13:59:11.535999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435752372042148:2077] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.536034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:59:11.536053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:11.592160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435751945118250:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:11.592174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435751945118251:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:11.632052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435751396712896:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:11.632059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435751396712895:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:11.658632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435752016446444:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:11.658644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435752016446445:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:11.684202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435753113816298:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:11.684215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435753113816299:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:11.684220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435753113816300:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:11.695936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435752335041992:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:11.695947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435752335041993:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:11.695952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435752335041994:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:11.732041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435750131780759:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:11.732111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435750131780758:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:11.736194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435750131780760:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:11.760260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435751945118311:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.761555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435751396712955:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31051
2025-11-19T13:59:11.761669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752016446504:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59494
2025-11-19T13:59:11.761776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435752335042052:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43797
2025-11-19T13:59:11.764085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435752335042055:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:11.764094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435750131780792:2074] [node 4] ICP01 ready to work
2025-11-19T13:59:11.764155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435750131780794:2076] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.768817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435752335042057:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43797
2025-11-19T13:59:11.768845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752016446505:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59494
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.769000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435751396712959:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31051
2025-11-19T13:59:11.771435Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752016446505:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.771851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435752016446445:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.772931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435752016446510:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.774844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435753113816360:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4855
2025-11-19T13:59:11.773518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435751396712964:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:11.773576Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435751396712957:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.773582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435751396712957:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.773995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435751396712965:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.775149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435752335042060:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43797
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.775442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435753113816361:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4855
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.775873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752016446513:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59494
2025-11-19T13:59:11.775559Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435752335042057:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.775678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435752335042062:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:11.775995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435752335042062:2113] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.776137Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435750131780809:2093] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.776146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435750131780809:2093] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.777055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435751945118322:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:11.777796Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435750131780809:2093] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:11.778955Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435751396712957:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:11.779797Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435752016446505:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:11.779902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435751396712963:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.795918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435752335042064:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43797
2025-11-19T13:59:11.795875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435751945118322:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18614271ns) {Configure:104(18612998ns) {ProcessConfigured:135(18318944ns) {StartInitialHandshake:157(18318227ns) {DropHandshakes:469(533ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(84ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:476(1368ns) {SwitchToState:178(321ns)}}}}}
2025-11-19T13:59:11.796176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435751945118326:2112] [node 6] ICP01 ready to work
2025-11-19T13:59:11.796597Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435753113816299:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19104442ns) {Configure:104(19103987ns) {ProcessConfigured:135(18837280ns) {StartInitialHandshake:157(18836914ns) {DropHandshakes:469(367ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:476(954ns) {SwitchToState:178(129ns)}}}}}
2025-11-19T13:59:11.797012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435753113816298:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24313285ns) {Configure:104(24312380ns) {ProcessConfigured:135(23943003ns) {StartInitialHandshake:157(23942594ns) {DropHandshakes:469(567ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(92ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:476(544ns) {SwitchToState:178(348ns)}}}}}
2025-11-19T13:59:11.796601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435750131780794:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.796620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:11.796627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435750131780794:2076] [node 5] ICP22 created new session: [6:7574435750131780835:2076]
2025-11-19T13:59:11.796633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435750131780835:2076] [node 5] ICS09 handshake done sender: [6:7574435750131780809:2093] self: [6:7574435750131780808:0] peer: [5:7574435752335042063:0] socket: 294 qp: -1
2025-11-19T13:59:11.796642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435750131780835:2076] [node 5] ICS10 traffic start
2025-11-19T13:59:11.796658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:11.798223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435753113816370:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:11.798285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435753113816371:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.796069Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435752335042052:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.796401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435752335041992:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.801051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435750131780838:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64885
2025-11-19T13:59:11.803460Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435752335042057:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.803532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435752335042066:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43797
2025-11-19T13:59:11.802070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435751396712896:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.802089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:11.802094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435751396712896:2049] [node 3] ICP22 created new session: [2:7574435751396712975:2049]
2025-11-19T13:59:11.802101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435751396712975:2049] [node 3] ICS09 handshake done sender: [2:7574435751396712957:2108] self: [2:7574435751396712956:0] peer: [3:7574435752016446506:0] socket: 295 qp: -1
2025-11-19T13:59:11.802109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435751396712975:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:11.802128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:11.802290Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435751396712955:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.802366Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435751396712959:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.802774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435751396712895:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.802805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435751396712977:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31051
2025-11-19T13:59:11.803695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435752335042062:2113] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.803707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:11.803711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435752335042062:2113] [node 6] ICP22 created new session: [5:7574435752335042067:2113]
2025-11-19T13:59:11.803716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435752335042067:2113] [node 6] ICS09 handshake done sender: [5:7574435752335042057:2110] self: [5:7574435752335042063:0] peer: [6:7574435750131780808:0] socket: 297 qp: -1
2025-11-19T13:59:11.803721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435752335042067:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:11.803727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.804754Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435753113816360:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.804850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435753113816377:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4855
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.805721Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435752016446510:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(32732063ns) {Configure:104(32731649ns) {ProcessConfigured:135(32541099ns) {StartInitialHandshake:157(32540750ns) {DropHandshakes:469(304ns) {DropIncomingHandshake:441(78ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(67ns) PrepareNewSessionHandshake:476(1003ns) {SwitchToState:178(26ns)}}}}}
2025-11-19T13:59:11.805740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435752016446445:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.805749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:11.805755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435752016446445:2049] [node 2] ICP22 created new session: [3:7574435752016446517:2049]
2025-11-19T13:59:11.805760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435752016446517:2049] [node 2] ICS09 handshake done sender: [3:7574435752016446505:2108] self: [3:7574435752016446506:0] peer: [2:7574435751396712956:0] socket: 298 qp: -1
2025-11-19T13:59:11.805766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435752016446517:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:11.805782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:11.806038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435752016446519:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:11.806114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435752016446520:2113] [node 4] ICP01 ready to work
2025-11-19T13:59:11.806786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752016446521:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59494
2025-11-19T13:59:11.809569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435753113816300:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(28386758ns) {Configure:104(28385664ns) {ProcessConfigured:135(28106158ns) {StartInitialHandshake:157(28105435ns) {DropHandshakes:469(573ns) {DropIncomingHandshake:441(43ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:476(882ns) {SwitchToState:178(727ns)}}}}}
2025-11-19T13:59:11.809750Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435753113816377:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.809781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435753113816299:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.810309Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435750131780838:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.810530Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435752335042066:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.805478Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435751396712977:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.805525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435751396712983:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31051
2025-11-19T13:59:11.810619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435750131780759:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.810793Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752016446521:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.820128Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435752335042060:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.820126Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435751945118317:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.820141Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435751945118317:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.820744Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435751945118313:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.820747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435751945118313:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.822928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435751945118330:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51
2025-11-19T13:59:11.822981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435751945118331:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51
2025-11-19T13:59:11.823180Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435751945118317:2110] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:11.824034Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435751396712974:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.824049Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435751396712974:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.824363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435752335042055:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.824430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435752335041993:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.824257Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435751945118313:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:11.826032Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435751396712968:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.826049Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435751396712968:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.829605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435750131780842:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64885
2025-11-19T13:59:11.829831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435752335042077:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43797
2025-11-19T13:59:11.829867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435752335042078:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43797
2025-11-19T13:59:11.830271Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435752335042052:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:11.830358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435752335041992:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.830373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:11.830380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435752335041992:2048] [node 1] ICP22 created new session: [5:7574435752335042079:2048]
2025-11-19T13:59:11.830385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435752335042079:2048] [node 1] ICS09 handshake done sender: [5:7574435752335042052:2107] self: [5:7574435752335042065:0] peer: [1:7574435751945118316:0] socket: 293 qp: -1
2025-11-19T13:59:11.830392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435752335042079:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:11.830398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:11.832091Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435750131780842:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.832122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435750131780760:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.832118Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435752335042077:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.832150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435752335041994:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.831969Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435751945118326:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(35738182ns) {Configure:104(35737622ns) {ProcessConfigured:135(35436625ns) {StartInitialHandshake:157(35436017ns) {DropHandshakes:469(414ns) {DropIncomingHandshake:441(37ns) DropOutgoingHandshake:456(104ns)} GenerateSessionVirtualId:988(314ns) PrepareNewSessionHandshake:476(868ns) {SwitchToState:178(566ns)}}}}}
2025-11-19T13:59:11.832057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435751945118331:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.832067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435751945118311:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.832076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:11.832084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435751945118311:2107] [node 5] ICP22 created new session: [1:7574435751945118333:2107]
2025-11-19T13:59:11.832090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435751945118333:2107] [node 5] ICS09 handshake done sender: [1:7574435751945118317:2110] self: [1:7574435751945118316:0] peer: [5:7574435752335042065:0] socket: 290 qp: -1
2025-11-19T13:59:11.832095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435751945118333:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:11.832105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:11.832254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435751945118250:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.832260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:11.832267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435751945118250:2048] [node 2] ICP22 created new session: [1:7574435751945118335:2048]
2025-11-19T13:59:11.832271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435751945118335:2048] [node 2] ICS09 handshake done sender: [1:7574435751945118313:2108] self: [1:7574435751945118312:0] peer: [2:7574435751396712980:0] socket: 288 qp: -1
2025-11-19T13:59:11.832275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435751945118335:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:11.832281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:11.832358Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435751945118330:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.832960Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752016446504:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.833057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752016446523:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.833060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435752016446523:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.836988Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752016446516:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.837004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435752016446516:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.839776Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435753113816373:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.839797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435753113816373:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.842475Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435751396712970:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.842488Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435751396712970:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.844735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435752335042082:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43797
2025-11-19T13:59:11.844984Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435753113816361:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.845111Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435751396712974:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:11.845254Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435751396712970:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:11.845424Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435752335042060:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:11.845485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435752335041993:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.845502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:11.845509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435752335041993:2049] [node 2] ICP22 created new session: [5:7574435752335042083:2049]
2025-11-19T13:59:11.845515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435752335042083:2049] [node 2] ICS09 handshake done sender: [5:7574435752335042060:2112] self: [5:7574435752335042075:0] peer: [2:7574435751396712969:0] socket: 306 qp: -1
2025-11-19T13:59:11.845521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435752335042083:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:11.845529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:11.845750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435751396712963:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.845766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:11.845771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435751396712963:2110] [node 4] ICP22 created new session: [2:7574435751396712987:2110]
2025-11-19T13:59:11.845775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435751396712987:2110] [node 4] ICS09 handshake done sender: [2:7574435751396712974:2115] self: [2:7574435751396712973:0] peer: [4:7574435753113816380:0] socket: 319 qp: -1
2025-11-19T13:59:11.845781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435751396712987:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:11.845786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:11.845860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435751396712965:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.845865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:11.845868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435751396712965:2112] [node 5] ICP22 created new session: [2:7574435751396712989:2112]
2025-11-19T13:59:11.845872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435751396712989:2112] [node 5] ICS09 handshake done sender: [2:7574435751396712970:2114] self: [2:7574435751396712969:0] peer: [5:7574435752335042075:0] socket: 305 qp: -1
2025-11-19T13:59:11.845875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435751396712989:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:11.845880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:11.847013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435753113816299:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435753113816377:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.848362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435751945118341:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51
2025-11-19T13:59:11.849657Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435751945118341:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.849693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435751945118250:2048] [node 2] ICP18 handshake (actor [1:7574435751945118341:2117]) is too old
2025-11-19T13:59:11.850388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435751396712895:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13056) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435751396712955:2107] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.852469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435751396712993:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31051
2025-11-19T13:59:11.852598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435752016446444:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.852622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752016446529:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59494
2025-11-19T13:59:11.856228Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435751396712968:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:11.856385Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752016446529:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.856407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435751396712964:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.856427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:11.856433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435751396712964:2111] [node 6] ICP22 created new session: [2:7574435751396712994:2111]
2025-11-19T13:59:11.856439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435751396712994:2111] [node 6] ICS09 handshake done sender: [2:7574435751396712968:2113] self: [2:7574435751396712967:0] peer: [6:7574435750131780841:0] socket: 304 qp: -1
2025-11-19T13:59:11.856445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435751396712994:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:11.856453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:11.856676Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435751396712955:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:11.856771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435751396712895:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.856783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:11.856787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435751396712895:2048] [node 1] ICP22 created new session: [2:7574435751396712996:2048]
2025-11-19T13:59:11.856793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435751396712996:2048] [node 1] ICS09 handshake done sender: [2:7574435751396712955:2107] self: [2:7574435751396712980:0] peer: [1:7574435751945118312:0] socket: 291 qp: -1
2025-11-19T13:59:11.856797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435751396712996:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:11.856802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.857164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435752335042085:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43797
2025-11-19T13:59:11.857293Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435752016446523:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:11.857489Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435752335042077:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:11.857597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435752016446519:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.857612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:11.857617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435752016446519:2112] [node 5] ICP22 created new session: [3:7574435752016446533:2112]
2025-11-19T13:59:11.857622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435752016446533:2112] [node 5] ICS09 handshake done sender: [3:7574435752016446523:2116] self: [3:7574435752016446522:0] peer: [5:7574435752335042081:0] socket: 326 qp: -1
2025-11-19T13:59:11.857627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435752016446533:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:11.857632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:11.857867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435752335041994:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.857880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:11.857884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435752335041994:2050] [node 3] ICP22 created new session: [5:7574435752335042086:2050]
2025-11-19T13:59:11.857888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435752335042086:2050] [node 3] ICS09 handshake done sender: [5:7574435752335042077:2118] self: [5:7574435752335042081:0] peer: [3:7574435752016446522:0] socket: 328 qp: -1
2025-11-19T13:59:11.857893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435752335042086:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:11.857898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:11.859390Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435750131780759:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435750131780838:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.859927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435753113816383:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4855
2025-11-19T13:59:11.859949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435753113816384:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4855
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.860971Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435751945118315:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.860986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435751945118315:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.865641Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435752016446516:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:11.865747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435752016446510:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.865767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:11.865773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435752016446510:2109] [node 6] ICP22 created new session: [3:7574435752016446537:2109]
2025-11-19T13:59:11.865779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435752016446537:2109] [node 6] ICS09 handshake done sender: [3:7574435752016446516:2114] self: [3:7574435752016446511:0] peer: [6:7574435750131780843:0] socket: 325 qp: -1
2025-11-19T13:59:11.865785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435752016446537:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:11.865793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:11.866402Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435751396712965:2112] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:11.868220Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435753113816377:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:11.868603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435753113816385:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4855
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.875876Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435753113816373:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:11.875937Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435753113816383:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.875971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435753113816298:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.872071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435752335042088:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43797
2025-11-19T13:59:11.876163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435753113816299:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.876178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:11.876183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435753113816299:2049] [node 2] ICP22 created new session: [4:7574435753113816388:2049]
2025-11-19T13:59:11.876188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435753113816388:2049] [node 2] ICS09 handshake done sender: [4:7574435753113816377:2116] self: [4:7574435753113816380:0] peer: [2:7574435751396712973:0] socket: 322 qp: -1
2025-11-19T13:59:11.876196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435753113816388:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:11.876219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:11.877037Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435752335042066:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.877289Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435751945118315:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:11.877763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435750131780760:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26344) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435750131780842:2116] held: no
2025-11-19T13:59:11.879173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435751945118251:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.879192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:11.879197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435751945118251:2049] [node 3] ICP22 created new session: [1:7574435751945118344:2049]
2025-11-19T13:59:11.879201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435751945118344:2049] [node 3] ICS09 handshake done sender: [1:7574435751945118315:2109] self: [1:7574435751945118314:0] peer: [3:7574435752016446530:0] socket: 289 qp: -1
2025-11-19T13:59:11.879206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435751945118344:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:11.879212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.877396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752016446539:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59494
2025-11-19T13:59:11.879087Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435752016446504:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:11.879462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435752016446444:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.879471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:11.879475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435752016446444:2048] [node 1] ICP22 created new session: [3:7574435752016446540:2048]
2025-11-19T13:59:11.879479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435752016446540:2048] [node 1] ICS09 handshake done sender: [3:7574435752016446504:2107] self: [3:7574435752016446530:0] peer: [1:7574435751945118314:0] socket: 292 qp: -1
2025-11-19T13:59:11.879484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435752016446540:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:11.879490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:11.877928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435752335042055:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.877944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:11.877952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435752335042055:2109] [node 4] ICP22 created new session: [5:7574435752335042089:2109]
2025-11-19T13:59:11.877957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435752335042089:2109] [node 4] ICS09 handshake done sender: [5:7574435752335042066:2115] self: [5:7574435752335042074:0] peer: [4:7574435753113816372:0] socket: 320 qp: -1
2025-11-19T13:59:11.877963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435752335042089:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:11.877970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:11.884004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435751945118343:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51
2025-11-19T13:59:11.884068Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435751945118325:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.884081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435751945118325:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.884286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435753113816370:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.884309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:11.884314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435753113816370:2112] [node 5] ICP22 created new session: [4:7574435753113816390:2112]
2025-11-19T13:59:11.884319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435753113816390:2112] [node 5] ICS09 handshake done sender: [4:7574435753113816373:2114] self: [4:7574435753113816372:0] peer: [5:7574435752335042074:0] socket: 317 qp: -1
2025-11-19T13:59:11.884325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435753113816390:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:11.884331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.888174Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435753113816385:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.888211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435753113816300:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.888107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435752016446542:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59494
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.891816Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435751945118343:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.892026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435750131780846:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64885
2025-11-19T13:59:11.892074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435750131780847:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64885
2025-11-19T13:59:11.892265Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752016446542:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.892302Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435752016446525:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.892305Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435752016446525:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.893050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435750131780848:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64885
2025-11-19T13:59:11.893203Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435750131780842:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:11.893383Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435750131780838:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:11.893278Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435752016446525:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:11.893771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435750131780760:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.893783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:11.893787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435750131780760:2050] [node 3] ICP22 created new session: [6:7574435750131780849:2050]
2025-11-19T13:59:11.893793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435750131780849:2050] [node 3] ICS09 handshake done sender: [6:7574435750131780842:2116] self: [6:7574435750131780843:0] peer: [3:7574435752016446511:0] socket: 327 qp: -1
2025-11-19T13:59:11.893797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435750131780849:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:11.893802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:11.893472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435752016446520:2113] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.893489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:11.893495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435752016446520:2113] [node 4] ICP22 created new session: [3:7574435752016446545:2113]
2025-11-19T13:59:11.893500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435752016446545:2113] [node 4] ICS09 handshake done sender: [3:7574435752016446525:2117] self: [3:7574435752016446524:0] peer: [4:7574435753113816393:0] socket: 299 qp: -1
2025-11-19T13:59:11.893505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435752016446545:2113] [node 4] ICS10 traffic start
2025-11-19T13:59:11.893513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:11.894037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435750131780759:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.894046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:11.894049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435750131780759:2049] [node 2] ICP22 created new session: [6:7574435750131780851:2049]
2025-11-19T13:59:11.894054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435750131780851:2049] [node 2] ICS09 handshake done sender: [6:7574435750131780838:2114] self: [6:7574435750131780841:0] peer: [2:7574435751396712967:0] socket: 308 qp: -1
2025-11-19T13:59:11.894057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435750131780851:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:11.894062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:11.894607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435750131780845:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64885
2025-11-19T13:59:11.895878Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435750131780846:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.895903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435750131780792:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:11.896068Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435750131780845:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.896082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435750131780758:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.897617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435750131780855:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64885
2025-11-19T13:59:11.896165Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435751945118332:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.896174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435751945118332:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.896932Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435751945118332:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:11.897002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435751945118326:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.897016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:11.897021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435751945118326:2112] [node 6] ICP22 created new session: [1:7574435751945118348:2112]
2025-11-19T13:59:11.897027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435751945118348:2112] [node 6] ICS09 handshake done sender: [1:7574435751945118332:2116] self: [1:7574435751945118327:0] peer: [6:7574435750131780854:0] socket: 337 qp: -1
2025-11-19T13:59:11.897031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435751945118348:2112] [node 6] ICS10 traffic start
2025-11-19T13:59:11.897038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:11.897739Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435751945118325:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:11.898049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435750131780758:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13056) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435750131780845:2117] held: no
2025-11-19T13:59:11.898080Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435750131780845:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:11.898576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435751945118322:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.898594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:11.898601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435751945118322:2111] [node 4] ICP22 created new session: [1:7574435751945118351:2111]
2025-11-19T13:59:11.898605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435751945118351:2111] [node 4] ICS09 handshake done sender: [1:7574435751945118325:2113] self: [1:7574435751945118323:0] peer: [4:7574435753113816387:0] socket: 336 qp: -1
2025-11-19T13:59:11.898611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435751945118351:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:11.898619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.900010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435750131780758:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.900022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:11.900027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435750131780758:2048] [node 1] ICP22 created new session: [6:7574435750131780856:2048]
2025-11-19T13:59:11.900032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435750131780856:2048] [node 1] ICS09 handshake done sender: [6:7574435750131780845:2117] self: [6:7574435750131780854:0] peer: [1:7574435751945118327:0] socket: 339 qp: -1
2025-11-19T13:59:11.900037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435750131780856:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:11.900042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:11.900331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435750131780858:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64885
2025-11-19T13:59:11.899005Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435753113816300:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26344) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435753113816385:2120] held: no
2025-11-19T13:59:11.899059Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435753113816376:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:11.899062Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435753113816376:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:11.899671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435753113816394:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4855
2025-11-19T13:59:11.899921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435753113816395:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4855
2025-11-19T13:59:11.903716Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435753113816376:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:11.904348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435753113816298:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13056) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435753113816383:2118] held: no
2025-11-19T13:59:11.904368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435753113816371:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:11.904381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:11.904387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435753113816371:2113] [node 6] ICP22 created new session: [4:7574435753113816397:2113]
2025-11-19T13:59:11.904392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435753113816397:2113] [node 6] ICS09 handshake done sender: [4:7574435753113816376:2115] self: [4:7574435753113816374:0] peer: [6:7574435750131780853:0] socket: 314 qp: -1
2025-11-19T13:59:11.904397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435753113816397:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:11.904405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:11.904518Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435753113816385:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:11.904630Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435753113816383:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:11.904898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435753113816300:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.904909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:11.904914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435753113816300:2050] [node 3] ICP22 created new session: [4:7574435753113816399:2050]
2025-11-19T13:59:11.904918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435753113816399:2050] [node 3] ICS09 handshake done sender: [4:7574435753113816385:2120] self: [4:7574435753113816393:0] peer: [3:7574435752016446524:0] socket: 345 qp: -1
2025-11-19T13:59:11.904923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435753113816399:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:11.904928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:11.905012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435753113816298:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.905017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:11.905020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435753113816298:2048] [node 1] ICP22 created new session: [4:7574435753113816401:2048]
2025-11-19T13:59:11.905025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435753113816401:2048] [node 1] ICS09 handshake done sender: [4:7574435753113816383:2118] self: [4:7574435753113816387:0] peer: [1:7574435751945118323:0] socket: 338 qp: -1
2025-11-19T13:59:11.905028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435753113816401:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:11.905032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:11.908620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435750131780792:2074] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63250) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435750131780846:2118] held: no
2025-11-19T13:59:11.910383Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435750131780846:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:11.910550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435750131780792:2074] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:11.910558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:11.910564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435750131780792:2074] [node 4] ICP22 created new session: [6:7574435750131780859:2074]
2025-11-19T13:59:11.910569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435750131780859:2074] [node 4] ICS09 handshake done sender: [6:7574435750131780846:2118] self: [6:7574435750131780853:0] peer: [4:7574435753113816374:0] socket: 300 qp: -1
2025-11-19T13:59:11.910575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435750131780859:2074] [node 4] ICS10 traffic start
2025-11-19T13:59:11.910582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:11.918593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:11.918678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435751396712996:2048] [node 1] ICS01 socket: 291 reason# EndOfStream
2025-11-19T13:59:11.918687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435751396712895:2048] [node 1] ICP30 unregister session Session# [2:7574435751396712996:2048] VirtualId# [2:7574435751396712980:0]
2025-11-19T13:59:11.918694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435751396712996:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.918726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.918744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.919344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:11.919408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435752016446540:2048] [node 1] ICS01 socket: 292 reason# EndOfStream
2025-11-19T13:59:11.919413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435752016446444:2048] [node 1] ICP30 unregister session Session# [3:7574435752016446540:2048] VirtualId# [3:7574435752016446530:0]
2025-11-19T13:59:11.919418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435752016446540:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.919445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.919462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.944454Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:11.944556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435753113816401:2048] [node 1] ICS01 socket: 338 reason# EndOfStream
2025-11-19T13:59:11.944564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435753113816298:2048] [node 1] ICP30 unregister session Session# [4:7574435753113816401:2048] VirtualId# [4:7574435753113816387:0]
2025-11-19T13:59:11.944571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435753113816401:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.944620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.944646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.944789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:11.944849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435752335042079:2048] [node 1] ICS01 socket: 293 reason# EndOfStream
2025-11-19T13:59:11.944853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435752335041992:2048] [node 1] ICP30 unregister session Session# [5:7574435752335042079:2048] VirtualId# [5:7574435752335042065:0]
2025-11-19T13:59:11.944857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435752335042079:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.944882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.944903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.947817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:11.947894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435750131780856:2048] [node 1] ICS01 socket: 339 reason# EndOfStream
2025-11-19T13:59:11.947900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435750131780758:2048] [node 1] ICP30 unregister session Session# [6:7574435750131780856:2048] VirtualId# [6:7574435750131780854:0]
2025-11-19T13:59:11.947905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435750131780856:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.947941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:11.947970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:11.952202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:11.952278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435752016446517:2049] [node 2] ICS01 socket: 298 reason# EndOfStream
2025-11-19T13:59:11.952286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435752016446445:2049] [node 2] ICP30 unregister session Session# [3:7574435752016446517:2049] VirtualId# [3:7574435752016446506:0]
2025-11-19T13:59:11.952290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435752016446517:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.952339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:11.952360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:11.959371Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:11.959459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435753113816388:2049] [node 2] ICS01 socket: 322 reason# EndOfStream
2025-11-19T13:59:11.959467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435753113816299:2049] [node 2] ICP30 unregister session Session# [4:7574435753113816388:2049] VirtualId# [4:7574435753113816380:0]
2025-11-19T13:59:11.959474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435753113816388:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.959528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:11.959551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:11.959588Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:11.959646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435750131780851:2049] [node 2] ICS01 socket: 308 reason# EndOfStream
2025-11-19T13:59:11.959651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435750131780759:2049] [node 2] ICP30 unregister session Session# [6:7574435750131780851:2049] VirtualId# [6:7574435750131780841:0]
2025-11-19T13:59:11.959655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435750131780851:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.959680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:11.959699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:11.959725Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:11.959795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435752335042083:2049] [node 2] ICS01 socket: 306 reason# EndOfStream
2025-11-19T13:59:11.959800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435752335041993:2049] [node 2] ICP30 unregister session Session# [5:7574435752335042083:2049] VirtualId# [5:7574435752335042075:0]
2025-11-19T13:59:11.959804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435752335042083:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.959831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:11.959851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:11.973860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:11.973951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435750131780849:2050] [node 3] ICS01 socket: 327 reason# EndOfStream
2025-11-19T13:59:11.973961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435750131780760:2050] [node 3] ICP30 unregister session Session# [6:7574435750131780849:2050] VirtualId# [6:7574435750131780843:0]
2025-11-19T13:59:11.973968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435750131780849:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.974018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:11.974040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:11.974074Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:11.974135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435752335042086:2050] [node 3] ICS01 socket: 328 reason# EndOfStream
2025-11-19T13:59:11.974138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435752335041994:2050] [node 3] ICP30 unregister session Session# [5:7574435752335042086:2050] VirtualId# [5:7574435752335042081:0]
2025-11-19T13:59:11.974142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435752335042086:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:11.974165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:11.974184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:12.003934Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:12.004035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435750131780859:2074] [node 4] ICS01 socket: 300 reason# EndOfStream
2025-11-19T13:59:12.004044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435750131780792:2074] [node 4] ICP30 unregister session Session# [6:7574435750131780859:2074] VirtualId# [6:7574435750131780853:0]
2025-11-19T13:59:12.004053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435750131780859:2074] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.004112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:12.004138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:12.080092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435754525177349:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:12.080103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435754525177350:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:12.115867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435758460134927:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:12.115878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435758460134928:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:12.176090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435758152514139:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:12.176104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435758152514140:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:12.231869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435755381561188:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:12.231884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435755381561189:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:12.231890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435755381561190:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:12.236484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435758604309011:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:12.236496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435758604309012:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:12.236500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435758604309013:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:12.255943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435754710060796:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:12.255955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435754710060797:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:12.255958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435754710060798:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:12.308009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435754710060811:2059] [node 5] ICP01 ready to work
2025-11-19T13:59:12.308238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435758604309067:2101] [node 6] ICP01 ready to work
2025-11-19T13:59:12.307426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435754525177410:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:12.308372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435754525177418:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:12.312249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435754525177422:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.317194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435758460134990:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52031
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.322426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435754525177426:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50504
2025-11-19T13:59:12.322481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435754525177427:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50504
2025-11-19T13:59:12.320327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435758152514205:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:12.320413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435758152514206:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:12.324230Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435754525177427:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.329147Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435758460134990:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.330175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435758152514209:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41798
2025-11-19T13:59:12.330234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435758152514210:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41798
2025-11-19T13:59:12.330247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435758152514211:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41798
2025-11-19T13:59:12.330615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435758152514200:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:12.330912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435758460134992:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:12.334228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435754710060833:2079] [node 4] ICP01 ready to work
2025-11-19T13:59:12.328220Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435754525177426:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.329494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435754525177422:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17188523ns) {Configure:104(17187838ns) {ProcessConfigured:135(16939288ns) {StartInitialHandshake:157(16938345ns) {DropHandshakes:469(628ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(80ns) PrepareNewSessionHandshake:476(683ns) {SwitchToState:178(532ns)}}}}}
2025-11-19T13:59:12.340406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435758460134998:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.345764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755381561251:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64368
2025-11-19T13:59:12.345914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435758604309067:2101] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(37612411ns) {Configure:104(37611912ns) {ProcessConfigured:135(37238881ns) {StartInitialHandshake:157(37238344ns) {DropHandshakes:469(592ns) {DropIncomingHandshake:441(122ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(308ns) PrepareNewSessionHandshake:476(1462ns) {SwitchToState:178(840ns)}}}}}
2025-11-19T13:59:12.346150Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435758152514140:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(25699397ns) {Configure:104(25698958ns) {ProcessConfigured:135(25465309ns) {StartInitialHandshake:157(25465136ns) {DropHandshakes:469(276ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(39ns)} GenerateSessionVirtualId:988(77ns) PrepareNewSessionHandshake:476(917ns) {SwitchToState:178(158ns)}}}}}
2025-11-19T13:59:12.346194Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758152514209:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.346236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435758152514139:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.346480Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435758152514205:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(25629273ns) {Configure:104(25621054ns) {ProcessConfigured:135(25403491ns) {StartInitialHandshake:157(25403205ns) {DropHandshakes:469(237ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(31ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:476(459ns) {SwitchToState:178(28ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.346632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758604309077:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.353003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435758460134992:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(22024453ns) {Configure:104(22023853ns) {ProcessConfigured:135(21716794ns) {StartInitialHandshake:157(21707979ns) {DropHandshakes:469(525ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(87ns)} GenerateSessionVirtualId:988(631ns) PrepareNewSessionHandshake:476(588ns) {SwitchToState:178(222ns)}} IncomingHandshake:261(3311ns) {DropIncomingHandshake:441(247ns) PrepareNewSessionHandshake:476(147ns) {SwitchToState:178(109ns)} GenerateSessionVirtualId:988(551ns) IssueIncomingHandshakeReply:193(223ns)}}}}
2025-11-19T13:59:12.353108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435758460135004:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52031
2025-11-19T13:59:12.353437Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435755381561251:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.352879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435758152514200:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(22212390ns) {Configure:104(22211677ns) {ProcessConfigured:135(21929802ns) {StartInitialHandshake:157(21929682ns) {DropHandshakes:469(561ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:476(744ns) {SwitchToState:178(19ns)}}}}}
2025-11-19T13:59:12.352755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435754525177349:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(43009808ns) {Configure:104(43009223ns) {ProcessConfigured:135(42760990ns) {StartInitialHandshake:157(42760304ns) {DropHandshakes:469(246ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(241ns) PrepareNewSessionHandshake:476(1097ns) {SwitchToState:178(199ns)}}}}}
2025-11-19T13:59:12.353319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435754525177432:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50504
2025-11-19T13:59:12.359875Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758152514211:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.356018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435755381561188:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23919306ns) {Configure:104(23918037ns) {ProcessConfigured:135(23555420ns) {StartInitialHandshake:157(23554496ns) {DropHandshakes:469(539ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:476(798ns) {SwitchToState:178(359ns)}}}}}
2025-11-19T13:59:12.356060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435755381561188:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.360884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435755381561258:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:12.360976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435755381561259:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.361434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435758604309079:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:12.361947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758604309083:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.361976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754710060873:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
2025-11-19T13:59:12.361998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754710060874:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
2025-11-19T13:59:12.362008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754710060875:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.366700Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435754525177415:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.366711Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435754525177415:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.367265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435754525177433:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50504
2025-11-19T13:59:12.367303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435754525177434:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50504
2025-11-19T13:59:12.367212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435754710060833:2079] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(32909550ns) {Configure:104(32908931ns) {ProcessConfigured:135(32608962ns) {StartInitialHandshake:157(32607992ns) {DropHandshakes:469(499ns) {DropIncomingHandshake:441(45ns) DropOutgoingHandshake:456(112ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(813ns) {SwitchToState:178(592ns)}}}}}
2025-11-19T13:59:12.367381Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435755381561189:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(27471026ns) {Configure:104(27469655ns) {ProcessConfigured:135(27128321ns) {StartInitialHandshake:157(27128003ns) {DropHandshakes:469(768ns) {DropIncomingHandshake:441(157ns) DropOutgoingHandshake:456(99ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:476(998ns) {SwitchToState:178(281ns)}}}}}
2025-11-19T13:59:12.368291Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435754710060873:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.368119Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435758604309077:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.368020Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435754525177412:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.368037Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435754525177412:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.368539Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435754525177415:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:12.368637Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435754525177433:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.368543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435758604309011:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.368643Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435758604309083:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.368661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435758604309079:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.370052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435754525177432:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.370067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435754525177410:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.370081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:12.370086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435754525177410:2107] [node 4] ICP22 created new session: [1:7574435754525177437:2107]
2025-11-19T13:59:12.370091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435754525177437:2107] [node 4] ICS09 handshake done sender: [1:7574435754525177415:2110] self: [1:7574435754525177413:0] peer: [4:7574435755381561254:0] socket: 295 qp: -1
2025-11-19T13:59:12.370098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435754525177437:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:12.370117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:12.370167Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758152514213:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.370180Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435758152514213:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.372281Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435758460135002:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.372293Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435758460135002:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.369981Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435754710060875:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.370012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435754710060797:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.373330Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435755381561190:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(25010661ns) {Configure:104(25010069ns) {ProcessConfigured:135(24689077ns) {StartInitialHandshake:157(24688652ns) {DropHandshakes:469(597ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(149ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:476(369ns) {SwitchToState:178(87ns)}}}}}
2025-11-19T13:59:12.372532Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435758152514213:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:12.372621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435758152514206:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.372640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:12.372647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435758152514206:2109] [node 6] ICP22 created new session: [3:7574435758152514224:2109]
2025-11-19T13:59:12.372653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435758152514224:2109] [node 6] ICS09 handshake done sender: [3:7574435758152514213:2114] self: [3:7574435758152514212:0] peer: [6:7574435754710060877:0] socket: 311 qp: -1
2025-11-19T13:59:12.372663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435758152514224:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:12.372679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:12.377229Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435758460135002:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:12.377844Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435755381561264:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.377858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435755381561264:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.378393Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435754710060874:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.378424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435754710060811:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.379792Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435754525177424:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.379811Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435754525177424:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.383615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758604309090:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T13:59:12.383908Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758152514210:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.384069Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435758604309090:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.387320Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435758460135004:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.387784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435758460135001:2112] [node 4] ICP01 ready to work
2025-11-19T13:59:12.387820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435758460134992:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.387837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:12.387844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435758460134992:2109] [node 6] ICP22 created new session: [2:7574435758460135007:2109]
2025-11-19T13:59:12.387850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435758460135007:2109] [node 6] ICS09 handshake done sender: [2:7574435758460135002:2113] self: [2:7574435758460134996:0] peer: [6:7574435754710060878:0] socket: 317 qp: -1
2025-11-19T13:59:12.387859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435758460135007:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:12.387878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.392717Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435758604309012:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24511407ns) {Configure:104(24510663ns) {ProcessConfigured:135(24218459ns) {StartInitialHandshake:157(24217946ns) {DropHandshakes:469(422ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(258ns) PrepareNewSessionHandshake:476(729ns) {SwitchToState:178(299ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.392930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754710060880:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
2025-11-19T13:59:12.392983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754710060881:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
2025-11-19T13:59:12.393153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754710060882:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
2025-11-19T13:59:12.393229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435754710060798:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.393259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435754710060798:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(24921852ns) {IncomingHandshake:261(24921467ns) {DropIncomingHandshake:441(305ns) PrepareNewSessionHandshake:476(756ns) {SwitchToState:178(537ns)} GenerateSessionVirtualId:988(214ns) IssueIncomingHandshakeReply:193(24916774ns) {SwitchToState:178(256ns)}}}
2025-11-19T13:59:12.393841Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435758460134994:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.393852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435758460134994:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.394062Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435754525177412:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:12.394445Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435754525177434:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.394454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435754525177350:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.394482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:12.394488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435754525177350:2049] [node 3] ICP22 created new session: [1:7574435754525177442:2049]
2025-11-19T13:59:12.394494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435754525177442:2049] [node 3] ICS09 handshake done sender: [1:7574435754525177412:2108] self: [1:7574435754525177411:0] peer: [3:7574435758152514218:0] socket: 294 qp: -1
2025-11-19T13:59:12.394499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435754525177442:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:12.394504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:12.393794Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435758604309075:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.393810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435758604309075:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.397267Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435755381561188:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18629) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435755381561251:2108] held: no
2025-11-19T13:59:12.397812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435758460134998:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(57355067ns) {Configure:104(57354196ns) {ProcessConfigured:135(57075884ns) {StartInitialHandshake:157(57075384ns) {DropHandshakes:469(164ns) {DropIncomingHandshake:441(28ns) DropOutgoingHandshake:456(24ns)} GenerateSessionVirtualId:988(36ns) PrepareNewSessionHandshake:476(711ns) {SwitchToState:178(464ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.398690Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435755381561264:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:12.398949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435755381561259:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.398965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:12.398970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435755381561259:2111] [node 5] ICP22 created new session: [4:7574435755381561269:2111]
2025-11-19T13:59:12.398986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435755381561269:2111] [node 5] ICS09 handshake done sender: [4:7574435755381561264:2113] self: [4:7574435755381561263:0] peer: [5:7574435758604309089:0] socket: 320 qp: -1
2025-11-19T13:59:12.398992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435755381561269:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:12.398999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:12.398574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758604309093:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T13:59:12.398866Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435758604309083:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:12.399328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435758604309079:2111] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.399336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:12.399341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435758604309079:2111] [node 4] ICP22 created new session: [5:7574435758604309094:2111]
2025-11-19T13:59:12.399345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435758604309094:2111] [node 4] ICS09 handshake done sender: [5:7574435758604309083:2113] self: [5:7574435758604309089:0] peer: [4:7574435755381561263:0] socket: 324 qp: -1
2025-11-19T13:59:12.399351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435758604309094:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:12.399356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.400077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435758460135012:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52031
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.403850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435758604309013:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(35640283ns) {Configure:104(25754042ns) {ProcessConfigured:135(25430003ns) {StartInitialHandshake:157(25429371ns) {DropHandshakes:469(642ns) {DropIncomingHandshake:441(115ns) DropOutgoingHandshake:456(240ns)} GenerateSessionVirtualId:988(360ns) PrepareNewSessionHandshake:476(807ns) {SwitchToState:178(381ns)}}}}}
2025-11-19T13:59:12.403890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435758604309013:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.404819Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758152514220:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.404830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435758152514220:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.405363Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435758460135012:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.405709Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435758152514220:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:12.405790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435758152514205:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.405803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:12.405807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435758152514205:2108] [node 5] ICP22 created new session: [3:7574435758152514229:2108]
2025-11-19T13:59:12.405812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435758152514229:2108] [node 5] ICS09 handshake done sender: [3:7574435758152514220:2116] self: [3:7574435758152514208:0] peer: [5:7574435758604309098:0] socket: 309 qp: -1
2025-11-19T13:59:12.405817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435758152514229:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:12.405833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.407366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435758460135016:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52031
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.408738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758604309099:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.408906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758604309100:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T13:59:12.408924Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435758604309090:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:12.409145Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435754525177424:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:12.408069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435758152514206:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(35416824ns) {HandleHandshakeStatus:329(35415903ns) {DropIncomingHandshake:441(3643ns) DropOutgoingHandshake:456(104ns) SwitchToState:178(75ns) ProcessPendingSessionEvents:495(27891655ns) {ForwardSessionEventToSession:612(27879927ns) ForwardSessionEventToSession:612(1223ns) ForwardSessionEventToSession:612(1095ns) ForwardSessionEventToSession:612(957ns) ForwardSessionEventToSession:612(582ns) ForwardSessionEventToSession:612(598ns) ForwardSessionEventToSession:612(666ns) ForwardSessionEventToSession:612(1072ns) ForwardSessionEventToSession:612(452ns) ForwardSessionEventToSession:612(1175ns)}}}
2025-11-19T13:59:12.408094Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435758152514139:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18629) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435758152514209:2111] held: no
2025-11-19T13:59:12.409043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435758604309013:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.409061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:12.409067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435758604309013:2050] [node 3] ICP22 created new session: [5:7574435758604309101:2050]
2025-11-19T13:59:12.409073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435758604309101:2050] [node 3] ICS09 handshake done sender: [5:7574435758604309090:2115] self: [5:7574435758604309098:0] peer: [3:7574435758152514208:0] socket: 333 qp: -1
2025-11-19T13:59:12.409078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435758604309101:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:12.409095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.409554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435758460135017:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52031
2025-11-19T13:59:12.409425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435754525177418:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.409443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:12.409448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435754525177418:2109] [node 5] ICP22 created new session: [1:7574435754525177447:2109]
2025-11-19T13:59:12.409452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435754525177447:2109] [node 5] ICS09 handshake done sender: [1:7574435754525177424:2112] self: [1:7574435754525177419:0] peer: [5:7574435758604309088:0] socket: 298 qp: -1
2025-11-19T13:59:12.409456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435754525177447:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:12.409464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:12.409724Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435758604309077:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:12.415878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754710060883:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
2025-11-19T13:59:12.415956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435758152514140:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.415950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435754710060798:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18083) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435754710060873:2114] held: no
2025-11-19T13:59:12.416000Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435754710060880:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.415994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435758152514140:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(32009383ns) {IncomingHandshake:261(32009053ns) {DropIncomingHandshake:441(128ns) PrepareNewSessionHandshake:476(341ns) {SwitchToState:178(96ns)} GenerateSessionVirtualId:988(181ns) IssueIncomingHandshakeReply:193(32005476ns) {SwitchToState:178(253ns)}}}
2025-11-19T13:59:12.415995Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435758460135016:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.416034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435758460134927:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.416220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758604309096:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T13:59:12.416281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435758604309011:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.416295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:12.416301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435758604309011:2048] [node 1] ICP22 created new session: [5:7574435758604309104:2048]
2025-11-19T13:59:12.416307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435758604309104:2048] [node 1] ICS09 handshake done sender: [5:7574435758604309077:2110] self: [5:7574435758604309088:0] peer: [1:7574435754525177419:0] socket: 303 qp: -1
2025-11-19T13:59:12.416314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435758604309104:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:12.416324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:12.416396Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435758604309011:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.416666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435758152514232:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41798
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.416859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755381561272:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64368
2025-11-19T13:59:12.416905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755381561273:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64368
2025-11-19T13:59:12.419333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755381561274:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64368
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.419585Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435754525177350:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(25105357ns) {HandleHandshakeStatus:329(25104758ns) {DropIncomingHandshake:441(2226ns) DropOutgoingHandshake:456(275ns) SwitchToState:178(82ns) ProcessPendingSessionEvents:495(25020114ns) {ForwardSessionEventToSession:612(25008599ns) ForwardSessionEventToSession:612(1401ns) ForwardSessionEventToSession:612(659ns) ForwardSessionEventToSession:612(605ns) ForwardSessionEventToSession:612(864ns) ForwardSessionEventToSession:612(853ns) ForwardSessionEventToSession:612(609ns) ForwardSessionEventToSession:612(611ns) ForwardSessionEventToSession:612(533ns) ForwardSessionEventToSession:612(742ns)}}}
2025-11-19T13:59:12.417119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435758460134992:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(29276732ns) {HandleHandshakeStatus:329(29275911ns) {DropIncomingHandshake:441(4337ns) DropOutgoingHandshake:456(252ns) SwitchToState:178(152ns) ProcessPendingSessionEvents:495(23460725ns) {ForwardSessionEventToSession:612(23451583ns) ForwardSessionEventToSession:612(981ns) ForwardSessionEventToSession:612(744ns) ForwardSessionEventToSession:612(564ns) ForwardSessionEventToSession:612(589ns) ForwardSessionEventToSession:612(944ns) ForwardSessionEventToSession:612(685ns) ForwardSessionEventToSession:612(515ns) ForwardSessionEventToSession:612(484ns) ForwardSessionEventToSession:612(644ns)}}}
2025-11-19T13:59:12.419841Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435758460135017:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.419898Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435758460134994:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:12.420086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435758460134928:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.420102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:12.420108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435758460134928:2049] [node 3] ICP22 created new session: [2:7574435758460135021:2049]
2025-11-19T13:59:12.420114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435758460135021:2049] [node 3] ICS09 handshake done sender: [2:7574435758460134994:2110] self: [2:7574435758460134993:0] peer: [3:7574435758152514227:0] socket: 306 qp: -1
2025-11-19T13:59:12.420118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435758460135021:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:12.420124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:12.420313Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435758604309075:2109] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:12.420522Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435758604309096:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.421844Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435754710060873:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:12.422398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435755381561259:2111] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:12.419667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435758152514233:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41798
2025-11-19T13:59:12.419953Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435758152514210:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:12.421634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435758152514140:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.421644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:12.421650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435758152514140:2049] [node 2] ICP22 created new session: [3:7574435758152514234:2049]
2025-11-19T13:59:12.421655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435758152514234:2049] [node 2] ICS09 handshake done sender: [3:7574435758152514210:2112] self: [3:7574435758152514227:0] peer: [2:7574435758460134993:0] socket: 308 qp: -1
2025-11-19T13:59:12.421661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435758152514234:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:12.421667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:12.421749Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435758152514209:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:12.424272Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435755381561251:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:12.424492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435758152514139:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.424508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:12.424513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435758152514139:2048] [node 1] ICP22 created new session: [3:7574435758152514236:2048]
2025-11-19T13:59:12.424518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435758152514236:2048] [node 1] ICS09 handshake done sender: [3:7574435758152514209:2111] self: [3:7574435758152514218:0] peer: [1:7574435754525177411:0] socket: 296 qp: -1
2025-11-19T13:59:12.424523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435758152514236:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:12.424530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:12.424742Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435755381561274:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.424765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435755381561190:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.422169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435758604309012:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.424863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435755381561188:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.424873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:12.424877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435755381561188:2048] [node 1] ICP22 created new session: [4:7574435755381561277:2048]
2025-11-19T13:59:12.424881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435755381561277:2048] [node 1] ICS09 handshake done sender: [4:7574435755381561251:2108] self: [4:7574435755381561254:0] peer: [1:7574435754525177413:0] socket: 297 qp: -1
2025-11-19T13:59:12.424885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435755381561277:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:12.424912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:12.427947Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435754525177430:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.427965Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435754525177430:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.427854Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435754710060875:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:12.429422Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435758460135009:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.429433Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435758460135009:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.431893Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435758460135009:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.437239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755381561279:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64368
2025-11-19T13:59:12.436275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758604309108:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T13:59:12.436352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435758460134998:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.436376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:12.436392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435758460134998:2111] [node 5] ICP22 created new session: [2:7574435758460135025:2111]
2025-11-19T13:59:12.436398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435758460135025:2111] [node 5] ICS09 handshake done sender: [2:7574435758460135009:2115] self: [2:7574435758460134999:0] peer: [5:7574435758604309107:0] socket: 342 qp: -1
2025-11-19T13:59:12.436403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435758460135025:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:12.436411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.437541Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758152514221:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.437553Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435758152514221:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.438044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435758152514238:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41798
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.437105Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435754710060883:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.437136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435754710060797:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16331) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435754710060875:2116] held: no
2025-11-19T13:59:12.438321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755381561280:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64368
2025-11-19T13:59:12.437169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435754710060797:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.437178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:12.437184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435754710060797:2049] [node 2] ICP22 created new session: [6:7574435754710060885:2049]
2025-11-19T13:59:12.437190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435754710060885:2049] [node 2] ICS09 handshake done sender: [6:7574435754710060875:2116] self: [6:7574435754710060878:0] peer: [2:7574435758460134996:0] socket: 323 qp: -1
2025-11-19T13:59:12.437195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435754710060885:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:12.437202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:12.438573Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435758604309096:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:12.438696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435758604309012:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.438712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:12.438717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435758604309012:2049] [node 2] ICP22 created new session: [5:7574435758604309109:2049]
2025-11-19T13:59:12.438722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435758604309109:2049] [node 2] ICS09 handshake done sender: [5:7574435758604309096:2118] self: [5:7574435758604309107:0] peer: [2:7574435758460134999:0] socket: 343 qp: -1
2025-11-19T13:59:12.438727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435758604309109:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:12.438732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:12.439763Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758152514238:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.439820Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435758152514221:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:12.440077Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435755381561279:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.440099Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435755381561274:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:12.440160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435755381561189:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.440320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435755381561190:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.440336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:12.440341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435755381561190:2050] [node 3] ICP22 created new session: [4:7574435755381561282:2050]
2025-11-19T13:59:12.440346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435755381561282:2050] [node 3] ICS09 handshake done sender: [4:7574435755381561274:2118] self: [4:7574435755381561276:0] peer: [3:7574435758152514216:0] socket: 328 qp: -1
2025-11-19T13:59:12.440351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435755381561282:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:12.440356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:12.441451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435758152514200:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.441464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:12.441470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435758152514200:2107] [node 4] ICP22 created new session: [3:7574435758152514240:2107]
2025-11-19T13:59:12.441475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435758152514240:2107] [node 4] ICS09 handshake done sender: [3:7574435758152514221:2117] self: [3:7574435758152514216:0] peer: [4:7574435755381561276:0] socket: 345 qp: -1
2025-11-19T13:59:12.441480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435758152514240:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:12.441486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:12.441624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435758152514200:2107] [node 4] ICP18 handshake (actor [3:7574435758152514238:2120]) is too old
2025-11-19T13:59:12.444608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435758604309067:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.444630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:12.444639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435758604309067:2101] [node 6] ICP22 created new session: [5:7574435758604309111:2101]
2025-11-19T13:59:12.444645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435758604309111:2101] [node 6] ICS09 handshake done sender: [5:7574435758604309075:2109] self: [5:7574435758604309068:0] peer: [6:7574435754710060879:0] socket: 314 qp: -1
2025-11-19T13:59:12.444651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435758604309111:2101] [node 6] ICS10 traffic start
2025-11-19T13:59:12.444663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:12.444756Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435758460135011:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.444767Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435758460135011:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.447917Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435758460135011:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.448011Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435754525177430:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:12.448114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755381561284:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64368
2025-11-19T13:59:12.447978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435758460135028:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52031
2025-11-19T13:59:12.448183Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435758460135016:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:12.448267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435758460135001:2112] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.448281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:12.448295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435758460135001:2112] [node 4] ICP22 created new session: [2:7574435758460135029:2112]
2025-11-19T13:59:12.448301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435758460135029:2112] [node 4] ICS09 handshake done sender: [2:7574435758460135011:2116] self: [2:7574435758460135010:0] peer: [4:7574435755381561281:0] socket: 307 qp: -1
2025-11-19T13:59:12.448306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435758460135029:2112] [node 4] ICS10 traffic start
2025-11-19T13:59:12.448362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435754525177349:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.448382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:12.448386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435754525177349:2048] [node 2] ICP22 created new session: [1:7574435754525177450:2048]
2025-11-19T13:59:12.448391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435754525177450:2048] [node 2] ICS09 handshake done sender: [1:7574435754525177430:2116] self: [1:7574435754525177420:0] peer: [2:7574435758460135018:0] socket: 346 qp: -1
2025-11-19T13:59:12.448396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435754525177450:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:12.448402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:12.448313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.448645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435754710060833:2079] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.448725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435754710060798:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.448732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:12.448736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435754710060798:2050] [node 3] ICP22 created new session: [6:7574435754710060889:2050]
2025-11-19T13:59:12.448740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435754710060889:2050] [node 3] ICS09 handshake done sender: [6:7574435754710060873:2114] self: [6:7574435754710060877:0] peer: [3:7574435758152514212:0] socket: 313 qp: -1
2025-11-19T13:59:12.448745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435754710060889:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:12.448750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:12.448858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435754710060796:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.448919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754710060884:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
2025-11-19T13:59:12.448861Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435755381561279:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:12.450096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435758460134927:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.450115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:12.450124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435758460134927:2048] [node 1] ICP22 created new session: [2:7574435758460135031:2048]
2025-11-19T13:59:12.450130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435758460135031:2048] [node 1] ICS09 handshake done sender: [2:7574435758460135016:2118] self: [2:7574435758460135018:0] peer: [1:7574435754525177420:0] socket: 347 qp: -1
2025-11-19T13:59:12.450135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435758460135031:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:12.450140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:12.449763Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435755381561261:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.449770Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435755381561261:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.450591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755381561285:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64368
2025-11-19T13:59:12.450639Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435755381561261:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:12.449270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435758604309067:2101] [node 6] ICP35 event processing took too much time PendingConnection:278(28708590ns) {HandleHandshakeStatus:329(28708220ns) {DropIncomingHandshake:441(460ns) DropOutgoingHandshake:456(86ns) SwitchToState:178(69ns) ProcessPendingSessionEvents:495(26366ns) {ForwardSessionEventToSession:612(17390ns) ForwardSessionEventToSession:612(591ns) ForwardSessionEventToSession:612(755ns) ForwardSessionEventToSession:612(834ns) ForwardSessionEventToSession:612(669ns) ForwardSessionEventToSession:612(733ns) ForwardSessionEventToSession:612(596ns) ForwardSessionEventToSession:612(536ns) ForwardSessionEventToSession:612(674ns) ForwardSessionEventToSession:612(707ns)}}}
2025-11-19T13:59:12.450717Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435754525177428:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.450726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435754525177428:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.451465Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435754525177428:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:12.451591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435754525177422:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.451605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:12.451611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435754525177422:2111] [node 6] ICP22 created new session: [1:7574435754525177453:2111]
2025-11-19T13:59:12.451616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435754525177453:2111] [node 6] ICS09 handshake done sender: [1:7574435754525177428:2115] self: [1:7574435754525177423:0] peer: [6:7574435754710060891:0] socket: 334 qp: -1
2025-11-19T13:59:12.451620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435754525177453:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:12.451625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:12.451905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435755381561189:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16331) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435755381561279:2119] held: no
2025-11-19T13:59:12.451926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435755381561258:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.451933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:12.451939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435755381561258:2110] [node 6] ICP22 created new session: [4:7574435755381561287:2110]
2025-11-19T13:59:12.451944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435755381561287:2110] [node 6] ICS09 handshake done sender: [4:7574435755381561261:2112] self: [4:7574435755381561260:0] peer: [6:7574435754710060887:0] socket: 319 qp: -1
2025-11-19T13:59:12.451949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435755381561287:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:12.451954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:12.452154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435755381561189:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.452164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:12.452168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435755381561189:2049] [node 2] ICP22 created new session: [4:7574435755381561289:2049]
2025-11-19T13:59:12.452173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435755381561289:2049] [node 2] ICS09 handshake done sender: [4:7574435755381561279:2119] self: [4:7574435755381561281:0] peer: [2:7574435758460135010:0] socket: 355 qp: -1
2025-11-19T13:59:12.452177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435755381561289:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:12.452192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:12.449109Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435754710060874:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.454935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435754710060811:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.454952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:12.454959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435754710060811:2059] [node 5] ICP22 created new session: [6:7574435754710060893:2059]
2025-11-19T13:59:12.454965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435754710060893:2059] [node 5] ICS09 handshake done sender: [6:7574435754710060874:2115] self: [6:7574435754710060879:0] peer: [5:7574435758604309068:0] socket: 322 qp: -1
2025-11-19T13:59:12.454971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435754710060893:2059] [node 5] ICS10 traffic start
2025-11-19T13:59:12.454977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:12.455573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758604309113:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T13:59:12.457625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435758604309067:2101] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.461259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754710060895:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.463823Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435754710060880:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:12.464122Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435755381561285:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.464161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435755381561258:2110] [node 6] ICP18 handshake (actor [4:7574435755381561285:2122]) is too old
2025-11-19T13:59:12.464712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435754710060833:2079] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.464725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:12.464731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435754710060833:2079] [node 4] ICP22 created new session: [6:7574435754710060897:2079]
2025-11-19T13:59:12.464737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435754710060897:2079] [node 4] ICS09 handshake done sender: [6:7574435754710060880:2118] self: [6:7574435754710060887:0] peer: [4:7574435755381561260:0] socket: 325 qp: -1
2025-11-19T13:59:12.464742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435754710060897:2079] [node 4] ICS10 traffic start
2025-11-19T13:59:12.464750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:12.467233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754710060896:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58973
2025-11-19T13:59:12.473634Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435754710060883:2121] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:12.474463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435754710060796:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18629) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435754710060883:2121] held: no
2025-11-19T13:59:12.476010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435754710060796:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.476030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:12.476038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435754710060796:2048] [node 1] ICP22 created new session: [6:7574435754710060899:2048]
2025-11-19T13:59:12.476046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435754710060899:2048] [node 1] ICS09 handshake done sender: [6:7574435754710060883:2121] self: [6:7574435754710060891:0] peer: [1:7574435754525177423:0] socket: 337 qp: -1
2025-11-19T13:59:12.476054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435754710060899:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:12.476067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:12.489106Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:12.489198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574435758460135031:2048] [node 1] ICS07 socket disconnect 347 reason# EndOfStream
2025-11-19T13:59:12.489204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435758460135031:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.489238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:12.489253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435758460135031:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:12.495796Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435758460134927:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18629) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:12.495811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435758460135031:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:12.496167Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435758460134927:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18629) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:12.496177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435758460135031:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:12.496182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435758460134927:2048] [node 1] ICP30 unregister session Session# [2:7574435758460135031:2048] VirtualId# [2:7574435758460135018:0]
2025-11-19T13:59:12.496198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:12.496203Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574435758460134927:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:12.496205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:12.501184Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:12.501269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435758604309104:2048] [node 1] ICS01 socket: 303 reason# EndOfStream
2025-11-19T13:59:12.501276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435758604309011:2048] [node 1] ICP30 unregister session Session# [5:7574435758604309104:2048] VirtualId# [5:7574435758604309088:0]
2025-11-19T13:59:12.501283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435758604309104:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.501332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:12.501357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:12.501389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:12.501449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435754710060899:2048] [node 1] ICS01 socket: 337 reason# EndOfStream
2025-11-19T13:59:12.501454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435754710060796:2048] [node 1] ICP30 unregister session Session# [6:7574435754710060899:2048] VirtualId# [6:7574435754710060891:0]
2025-11-19T13:59:12.501457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435754710060899:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.501487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:12.501506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:12.502528Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:12.502596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435755381561277:2048] [node 1] ICS01 socket: 297 reason# EndOfStream
2025-11-19T13:59:12.502601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435755381561188:2048] [node 1] ICP30 unregister session Session# [4:7574435755381561277:2048] VirtualId# [4:7574435755381561254:0]
2025-11-19T13:59:12.502605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435755381561277:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.502643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:12.502662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:12.499807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:12.499889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435758152514236:2048] [node 1] ICS01 socket: 296 reason# EndOfStream
2025-11-19T13:59:12.499896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435758152514139:2048] [node 1] ICP30 unregister session Session# [3:7574435758152514236:2048] VirtualId# [3:7574435758152514218:0]
2025-11-19T13:59:12.499903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435758152514236:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.499955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:12.499977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:12.507900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:12.507987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435758152514234:2049] [node 2] ICS01 socket: 308 reason# EndOfStream
2025-11-19T13:59:12.507994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435758152514140:2049] [node 2] ICP30 unregister session Session# [3:7574435758152514234:2049] VirtualId# [3:7574435758152514227:0]
2025-11-19T13:59:12.508000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435758152514234:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.508057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:12.508088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:12.510163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:12.510247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435754710060885:2049] [node 2] ICS01 socket: 323 reason# EndOfStream
2025-11-19T13:59:12.510254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435754710060797:2049] [node 2] ICP30 unregister session Session# [6:7574435754710060885:2049] VirtualId# [6:7574435754710060878:0]
2025-11-19T13:59:12.510259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435754710060885:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.510316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:12.510340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:12.510436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:12.510514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435758604309109:2049] [node 2] ICS01 socket: 343 reason# EndOfStream
2025-11-19T13:59:12.510527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435758604309012:2049] [node 2] ICP30 unregister session Session# [5:7574435758604309109:2049] VirtualId# [5:7574435758604309107:0]
2025-11-19T13:59:12.510533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435758604309109:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.510561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:12.510581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:12.510611Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:12.510668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435755381561289:2049] [node 2] ICS01 socket: 355 reason# EndOfStream
2025-11-19T13:59:12.510672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435755381561189:2049] [node 2] ICP30 unregister session Session# [4:7574435755381561289:2049] VirtualId# [4:7574435755381561281:0]
2025-11-19T13:59:12.510676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435755381561289:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.510700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:12.510718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:12.519590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:12.519680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435754710060889:2050] [node 3] ICS01 socket: 313 reason# EndOfStream
2025-11-19T13:59:12.519689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435754710060798:2050] [node 3] ICP30 unregister session Session# [6:7574435754710060889:2050] VirtualId# [6:7574435754710060877:0]
2025-11-19T13:59:12.519695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435754710060889:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.519759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:12.519784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:12.519864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:12.519931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435758604309101:2050] [node 3] ICS01 socket: 333 reason# EndOfStream
2025-11-19T13:59:12.519937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435758604309013:2050] [node 3] ICP30 unregister session Session# [5:7574435758604309101:2050] VirtualId# [5:7574435758604309098:0]
2025-11-19T13:59:12.519942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435758604309101:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.519970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:12.519994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:12.534314Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:12.534402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435754710060897:2079] [node 4] ICS07 socket disconnect 325 reason# EndOfStream
2025-11-19T13:59:12.534408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435754710060897:2079] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.534460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:12.534492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435754710060897:2079] [node 4] ICS15 start handshake
2025-11-19T13:59:12.534993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435754710060833:2079] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28923) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:12.535005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435754710060897:2079] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:12.535010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435754710060833:2079] [node 4] ICP30 unregister session Session# [6:7574435754710060897:2079] VirtualId# [6:7574435754710060887:0]
2025-11-19T13:59:12.535023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:12.535027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435754710060833:2079] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28923) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:12.535030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28923) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:12.545465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435754710060833:2079] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:12.604092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435757923306321:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:12.604108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435757923306322:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:12.671999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435757896280155:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:12.672045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435757896280156:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:12.700329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435758625180358:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:12.700346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435758625180359:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:12.715983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435755779079837:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:12.715999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435755779079838:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:12.716003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435755779079839:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:12.732082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435755164339965:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:12.732106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435755164339966:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:12.732110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435755164339967:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:12.760034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435754436462595:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:12.760047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435754436462596:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:12.760051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435754436462597:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:12.781177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435755779079883:2092] [node 6] ICP01 ready to work
2025-11-19T13:59:12.781446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435755779079886:2094] [node 5] ICP01 ready to work
2025-11-19T13:59:12.781607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435755164339992:2073] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.782780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754436462609:2061] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.782838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755779079893:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-11-19T13:59:12.781696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435757923306382:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:12.783057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435758625180419:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:12.784506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435757896280208:2099] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.785675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755779079897:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.783430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435755164339999:2077] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.783816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435754436462617:2066] [node 4] ICP01 ready to work
2025-11-19T13:59:12.787030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435758625180423:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10807
2025-11-19T13:59:12.784931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435757923306390:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:12.788301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435758625180426:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:12.789377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435757923306397:2113] [node 5] ICP01 ready to work
2025-11-19T13:59:12.791688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435757896280212:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.796836Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435754436462609:2061] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.796922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435754436462595:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.797379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754436462668:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-19T13:59:12.797865Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435755779079893:2097] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.800056Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435757896280212:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.799366Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435757923306384:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.799379Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435757923306384:2108] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.803434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755779079917:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-11-19T13:59:12.803930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435757896280224:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:12.803382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435754436462617:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19502930ns) {Configure:104(19502265ns) {ProcessConfigured:135(19230863ns) {StartInitialHandshake:157(19230457ns) {DropHandshakes:469(524ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(89ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:476(982ns) {SwitchToState:178(409ns)}}}}}
2025-11-19T13:59:12.806410Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435755164339965:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(22592249ns) {Configure:104(22591681ns) {ProcessConfigured:135(22393996ns) {StartInitialHandshake:157(22393601ns) {DropHandshakes:469(394ns) {DropIncomingHandshake:441(38ns) DropOutgoingHandshake:456(45ns)} GenerateSessionVirtualId:988(187ns) PrepareNewSessionHandshake:476(1031ns) {SwitchToState:178(497ns)}}}}}
2025-11-19T13:59:12.806732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435757896280155:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17402715ns) {Configure:104(17400943ns) {ProcessConfigured:135(17126508ns) {StartInitialHandshake:157(17125993ns) {DropHandshakes:469(405ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(45ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(510ns) {SwitchToState:178(353ns)}}}}}
2025-11-19T13:59:12.806766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435757896280155:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.806793Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435754436462668:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.807715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435754436462625:2069] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.806943Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435757923306384:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:12.807904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754436462667:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-19T13:59:12.806167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435755779079837:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17626920ns) {Configure:104(17626204ns) {ProcessConfigured:135(17301497ns) {StartInitialHandshake:157(17301150ns) {DropHandshakes:469(451ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:476(812ns) {SwitchToState:178(253ns)}}}}}
2025-11-19T13:59:12.808485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435755779079917:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.808523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435755779079838:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.809224Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435755779079897:2101] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.809245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435755779079839:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.809504Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435754436462667:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.812633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435755164340041:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64862
2025-11-19T13:59:12.812717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435755164340042:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64862
2025-11-19T13:59:12.813109Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435755164340042:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.815851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435758625180358:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23841473ns) {Configure:104(23838857ns) {ProcessConfigured:135(23569568ns) {StartInitialHandshake:157(23568923ns) {DropHandshakes:469(323ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(29ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:476(1016ns) {SwitchToState:178(495ns)}}}}}
2025-11-19T13:59:12.815919Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758625180421:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.815924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435758625180421:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.816361Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758625180423:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.816481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435758625180358:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.816722Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435755164339967:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(24524775ns) {Configure:104(24523682ns) {ProcessConfigured:135(24230574ns) {StartInitialHandshake:157(24230063ns) {DropHandshakes:469(419ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:476(574ns) {SwitchToState:178(41ns)}}}}}
2025-11-19T13:59:12.816773Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435755164340041:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.816800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435755164339992:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.817014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435755164339966:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24493824ns) {Configure:104(24493442ns) {ProcessConfigured:135(24297026ns) {StartInitialHandshake:157(24296837ns) {DropHandshakes:469(240ns) {DropIncomingHandshake:441(25ns) DropOutgoingHandshake:456(41ns)} GenerateSessionVirtualId:988(183ns) PrepareNewSessionHandshake:476(208ns) {SwitchToState:178(41ns)}}}}}
2025-11-19T13:59:12.817030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435755164339966:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.817205Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435757896280226:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.817216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435757896280226:2109] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.818047Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435755779079888:2095] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.818058Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435755779079888:2095] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.818610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755779079922:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-11-19T13:59:12.818820Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435755779079888:2095] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:12.818854Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435757896280226:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:12.819095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435757896280224:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.819109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:12.819115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435757896280224:2108] [node 5] ICP22 created new session: [2:7574435757896280234:2108]
2025-11-19T13:59:12.819121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435757896280234:2108] [node 5] ICS09 handshake done sender: [2:7574435757896280226:2109] self: [2:7574435757896280225:0] peer: [5:7574435755164340046:0] socket: 328 qp: -1
2025-11-19T13:59:12.819128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435757896280234:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:12.819145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:12.819014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435755779079886:2094] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.819035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:12.819041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435755779079886:2094] [node 5] ICP22 created new session: [4:7574435755779079924:2094]
2025-11-19T13:59:12.819046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435755779079924:2094] [node 5] ICS09 handshake done sender: [4:7574435755779079888:2095] self: [4:7574435755779079887:0] peer: [5:7574435755164340044:0] socket: 300 qp: -1
2025-11-19T13:59:12.819052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435755779079924:2094] [node 5] ICS10 traffic start
2025-11-19T13:59:12.819071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:12.820249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435757923306382:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.820293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:12.820303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435757923306382:2107] [node 6] ICP22 created new session: [1:7574435757923306401:2107]
2025-11-19T13:59:12.820312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435757923306401:2107] [node 6] ICS09 handshake done sender: [1:7574435757923306384:2108] self: [1:7574435757923306383:0] peer: [6:7574435754436462665:0] socket: 302 qp: -1
2025-11-19T13:59:12.820319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435757923306401:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:12.820330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:12.819972Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435758625180421:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:12.820392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435758625180429:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:12.820583Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435757896280221:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.820601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435757896280221:2107] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.821058Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435755164339992:2073] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18448) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435755164340041:2113] held: no
2025-11-19T13:59:12.821579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755779079921:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-11-19T13:59:12.823822Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435757923306386:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.823843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435757923306386:2109] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.823903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754436462672:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-19T13:59:12.823983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754436462673:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-19T13:59:12.823394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435757896280217:2104] [node 6] ICP01 ready to work
2025-11-19T13:59:12.824558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435757896280239:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58681
2025-11-19T13:59:12.824613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435757896280240:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58681
2025-11-19T13:59:12.824623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435757896280241:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58681
2025-11-19T13:59:12.821245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435758625180419:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.821258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:12.821263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435758625180419:2107] [node 4] ICP22 created new session: [3:7574435758625180442:2107]
2025-11-19T13:59:12.821268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435758625180442:2107] [node 4] ICS09 handshake done sender: [3:7574435758625180421:2108] self: [3:7574435758625180420:0] peer: [4:7574435755779079920:0] socket: 308 qp: -1
2025-11-19T13:59:12.821273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435758625180442:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:12.821289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.828148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435758625180438:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10807
2025-11-19T13:59:12.828221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435758625180439:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10807
2025-11-19T13:59:12.828262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435758625180440:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10807
2025-11-19T13:59:12.828813Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758625180439:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.828203Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435755779079897:2101] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:12.828352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435755779079839:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.828364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:12.828370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435755779079839:2050] [node 3] ICP22 created new session: [4:7574435755779079927:2050]
2025-11-19T13:59:12.828376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435755779079927:2050] [node 3] ICS09 handshake done sender: [4:7574435755779079897:2101] self: [4:7574435755779079920:0] peer: [3:7574435758625180420:0] socket: 310 qp: -1
2025-11-19T13:59:12.828384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435755779079927:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:12.828399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:12.827049Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435754436462609:2061] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:12.828940Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435754436462673:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.828969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435754436462597:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.828627Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435757896280239:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.828656Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435757896280240:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.831969Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435757923306392:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.831987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435757923306392:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.831483Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435755779079891:2096] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.831989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435754436462617:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.831499Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435755779079891:2096] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.832022Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435754436462617:2066] [node 4] ICP35 event processing took too much time PendingConnection:278(22465518ns) {IncomingHandshake:261(22465054ns) {DropIncomingHandshake:441(65ns) PrepareNewSessionHandshake:476(334ns) {SwitchToState:178(174ns)} GenerateSessionVirtualId:988(214ns) IssueIncomingHandshakeReply:193(22460959ns) {SwitchToState:178(682ns)}}}
2025-11-19T13:59:12.835949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435754436462625:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.836002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435754436462625:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(28100595ns) {Configure:104(28100085ns) {ProcessConfigured:135(27759338ns) {StartInitialHandshake:157(27720064ns) {DropHandshakes:469(600ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(473ns) PrepareNewSessionHandshake:476(664ns) {SwitchToState:178(476ns)}} IncomingHandshake:261(29649ns) {DropIncomingHandshake:441(184ns) PrepareNewSessionHandshake:476(172ns) {SwitchToState:178(129ns)} GenerateSessionVirtualId:988(234ns) IssueIncomingHandshakeReply:193(22668ns) {SwitchToState:178(47ns)}}}}}
2025-11-19T13:59:12.836721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435754436462595:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.836740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:12.836747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435754436462595:2048] [node 1] ICP22 created new session: [6:7574435754436462682:2048]
2025-11-19T13:59:12.836753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435754436462682:2048] [node 1] ICS09 handshake done sender: [6:7574435754436462609:2061] self: [6:7574435754436462665:0] peer: [1:7574435757923306383:0] socket: 305 qp: -1
2025-11-19T13:59:12.836760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435754436462682:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:12.836795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:12.839845Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435755164340003:2080] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.839859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435755164340003:2080] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.841675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755779079930:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-11-19T13:59:12.841789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435757923306382:2107] [node 6] ICP35 event processing took too much time PendingConnection:278(26981030ns) {HandleHandshakeStatus:329(26980280ns) {DropIncomingHandshake:441(564ns) DropOutgoingHandshake:456(474ns) SwitchToState:178(219ns) ProcessPendingSessionEvents:495(21386309ns) {ForwardSessionEventToSession:612(19363111ns) ForwardSessionEventToSession:612(1281ns) ForwardSessionEventToSession:612(924ns) ForwardSessionEventToSession:612(2012659ns) ForwardSessionEventToSession:612(589ns) ForwardSessionEventToSession:612(1570ns) ForwardSessionEventToSession:612(817ns)}}}
2025-11-19T13:59:12.842334Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435757923306395:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.842339Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435757923306395:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.843500Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435755164340003:2080] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:12.843700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435755164339999:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.843713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:12.843720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435755164339999:2077] [node 6] ICP22 created new session: [5:7574435755164340048:2077]
2025-11-19T13:59:12.843725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435755164340048:2077] [node 6] ICS09 handshake done sender: [5:7574435755164340003:2080] self: [5:7574435755164340002:0] peer: [6:7574435754436462681:0] socket: 316 qp: -1
2025-11-19T13:59:12.843731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435755164340048:2077] [node 6] ICS10 traffic start
2025-11-19T13:59:12.843739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.844864Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435757896280221:2107] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:12.845041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435757896280208:2099] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.845059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:12.845065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435757896280208:2099] [node 4] ICP22 created new session: [2:7574435757896280244:2099]
2025-11-19T13:59:12.845071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435757896280244:2099] [node 4] ICS09 handshake done sender: [2:7574435757896280221:2107] self: [2:7574435757896280214:0] peer: [4:7574435755779079919:0] socket: 326 qp: -1
2025-11-19T13:59:12.845077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435757896280244:2099] [node 4] ICS10 traffic start
2025-11-19T13:59:12.845084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:12.842221Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435755779079921:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.842254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435755779079837:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.843423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755779079932:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.846397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435758625180429:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(25917593ns) {Configure:104(25916850ns) {ProcessConfigured:135(25599755ns) {StartInitialHandshake:157(25599379ns) {DropHandshakes:469(625ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:476(796ns) {SwitchToState:178(167ns)}}}}}
2025-11-19T13:59:12.846450Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758625180438:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.846492Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758625180430:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.846495Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435758625180430:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.846968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435758625180359:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.847519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435758625180447:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10807
2025-11-19T13:59:12.845313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435757923306397:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(55877330ns) {Configure:104(55876269ns) {ProcessConfigured:135(55610856ns) {StartInitialHandshake:157(55610092ns) {DropHandshakes:469(613ns) {DropIncomingHandshake:441(43ns) DropOutgoingHandshake:456(42ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:476(953ns) {SwitchToState:178(515ns)}}}}}
2025-11-19T13:59:12.845394Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435757923306395:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:12.846046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435757923306390:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.846054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:12.846059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435757923306390:2110] [node 4] ICP22 created new session: [1:7574435757923306405:2110]
2025-11-19T13:59:12.846063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435757923306405:2110] [node 4] ICS09 handshake done sender: [1:7574435757923306395:2112] self: [1:7574435757923306393:0] peer: [4:7574435755779079931:0] socket: 334 qp: -1
2025-11-19T13:59:12.846068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435757923306405:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:12.846081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.844691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754436462685:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-19T13:59:12.845178Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435754436462668:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:12.845353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435754436462625:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.845367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:12.845373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435754436462625:2069] [node 5] ICP22 created new session: [6:7574435754436462686:2069]
2025-11-19T13:59:12.845378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435754436462686:2069] [node 5] ICS09 handshake done sender: [6:7574435754436462668:2111] self: [6:7574435754436462681:0] peer: [5:7574435755164340002:0] socket: 319 qp: -1
2025-11-19T13:59:12.845383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435754436462686:2069] [node 5] ICS10 traffic start
2025-11-19T13:59:12.845389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:12.847693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754436462688:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-19T13:59:12.848098Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435758625180430:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:12.848238Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435754436462673:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:12.847971Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435757896280218:2105] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.847985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435757896280218:2105] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.848303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435754436462597:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.848316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:12.848322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435754436462597:2050] [node 3] ICP22 created new session: [6:7574435754436462689:2050]
2025-11-19T13:59:12.848328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435754436462689:2050] [node 3] ICS09 handshake done sender: [6:7574435754436462673:2115] self: [6:7574435754436462679:0] peer: [3:7574435758625180427:0] socket: 332 qp: -1
2025-11-19T13:59:12.848334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435754436462689:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:12.848339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:12.849005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435758625180426:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.849020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:12.849025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435758625180426:2111] [node 6] ICP22 created new session: [3:7574435758625180451:2111]
2025-11-19T13:59:12.849030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435758625180451:2111] [node 6] ICS09 handshake done sender: [3:7574435758625180430:2112] self: [3:7574435758625180427:0] peer: [6:7574435754436462679:0] socket: 325 qp: -1
2025-11-19T13:59:12.849034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435758625180451:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:12.849062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.849422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435758625180450:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10807
2025-11-19T13:59:12.850073Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435758625180439:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:12.847544Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435755779079921:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:12.847816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435755779079933:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-11-19T13:59:12.847949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435755779079837:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.847960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:12.847967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435755779079837:2048] [node 1] ICP22 created new session: [4:7574435755779079934:2048]
2025-11-19T13:59:12.847973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435755779079934:2048] [node 1] ICS09 handshake done sender: [4:7574435755779079921:2118] self: [4:7574435755779079931:0] peer: [1:7574435757923306393:0] socket: 335 qp: -1
2025-11-19T13:59:12.847978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435755779079934:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:12.847985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.849141Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435757896280218:2105] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:12.849341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435757896280156:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.849351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:12.849357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435757896280156:2049] [node 3] ICP22 created new session: [2:7574435757896280247:2049]
2025-11-19T13:59:12.849369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435757896280247:2049] [node 3] ICS09 handshake done sender: [2:7574435757896280218:2105] self: [2:7574435757896280213:0] peer: [3:7574435758625180446:0] socket: 322 qp: -1
2025-11-19T13:59:12.849377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435757896280247:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:12.849383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:12.850121Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435757896280217:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26645526ns) {Configure:104(26644495ns) {ProcessConfigured:135(26344508ns) {StartInitialHandshake:157(26343560ns) {DropHandshakes:469(482ns) {DropIncomingHandshake:441(44ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:476(1023ns) {SwitchToState:178(789ns)}}}}}
2025-11-19T13:59:12.851854Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435757896280241:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.851880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435757896280156:2049] [node 3] ICP18 handshake (actor [2:7574435757896280241:2117]) is too old
2025-11-19T13:59:12.852003Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435755779079917:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:12.852086Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435757923306386:2109] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:12.859330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435755779079838:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.859353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:12.859362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435755779079838:2049] [node 2] ICP22 created new session: [4:7574435755779079936:2049]
2025-11-19T13:59:12.859370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435755779079936:2049] [node 2] ICS09 handshake done sender: [4:7574435755779079917:2116] self: [4:7574435755779079919:0] peer: [2:7574435757896280214:0] socket: 327 qp: -1
2025-11-19T13:59:12.859376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435755779079936:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:12.859384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:12.860336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435758625180419:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(39061639ns) {HandleHandshakeStatus:329(39061147ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(110ns) SwitchToState:178(72ns) ProcessPendingSessionEvents:495(28358640ns) {ForwardSessionEventToSession:612(28351906ns) ForwardSessionEventToSession:612(1603ns) ForwardSessionEventToSession:612(971ns) ForwardSessionEventToSession:612(619ns)}}}
2025-11-19T13:59:12.860423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435758625180419:2107] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.863363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435758625180454:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10807
2025-11-19T13:59:12.865385Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435757923306392:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:12.865628Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435758625180423:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:12.865763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435758625180358:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.865784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:12.865791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435758625180358:2048] [node 1] ICP22 created new session: [3:7574435758625180455:2048]
2025-11-19T13:59:12.865798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435758625180455:2048] [node 1] ICS09 handshake done sender: [3:7574435758625180423:2109] self: [3:7574435758625180437:0] peer: [1:7574435757923306391:0] socket: 313 qp: -1
2025-11-19T13:59:12.865805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435758625180455:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:12.865812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:12.866800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435757923306322:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.866817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:12.866823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435757923306322:2049] [node 3] ICP22 created new session: [1:7574435757923306411:2049]
2025-11-19T13:59:12.866830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435757923306411:2049] [node 3] ICS09 handshake done sender: [1:7574435757923306392:2111] self: [1:7574435757923306391:0] peer: [3:7574435758625180437:0] socket: 312 qp: -1
2025-11-19T13:59:12.866835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435757923306411:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:12.866842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:12.867422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435755779079838:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:12.867974Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435755779079891:2096] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:12.868099Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435755779079930:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.868112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435755779079883:2092] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.868120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:12.868128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435755779079883:2092] [node 6] ICP22 created new session: [4:7574435755779079939:2092]
2025-11-19T13:59:12.868134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435755779079939:2092] [node 6] ICS09 handshake done sender: [4:7574435755779079891:2096] self: [4:7574435755779079890:0] peer: [6:7574435754436462676:0] socket: 306 qp: -1
2025-11-19T13:59:12.868139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435755779079939:2092] [node 6] ICS10 traffic start
2025-11-19T13:59:12.868145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:12.868258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435755779079883:2092] [node 6] ICP18 handshake (actor [4:7574435755779079930:2120]) is too old
2025-11-19T13:59:12.868648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435755779079837:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20673213ns) {HandleHandshakeStatus:329(20672314ns) {DropIncomingHandshake:441(113ns) DropOutgoingHandshake:456(1811ns) SwitchToState:178(60ns) ProcessPendingSessionEvents:495(20592152ns) {ForwardSessionEventToSession:612(20581832ns) ForwardSessionEventToSession:612(1095ns) ForwardSessionEventToSession:612(568ns) ForwardSessionEventToSession:612(557ns) ForwardSessionEventToSession:612(534ns) ForwardSessionEventToSession:612(1003ns) ForwardSessionEventToSession:612(697ns) ForwardSessionEventToSession:612(538ns) ForwardSessionEventToSession:612(579ns) ForwardSessionEventToSession:612(604ns)}}}
2025-11-19T13:59:12.869281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435757923306409:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50542
2025-11-19T13:59:12.869400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435758625180359:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.869423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:12.869428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435758625180359:2049] [node 2] ICP22 created new session: [3:7574435758625180457:2049]
2025-11-19T13:59:12.869434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435758625180457:2049] [node 2] ICS09 handshake done sender: [3:7574435758625180439:2116] self: [3:7574435758625180446:0] peer: [2:7574435757896280213:0] socket: 341 qp: -1
2025-11-19T13:59:12.869439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435758625180457:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:12.869460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:12.869674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435757923306382:2107] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:12.869843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435754436462617:2066] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18448) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435754436462667:2110] held: no
2025-11-19T13:59:12.870034Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758625180447:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.872463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435757896280252:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58681
2025-11-19T13:59:12.872633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435757896280253:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58681
2025-11-19T13:59:12.872585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754436462691:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-19T13:59:12.872602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754436462692:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-19T13:59:12.873255Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435757896280253:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.873302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435757896280224:2108] [node 5] ICP18 handshake (actor [2:7574435757896280253:2120]) is too old
2025-11-19T13:59:12.873350Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435754436462691:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.873371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435754436462596:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.873483Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435754436462667:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:12.873546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435754436462617:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.873558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:12.873564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435754436462617:2066] [node 4] ICP22 created new session: [6:7574435754436462694:2066]
2025-11-19T13:59:12.873569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435754436462694:2066] [node 4] ICS09 handshake done sender: [6:7574435754436462667:2110] self: [6:7574435754436462676:0] peer: [4:7574435755779079890:0] socket: 309 qp: -1
2025-11-19T13:59:12.873574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435754436462694:2066] [node 4] ICS10 traffic start
2025-11-19T13:59:12.873580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:12.873877Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435757896280249:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.873883Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435757896280249:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.874215Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435757896280212:2103] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:12.874400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435757896280155:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.874411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:12.874417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435757896280155:2048] [node 1] ICP22 created new session: [2:7574435757896280255:2048]
2025-11-19T13:59:12.874422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435757896280255:2048] [node 1] ICS09 handshake done sender: [2:7574435757896280212:2103] self: [2:7574435757896280229:0] peer: [1:7574435757923306385:0] socket: 307 qp: -1
2025-11-19T13:59:12.874427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435757896280255:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:12.874433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.874642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435754436462696:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31596
2025-11-19T13:59:12.875601Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435757896280249:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:12.884033Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435754436462691:2122] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.895825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435754436462596:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.895854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:12.895863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435754436462596:2049] [node 2] ICP22 created new session: [6:7574435754436462697:2049]
2025-11-19T13:59:12.895871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435754436462697:2049] [node 2] ICS09 handshake done sender: [6:7574435754436462691:2122] self: [6:7574435754436462693:0] peer: [2:7574435757896280237:0] socket: 363 qp: -1
2025-11-19T13:59:12.895880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435754436462697:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:12.895900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:12.892363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435755164340052:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64862
2025-11-19T13:59:12.895166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435755164340053:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64862
2025-11-19T13:59:12.895219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435755164340054:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64862
2025-11-19T13:59:12.895301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435755164340055:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64862
2025-11-19T13:59:12.895941Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435755164340054:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.895958Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435755164340055:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.895970Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435755164340042:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:12.896061Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435755164340041:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:12.896070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435755164339966:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.896080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:12.896085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435755164339966:2049] [node 2] ICP22 created new session: [5:7574435755164340056:2049]
2025-11-19T13:59:12.896090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435755164340056:2049] [node 2] ICS09 handshake done sender: [5:7574435755164340042:2114] self: [5:7574435755164340046:0] peer: [2:7574435757896280225:0] socket: 333 qp: -1
2025-11-19T13:59:12.896094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435755164340056:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:12.896113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:12.896143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435755164339992:2073] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.896150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:12.896155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435755164339992:2073] [node 4] ICP22 created new session: [5:7574435755164340058:2073]
2025-11-19T13:59:12.896159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435755164340058:2073] [node 4] ICS09 handshake done sender: [5:7574435755164340041:2113] self: [5:7574435755164340044:0] peer: [4:7574435755779079887:0] socket: 315 qp: -1
2025-11-19T13:59:12.896164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435755164340058:2073] [node 4] ICS10 traffic start
2025-11-19T13:59:12.896169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:12.896317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435755164339965:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.896329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435755164339967:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:12.899918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435757923306321:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.899973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:12.899990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435757923306321:2048] [node 2] ICP22 created new session: [1:7574435757923306413:2048]
2025-11-19T13:59:12.899999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435757923306413:2048] [node 2] ICS09 handshake done sender: [1:7574435757923306386:2109] self: [1:7574435757923306385:0] peer: [2:7574435757896280229:0] socket: 303 qp: -1
2025-11-19T13:59:12.900011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435757923306413:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:12.900062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:12.900226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435757923306321:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(44865099ns) {HandleHandshakeStatus:329(44864276ns) {DropIncomingHandshake:441(737ns) DropOutgoingHandshake:456(1193ns) SwitchToState:178(367ns) ProcessPendingSessionEvents:495(41711ns) {ForwardSessionEventToSession:612(28896ns) ForwardSessionEventToSession:612(1044ns) ForwardSessionEventToSession:612(1733ns) ForwardSessionEventToSession:612(1045ns) ForwardSessionEventToSession:612(589ns) ForwardSessionEventToSession:612(1038ns) ForwardSessionEventToSession:612(615ns) ForwardSessionEventToSession:612(568ns) ForwardSessionEventToSession:612(914ns) ForwardSessionEventToSession:612(1102ns)}}}
2025-11-19T13:59:12.904029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435757896280217:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.904059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:12.904070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435757896280217:2104] [node 6] ICP22 created new session: [2:7574435757896280257:2104]
2025-11-19T13:59:12.904077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435757896280257:2104] [node 6] ICS09 handshake done sender: [2:7574435757896280249:2118] self: [2:7574435757896280237:0] peer: [6:7574435754436462693:0] socket: 317 qp: -1
2025-11-19T13:59:12.904083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435757896280257:2104] [node 6] ICS10 traffic start
2025-11-19T13:59:12.904092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.916028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435757923306415:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50542
2025-11-19T13:59:12.916089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435757923306416:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50542
2025-11-19T13:59:12.916121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435757923306417:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50542
2025-11-19T13:59:12.916312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435757923306322:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:12.916467Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435758625180445:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.916494Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435758625180445:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.916395Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435757923306404:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.916405Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435757923306404:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:12.920128Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435757923306415:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:12.921549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435757923306418:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50542
2025-11-19T13:59:12.923929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435757923306390:2110] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:12.924122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435758625180359:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(73057671ns) {HandleHandshakeStatus:329(73056488ns) {DropIncomingHandshake:441(653ns) DropOutgoingHandshake:456(5569ns) SwitchToState:178(73ns) ProcessPendingSessionEvents:495(26853796ns) {ForwardSessionEventToSession:612(26839856ns) ForwardSessionEventToSession:612(2517ns) ForwardSessionEventToSession:612(1106ns) ForwardSessionEventToSession:612(922ns) ForwardSessionEventToSession:612(540ns) ForwardSessionEventToSession:612(939ns) ForwardSessionEventToSession:612(848ns) ForwardSessionEventToSession:612(595ns) ForwardSessionEventToSession:612(465ns) ForwardSessionEventToSession:612(1235ns)}}}
2025-11-19T13:59:12.925743Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435757923306404:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:12.926707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435755164339966:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(30604164ns) {HandleHandshakeStatus:329(30603727ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(1062ns) SwitchToState:178(57ns) ProcessPendingSessionEvents:495(29157139ns) {ForwardSessionEventToSession:612(29145028ns) ForwardSessionEventToSession:612(1590ns) ForwardSessionEventToSession:612(946ns) ForwardSessionEventToSession:612(515ns) ForwardSessionEventToSession:612(756ns) ForwardSessionEventToSession:612(908ns) ForwardSessionEventToSession:612(718ns) ForwardSessionEventToSession:612(553ns) ForwardSessionEventToSession:612(829ns) ForwardSessionEventToSession:612(584ns)}}}
2025-11-19T13:59:12.927713Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435758625180445:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:12.927989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435757923306397:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.928013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:12.928021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435757923306397:2113] [node 5] ICP22 created new session: [1:7574435757923306421:2113]
2025-11-19T13:59:12.928028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435757923306421:2113] [node 5] ICS09 handshake done sender: [1:7574435757923306404:2114] self: [1:7574435757923306398:0] peer: [5:7574435755164340060:0] socket: 354 qp: -1
2025-11-19T13:59:12.928035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435757923306421:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:12.928044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:12.929112Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435755164339965:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28357) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435755164340054:2119] held: no
2025-11-19T13:59:12.932232Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435757923306321:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:12.930598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435758625180429:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:12.930624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:12.930632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435758625180429:2110] [node 5] ICP22 created new session: [3:7574435758625180461:2110]
2025-11-19T13:59:12.930638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435758625180461:2110] [node 5] ICS09 handshake done sender: [3:7574435758625180445:2118] self: [3:7574435758625180441:0] peer: [5:7574435755164340061:0] socket: 320 qp: -1
2025-11-19T13:59:12.930643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435758625180461:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:12.930654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:12.932445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435755164340062:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64862
2025-11-19T13:59:12.932466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435755164340063:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64862
2025-11-19T13:59:12.932589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435755164339967:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14122) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435755164340055:2120] held: no
2025-11-19T13:59:12.932621Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435755164340054:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:12.932725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435755164339965:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.932737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:12.932742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435755164339965:2048] [node 1] ICP22 created new session: [5:7574435755164340064:2048]
2025-11-19T13:59:12.932747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435755164340064:2048] [node 1] ICS09 handshake done sender: [5:7574435755164340054:2119] self: [5:7574435755164340060:0] peer: [1:7574435757923306398:0] socket: 368 qp: -1
2025-11-19T13:59:12.932753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435755164340064:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:12.932758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:12.933334Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435755164340055:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:12.933527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435755164339967:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:12.933539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:12.933545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435755164339967:2050] [node 3] ICP22 created new session: [5:7574435755164340066:2050]
2025-11-19T13:59:12.933555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435755164340066:2050] [node 3] ICS09 handshake done sender: [5:7574435755164340055:2120] self: [5:7574435755164340061:0] peer: [3:7574435758625180441:0] socket: 369 qp: -1
2025-11-19T13:59:12.933561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435755164340066:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:12.933567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:12.952537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:12.952626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435757896280255:2048] [node 1] ICS01 socket: 307 reason# EndOfStream
2025-11-19T13:59:12.952634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435757896280155:2048] [node 1] ICP30 unregister session Session# [2:7574435757896280255:2048] VirtualId# [2:7574435757896280229:0]
2025-11-19T13:59:12.952641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435757896280255:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.952685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:12.952720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:12.953204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:12.953276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435758625180455:2048] [node 1] ICS01 socket: 313 reason# EndOfStream
2025-11-19T13:59:12.953282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435758625180358:2048] [node 1] ICP30 unregister session Session# [3:7574435758625180455:2048] VirtualId# [3:7574435758625180437:0]
2025-11-19T13:59:12.953288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435758625180455:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.953326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:12.953345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:12.957823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:12.957907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435755779079934:2048] [node 1] ICS07 socket disconnect 335 reason# EndOfStream
2025-11-19T13:59:12.957913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435755779079934:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.957962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:12.957983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435755779079934:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:12.958721Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435755779079837:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28357) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:12.958733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435755779079934:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:12.958738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435755779079837:2048] [node 1] ICP30 unregister session Session# [4:7574435755779079934:2048] VirtualId# [4:7574435755779079931:0]
2025-11-19T13:59:12.958751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:12.958756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435755779079837:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:12.958758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:12.960071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:12.960150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435754436462682:2048] [node 1] ICS07 socket disconnect 305 reason# EndOfStream
2025-11-19T13:59:12.960156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435754436462682:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.960204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:12.960225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435754436462682:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:12.960740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435754436462595:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28357) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:12.960751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435754436462682:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:12.960756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435754436462595:2048] [node 1] ICP30 unregister session Session# [6:7574435754436462682:2048] VirtualId# [6:7574435754436462665:0]
2025-11-19T13:59:12.960767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:12.960772Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435754436462595:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:12.960774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:12.961184Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:12.961254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435755164340064:2048] [node 1] ICS01 socket: 368 reason# EndOfStream
2025-11-19T13:59:12.961260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435755164339965:2048] [node 1] ICP30 unregister session Session# [5:7574435755164340064:2048] VirtualId# [5:7574435755164340060:0]
2025-11-19T13:59:12.961265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435755164340064:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.961313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:12.961335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:12.967997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435755779079837:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:12.972959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435754436462595:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:12.979849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:12.979947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435758625180457:2049] [node 2] ICS07 socket disconnect 341 reason# EndOfStream
2025-11-19T13:59:12.979954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435758625180457:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.980010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:12.980029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435758625180457:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:12.988731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435758625180359:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14821) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:12.988749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435758625180457:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:12.989140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435758625180359:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14821) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:12.989149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435758625180457:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:12.989154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435758625180359:2049] [node 2] ICP30 unregister session Session# [3:7574435758625180457:2049] VirtualId# [3:7574435758625180446:0]
2025-11-19T13:59:12.989172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:12.989177Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435758625180359:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14821) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:12.989181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14821) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:12.996547Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:12.996635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435755779079936:2049] [node 2] ICS01 socket: 327 reason# EndOfStream
2025-11-19T13:59:12.996644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435755779079838:2049] [node 2] ICP30 unregister session Session# [4:7574435755779079936:2049] VirtualId# [4:7574435755779079919:0]
2025-11-19T13:59:12.996650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435755779079936:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:12.996693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:12.996713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:13.001642Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:13.001726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435755164340056:2049] [node 2] ICS07 socket disconnect 333 reason# EndOfStream
2025-11-19T13:59:13.001732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435755164340056:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.001779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:13.001801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435755164340056:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:13.002584Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435755164339966:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14821) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:13.002597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435755164340056:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:13.002602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435755164339966:2049] [node 2] ICP30 unregister session Session# [5:7574435755164340056:2049] VirtualId# [5:7574435755164340046:0]
2025-11-19T13:59:13.002615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:13.002621Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435755164339966:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14821) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:13.002623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14821) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:13.003850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:13.003929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435754436462697:2049] [node 2] ICS01 socket: 363 reason# EndOfStream
2025-11-19T13:59:13.003945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435754436462596:2049] [node 2] ICP30 unregister session Session# [6:7574435754436462697:2049] VirtualId# [6:7574435754436462693:0]
2025-11-19T13:59:13.003951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435754436462697:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.004015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:13.004045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:13.006248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:13.006327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435755164340066:2050] [node 3] ICS01 socket: 369 reason# EndOfStream
2025-11-19T13:59:13.006333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435755164339967:2050] [node 3] ICP30 unregister session Session# [5:7574435755164340066:2050] VirtualId# [5:7574435755164340061:0]
2025-11-19T13:59:13.006339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435755164340066:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.006383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:13.006405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:13.006272Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:13.006356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435754436462689:2050] [node 3] ICS07 socket disconnect 332 reason# EndOfStream
2025-11-19T13:59:13.006362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435754436462689:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.006414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:13.006449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435754436462689:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:13.007034Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435754436462597:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14122) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:13.007047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435754436462689:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:13.007053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435754436462597:2050] [node 3] ICP30 unregister session Session# [6:7574435754436462689:2050] VirtualId# [6:7574435754436462679:0]
2025-11-19T13:59:13.007066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:13.007070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435754436462597:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14122) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:13.007073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14122) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:13.015840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435755164339966:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:13.016125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435754436462597:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:59:13.023984Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:13.024079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435754436462694:2066] [node 4] ICS01 socket: 309 reason# EndOfStream
2025-11-19T13:59:13.024089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435754436462617:2066] [node 4] ICP30 unregister session Session# [6:7574435754436462694:2066] VirtualId# [6:7574435754436462676:0]
2025-11-19T13:59:13.024095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435754436462694:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.024153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:13.024190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:13.119832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435761045869768:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:13.119857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435761045869769:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:13.123401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435759350702833:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:13.123413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435759350702834:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:13.147943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435762873352357:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:13.147957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435762873352358:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:13.179935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435762916029016:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:13.179954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435762916029017:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:13.179959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435762916029018:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:13.184038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435762310086105:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:13.184049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435762310086106:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:13.184054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435762310086107:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:13.243775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435760024613681:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:13.243828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435760024613682:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:13.243834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435760024613683:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:13.252399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435760024613699:2060] [node 4] ICP01 ready to work
2025-11-19T13:59:13.252509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435760024613704:2066] [node 5] ICP01 ready to work
2025-11-19T13:59:13.252053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435762916029078:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:13.252662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435762310086134:2068] [node 6] ICP01 ready to work
2025-11-19T13:59:13.252737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435762310086135:2071] [node 4] ICP01 ready to work
2025-11-19T13:59:13.252748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435761045869830:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:13.252841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435761045869831:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.267790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435761045869840:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:13.268189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435761045869839:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51210
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.276377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435762916029084:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:13.278804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435762916029083:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35881
2025-11-19T13:59:13.279853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435762873352357:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18575709ns) {Configure:104(18574337ns) {ProcessConfigured:135(18160720ns) {StartInitialHandshake:157(18160389ns) {DropHandshakes:469(1142ns) {DropIncomingHandshake:441(340ns) DropOutgoingHandshake:456(160ns)} GenerateSessionVirtualId:988(175ns) PrepareNewSessionHandshake:476(879ns) {SwitchToState:178(57ns)}}}}}
2025-11-19T13:59:13.280336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435762873352424:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:13.280414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435762873352425:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:13.280558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435762873352429:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:13.279059Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435760024613704:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(25568131ns) {Configure:104(25567787ns) {ProcessConfigured:135(25372819ns) {StartInitialHandshake:157(25372694ns) {DropHandshakes:469(200ns) {DropIncomingHandshake:441(23ns) DropOutgoingHandshake:456(14ns)} GenerateSessionVirtualId:988(40ns) PrepareNewSessionHandshake:476(1239ns) {SwitchToState:178(881ns)}}}}}
2025-11-19T13:59:13.279590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435760024613756:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41577
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.293786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435761045869846:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51210
2025-11-19T13:59:13.293814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435761045869847:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51210
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.296045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435759350702899:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:13.296141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435759350702900:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:13.296226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435759350702901:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:13.298449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435759350702902:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26937
2025-11-19T13:59:13.298462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435759350702903:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26937
2025-11-19T13:59:13.298474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435759350702904:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26937
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.308249Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435762873352424:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27604964ns) {Configure:104(27604358ns) {ProcessConfigured:135(27386404ns) {StartInitialHandshake:157(27385859ns) {DropHandshakes:469(302ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(116ns) PrepareNewSessionHandshake:476(1867ns) {SwitchToState:178(1011ns)}}}}}
2025-11-19T13:59:13.308970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435762873352426:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33827
2025-11-19T13:59:13.309047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435762873352427:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33827
2025-11-19T13:59:13.309068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435762873352428:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33827
2025-11-19T13:59:13.311451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435762916029095:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35881
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.324566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435762310086183:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33064
2025-11-19T13:59:13.324637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435762310086184:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33064
2025-11-19T13:59:13.328029Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435760024613756:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.329076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435761045869850:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51210
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.331466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435760024613760:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41577
2025-11-19T13:59:13.331487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435760024613761:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41577
2025-11-19T13:59:13.331504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435760024613699:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.328893Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435762873352426:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.329915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435762873352357:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.332968Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435762916029083:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.333074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435762916029080:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.333077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435762916029080:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.333643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435762916029016:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.334608Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435762310086183:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.334663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435762310086105:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.335151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435759350702915:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26937
2025-11-19T13:59:13.335306Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435761045869836:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.335317Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435761045869836:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.335893Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435761045869838:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.335897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435761045869838:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.336321Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435761045869833:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.336324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435761045869833:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.340112Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435762916029080:2109] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.340395Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759350702904:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.340238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435760024613763:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41577
2025-11-19T13:59:13.340627Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435760024613760:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.340644Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435760024613756:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:13.340705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435760024613699:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.340718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:13.340724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435760024613699:2060] [node 4] ICP22 created new session: [6:7574435760024613764:2060]
2025-11-19T13:59:13.340730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435760024613764:2060] [node 4] ICS09 handshake done sender: [6:7574435760024613756:2115] self: [6:7574435760024613762:0] peer: [4:7574435762916029079:0] socket: 320 qp: -1
2025-11-19T13:59:13.340738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435760024613764:2060] [node 4] ICS10 traffic start
2025-11-19T13:59:13.340746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:13.340876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435760024613682:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.341960Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435761045869839:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.342029Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435761045869836:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:13.342178Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435761045869838:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:13.344012Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435762916029095:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.344028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435762916029078:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.344039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:13.344047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435762916029078:2108] [node 6] ICP22 created new session: [4:7574435762916029099:2108]
2025-11-19T13:59:13.344054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435762916029099:2108] [node 6] ICS09 handshake done sender: [4:7574435762916029080:2109] self: [4:7574435762916029079:0] peer: [6:7574435760024613762:0] socket: 306 qp: -1
2025-11-19T13:59:13.344059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435762916029099:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:13.344067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:13.344455Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759350702903:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.347808Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435761045869833:2109] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.348063Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435762310086184:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.348199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435762310086106:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.348218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435762310086186:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33064
2025-11-19T13:59:13.348427Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759350702908:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.348435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435759350702908:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.348865Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759350702902:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.349074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435761045869847:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.349096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435761045869769:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.349106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:13.349112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435761045869769:2049] [node 3] ICP22 created new session: [1:7574435761045869854:2049]
2025-11-19T13:59:13.349117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435761045869854:2049] [node 3] ICS09 handshake done sender: [1:7574435761045869836:2111] self: [1:7574435761045869834:0] peer: [3:7574435762873352439:0] socket: 310 qp: -1
2025-11-19T13:59:13.349123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435761045869854:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:13.349130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:13.349228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435761045869831:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.349234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:13.349238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435761045869831:2108] [node 4] ICP22 created new session: [1:7574435761045869856:2108]
2025-11-19T13:59:13.349242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435761045869856:2108] [node 4] ICS09 handshake done sender: [1:7574435761045869838:2112] self: [1:7574435761045869837:0] peer: [4:7574435762916029097:0] socket: 311 qp: -1
2025-11-19T13:59:13.349245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435761045869856:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:13.349250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:13.349327Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435761045869846:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.349333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435761045869830:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.349338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:13.349342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435761045869830:2107] [node 5] ICP22 created new session: [1:7574435761045869858:2107]
2025-11-19T13:59:13.349345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435761045869858:2107] [node 5] ICS09 handshake done sender: [1:7574435761045869833:2109] self: [1:7574435761045869832:0] peer: [5:7574435762310086185:0] socket: 309 qp: -1
2025-11-19T13:59:13.349348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435761045869858:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:13.349352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:13.349565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435761045869830:2107] [node 5] ICP18 handshake (actor [1:7574435761045869846:2115]) is too old
2025-11-19T13:59:13.349863Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435759350702908:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:13.352135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435759350702900:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.352153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:13.352159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435759350702900:2109] [node 6] ICP22 created new session: [2:7574435759350702921:2109]
2025-11-19T13:59:13.352164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435759350702921:2109] [node 6] ICS09 handshake done sender: [2:7574435759350702908:2115] self: [2:7574435759350702907:0] peer: [6:7574435760024613766:0] socket: 333 qp: -1
2025-11-19T13:59:13.352169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435759350702921:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:13.352175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.352494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435759350702920:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26937
2025-11-19T13:59:13.353698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435762310086105:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2760) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435762310086183:2115] held: no
2025-11-19T13:59:13.353733Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435762310086183:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:13.356266Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759350702906:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.356276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435759350702906:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.357967Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435762873352428:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.358228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435762873352440:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33827
2025-11-19T13:59:13.358419Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435762873352426:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:13.358524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435762873352357:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.358538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:13.358543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435762873352357:2048] [node 1] ICP22 created new session: [3:7574435762873352442:2048]
2025-11-19T13:59:13.358548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435762873352442:2048] [node 1] ICS09 handshake done sender: [3:7574435762873352426:2112] self: [3:7574435762873352439:0] peer: [1:7574435761045869834:0] socket: 324 qp: -1
2025-11-19T13:59:13.358555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435762873352442:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:13.358561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:13.359432Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435761045869850:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.359583Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435759350702906:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:13.359678Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759350702915:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.359700Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759350702920:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.359708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435759350702899:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.359719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:13.359726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435759350702899:2108] [node 5] ICP22 created new session: [2:7574435759350702925:2108]
2025-11-19T13:59:13.359730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435759350702925:2108] [node 5] ICS09 handshake done sender: [2:7574435759350702906:2114] self: [2:7574435759350702905:0] peer: [5:7574435762310086187:0] socket: 332 qp: -1
2025-11-19T13:59:13.359734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435759350702925:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:13.359739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:13.363933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435759350702833:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.383923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435762916029078:2108] [node 6] ICP18 handshake (actor [4:7574435762916029095:2116]) is too old
2025-11-19T13:59:13.384002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435762916029098:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35881
2025-11-19T13:59:13.384581Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435762916029083:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.384671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435762916029016:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.384687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:13.384694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435762916029016:2048] [node 1] ICP22 created new session: [4:7574435762916029103:2048]
2025-11-19T13:59:13.384699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435762916029103:2048] [node 1] ICS09 handshake done sender: [4:7574435762916029083:2111] self: [4:7574435762916029097:0] peer: [1:7574435761045869837:0] socket: 317 qp: -1
2025-11-19T13:59:13.384705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435762916029103:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:13.384712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:13.384861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435762916029101:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35881
2025-11-19T13:59:13.386371Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435762916029101:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.386397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435762916029017:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.386514Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759350702910:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.386525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435759350702910:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.387186Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435759350702910:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:13.387264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435759350702901:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.387281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:13.387286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435759350702901:2110] [node 4] ICP22 created new session: [2:7574435759350702931:2110]
2025-11-19T13:59:13.387291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435759350702931:2110] [node 4] ICS09 handshake done sender: [2:7574435759350702910:2116] self: [2:7574435759350702909:0] peer: [4:7574435762916029106:0] socket: 331 qp: -1
2025-11-19T13:59:13.387297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435759350702931:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:13.387306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:13.388053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435762310086105:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.388073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:13.388080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435762310086105:2048] [node 1] ICP22 created new session: [5:7574435762310086188:2048]
2025-11-19T13:59:13.388087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435762310086188:2048] [node 1] ICS09 handshake done sender: [5:7574435762310086183:2115] self: [5:7574435762310086185:0] peer: [1:7574435761045869832:0] socket: 315 qp: -1
2025-11-19T13:59:13.388094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435762310086188:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:13.388103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.388503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435762873352444:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33827
2025-11-19T13:59:13.388912Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435761045869849:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.388927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435761045869849:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.390934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435761045869769:2049] [node 3] ICP18 handshake (actor [1:7574435761045869850:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.391367Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435760024613761:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.391477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435760024613682:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14697) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435760024613760:2116] held: no
2025-11-19T13:59:13.391546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435760024613768:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41577
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.391784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435761045869864:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51210
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.392333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435760024613769:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41577
2025-11-19T13:59:13.392383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435760024613683:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.392517Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435760024613760:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:13.392027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435762310086106:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14697) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435762310086184:2116] held: no
2025-11-19T13:59:13.392198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435762310086190:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33064
2025-11-19T13:59:13.392917Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435761045869849:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:13.393331Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435762873352427:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.393321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435761045869768:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.393339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:13.393345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435761045869768:2048] [node 2] ICP22 created new session: [1:7574435761045869866:2048]
2025-11-19T13:59:13.393350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435761045869866:2048] [node 2] ICS09 handshake done sender: [1:7574435761045869849:2117] self: [1:7574435761045869848:0] peer: [2:7574435759350702929:0] socket: 350 qp: -1
2025-11-19T13:59:13.393355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435761045869866:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:13.393361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.395883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435759350702933:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26937
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.395906Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435762873352431:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.395914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435762873352431:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.396338Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435762873352444:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.396486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435762873352358:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.396508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435762873352446:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33827
2025-11-19T13:59:13.396680Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435762873352431:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:13.396764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435762873352424:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.396778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:13.396784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435762873352424:2107] [node 6] ICP22 created new session: [3:7574435762873352450:2107]
2025-11-19T13:59:13.396790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435762873352450:2107] [node 6] ICS09 handshake done sender: [3:7574435762873352431:2115] self: [3:7574435762873352430:0] peer: [6:7574435760024613771:0] socket: 337 qp: -1
2025-11-19T13:59:13.396796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435762873352450:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:13.396803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.398334Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435759350702920:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:13.400940Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435760024613769:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.400957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435760024613682:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.400969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:13.400975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435760024613682:2049] [node 2] ICP22 created new session: [6:7574435760024613773:2049]
2025-11-19T13:59:13.400981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435760024613773:2049] [node 2] ICS09 handshake done sender: [6:7574435760024613760:2116] self: [6:7574435760024613766:0] peer: [2:7574435759350702907:0] socket: 339 qp: -1
2025-11-19T13:59:13.400986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435760024613773:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:13.400994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:13.401136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435760024613772:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41577
2025-11-19T13:59:13.401193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435760024613681:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.401205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435760024613775:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41577
2025-11-19T13:59:13.402468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435762916029017:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14697) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435762916029101:2118] held: no
2025-11-19T13:59:13.401396Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759350702912:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.401404Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435759350702912:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.401732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435759350702833:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.401741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:13.401746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435759350702833:2048] [node 1] ICP22 created new session: [2:7574435759350702934:2048]
2025-11-19T13:59:13.401751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435759350702934:2048] [node 1] ICS09 handshake done sender: [2:7574435759350702920:2119] self: [2:7574435759350702929:0] peer: [1:7574435761045869848:0] socket: 316 qp: -1
2025-11-19T13:59:13.401757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435759350702934:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:13.401762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.408068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435762310086191:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33064
2025-11-19T13:59:13.408236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435762310086193:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33064
2025-11-19T13:59:13.408355Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435762310086184:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.409553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435762310086106:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.409571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:13.409578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435762310086106:2049] [node 2] ICP22 created new session: [5:7574435762310086196:2049]
2025-11-19T13:59:13.409586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435762310086196:2049] [node 2] ICS09 handshake done sender: [5:7574435762310086184:2116] self: [5:7574435762310086187:0] peer: [2:7574435759350702905:0] socket: 334 qp: -1
2025-11-19T13:59:13.409592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435762310086196:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:13.409601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:13.409719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435762310086195:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33064
2025-11-19T13:59:13.412746Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435760024613761:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.412904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435762873352453:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33827
2025-11-19T13:59:13.413018Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435759350702912:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:13.413179Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435762873352444:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:13.413240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435762873352358:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.413254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:13.413260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435762873352358:2049] [node 2] ICP22 created new session: [3:7574435762873352454:2049]
2025-11-19T13:59:13.413265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435762873352454:2049] [node 2] ICS09 handshake done sender: [3:7574435762873352444:2119] self: [3:7574435762873352449:0] peer: [2:7574435759350702911:0] socket: 354 qp: -1
2025-11-19T13:59:13.413270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435762873352454:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:13.413277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:13.414124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435759350702834:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.414139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:13.414145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435759350702834:2049] [node 3] ICP22 created new session: [2:7574435759350702937:2049]
2025-11-19T13:59:13.414150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435759350702937:2049] [node 3] ICS09 handshake done sender: [2:7574435759350702912:2117] self: [2:7574435759350702911:0] peer: [3:7574435762873352449:0] socket: 329 qp: -1
2025-11-19T13:59:13.414154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435759350702937:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:13.414160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:13.415378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435760024613683:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.415393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:13.415401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435760024613683:2050] [node 3] ICP22 created new session: [6:7574435760024613777:2050]
2025-11-19T13:59:13.415407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435760024613777:2050] [node 3] ICS09 handshake done sender: [6:7574435760024613761:2117] self: [6:7574435760024613771:0] peer: [3:7574435762873352430:0] socket: 340 qp: -1
2025-11-19T13:59:13.415412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435760024613777:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:13.415419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:13.416318Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435761045869842:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.416332Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435761045869842:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.423593Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435762310086195:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.423645Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435762310086193:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.424837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435762310086107:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.426135Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435762873352435:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.426149Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435762873352435:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.426564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435762873352424:2107] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:13.426909Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435761045869842:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:13.427030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435761045869840:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.427052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:13.427058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435761045869840:2110] [node 6] ICP22 created new session: [1:7574435761045869869:2110]
2025-11-19T13:59:13.427063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435761045869869:2110] [node 6] ICS09 handshake done sender: [1:7574435761045869842:2114] self: [1:7574435761045869841:0] peer: [6:7574435760024613776:0] socket: 349 qp: -1
2025-11-19T13:59:13.427069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435761045869869:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:13.427079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:13.427538Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435762873352435:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:13.425208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435762916029107:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35881
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.428066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435762916029108:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35881
2025-11-19T13:59:13.428106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435762916029109:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35881
2025-11-19T13:59:13.428130Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435762916029101:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:13.427918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435760024613772:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.428608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435762916029017:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.428622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:13.428629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435762916029017:2049] [node 2] ICP22 created new session: [4:7574435762916029110:2049]
2025-11-19T13:59:13.428634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435762916029110:2049] [node 2] ICS09 handshake done sender: [4:7574435762916029101:2118] self: [4:7574435762916029106:0] peer: [2:7574435759350702909:0] socket: 335 qp: -1
2025-11-19T13:59:13.428640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435762916029110:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:13.428646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:13.432167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435762873352429:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.432186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:13.432192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435762873352429:2109] [node 5] ICP22 created new session: [3:7574435762873352457:2109]
2025-11-19T13:59:13.432197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435762873352457:2109] [node 5] ICS09 handshake done sender: [3:7574435762873352435:2117] self: [3:7574435762873352434:0] peer: [5:7574435762310086198:0] socket: 364 qp: -1
2025-11-19T13:59:13.432202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435762873352457:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:13.432208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:13.432553Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435762916029108:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.432580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435762916029018:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.432779Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435762873352433:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.432787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435762873352433:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.433579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435762916029113:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35881
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.433716Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435760024613681:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2760) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435760024613769:2120] held: no
2025-11-19T13:59:13.433779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435760024613704:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.433802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435760024613779:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41577
2025-11-19T13:59:13.433695Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435762310086191:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.435184Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435762916029109:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.435294Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435762873352433:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:13.435471Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435762916029108:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:13.434796Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435760024613769:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:13.434861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435760024613681:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.434871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:13.434878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435760024613681:2048] [node 1] ICP22 created new session: [6:7574435760024613781:2048]
2025-11-19T13:59:13.434884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435760024613781:2048] [node 1] ICS09 handshake done sender: [6:7574435760024613769:2120] self: [6:7574435760024613776:0] peer: [1:7574435761045869841:0] socket: 358 qp: -1
2025-11-19T13:59:13.434889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435760024613781:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:13.434897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:13.435547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435762916029018:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.435561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:13.435566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435762916029018:2050] [node 3] ICP22 created new session: [4:7574435762916029115:2050]
2025-11-19T13:59:13.435571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435762916029115:2050] [node 3] ICS09 handshake done sender: [4:7574435762916029108:2120] self: [4:7574435762916029112:0] peer: [3:7574435762873352432:0] socket: 366 qp: -1
2025-11-19T13:59:13.435576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435762916029115:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:13.435582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:13.436399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435762873352425:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.436425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:13.436430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435762873352425:2108] [node 4] ICP22 created new session: [3:7574435762873352460:2108]
2025-11-19T13:59:13.436434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435762873352460:2108] [node 4] ICS09 handshake done sender: [3:7574435762873352433:2116] self: [3:7574435762873352432:0] peer: [4:7574435762916029112:0] socket: 355 qp: -1
2025-11-19T13:59:13.436440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435762873352460:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:13.436447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.436957Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435762310086107:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9092) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435762310086195:2121] held: no
2025-11-19T13:59:13.437010Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435762310086177:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.437014Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435762310086177:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.437533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435762310086200:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33064
2025-11-19T13:59:13.437957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435762310086135:2071] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.438088Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435762310086195:2121] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.438278Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435762916029092:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.438284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435762916029092:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.438920Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435762916029092:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:13.438991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435762916029084:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.439005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:13.439010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435762916029084:2110] [node 5] ICP22 created new session: [4:7574435762916029118:2110]
2025-11-19T13:59:13.439015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435762916029118:2110] [node 5] ICS09 handshake done sender: [4:7574435762916029092:2115] self: [4:7574435762916029091:0] peer: [5:7574435762310086202:0] socket: 327 qp: -1
2025-11-19T13:59:13.439019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435762916029118:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:13.439025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:13.438214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435760024613783:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41577
2025-11-19T13:59:13.441416Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435762310086177:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:13.441609Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435760024613772:2121] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:13.441675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435760024613704:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.441700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:13.441706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435760024613704:2066] [node 5] ICP22 created new session: [6:7574435760024613784:2066]
2025-11-19T13:59:13.441711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435760024613784:2066] [node 5] ICS09 handshake done sender: [6:7574435760024613772:2121] self: [6:7574435760024613780:0] peer: [5:7574435762310086176:0] socket: 361 qp: -1
2025-11-19T13:59:13.441717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435760024613784:2066] [node 5] ICS10 traffic start
2025-11-19T13:59:13.441724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:13.443281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435762310086107:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.443296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:13.443303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435762310086107:2050] [node 3] ICP22 created new session: [5:7574435762310086204:2050]
2025-11-19T13:59:13.443309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435762310086204:2050] [node 3] ICS09 handshake done sender: [5:7574435762310086195:2121] self: [5:7574435762310086198:0] peer: [3:7574435762873352434:0] socket: 307 qp: -1
2025-11-19T13:59:13.443314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435762310086204:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:13.443322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:13.443401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435762310086134:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.443412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:13.443415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435762310086134:2068] [node 6] ICP22 created new session: [5:7574435762310086206:2068]
2025-11-19T13:59:13.443419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435762310086206:2068] [node 6] ICS09 handshake done sender: [5:7574435762310086177:2113] self: [5:7574435762310086176:0] peer: [6:7574435760024613780:0] socket: 360 qp: -1
2025-11-19T13:59:13.443422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435762310086206:2068] [node 6] ICS10 traffic start
2025-11-19T13:59:13.443426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:13.443545Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435762310086135:2071] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10636) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435762310086191:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.460119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435762310086203:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33064
2025-11-19T13:59:13.460507Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435762310086191:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:13.460570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435762310086135:2071] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.460585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:13.460593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435762310086135:2071] [node 4] ICP22 created new session: [5:7574435762310086208:2071]
2025-11-19T13:59:13.460599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435762310086208:2071] [node 4] ICS09 handshake done sender: [5:7574435762310086191:2119] self: [5:7574435762310086202:0] peer: [4:7574435762916029091:0] socket: 357 qp: -1
2025-11-19T13:59:13.460607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435762310086208:2071] [node 4] ICS10 traffic start
2025-11-19T13:59:13.460618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:13.484680Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:13.484764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435759350702934:2048] [node 1] ICS01 socket: 316 reason# EndOfStream
2025-11-19T13:59:13.484774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435759350702833:2048] [node 1] ICP30 unregister session Session# [2:7574435759350702934:2048] VirtualId# [2:7574435759350702929:0]
2025-11-19T13:59:13.484781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435759350702934:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.484823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:13.484843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:13.485416Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:13.485480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435762873352442:2048] [node 1] ICS01 socket: 324 reason# EndOfStream
2025-11-19T13:59:13.485485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435762873352357:2048] [node 1] ICP30 unregister session Session# [3:7574435762873352442:2048] VirtualId# [3:7574435762873352439:0]
2025-11-19T13:59:13.485489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435762873352442:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.485512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:13.485529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:13.493952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:13.494011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435760024613781:2048] [node 1] ICS01 socket: 358 reason# EndOfStream
2025-11-19T13:59:13.494016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435760024613681:2048] [node 1] ICP30 unregister session Session# [6:7574435760024613781:2048] VirtualId# [6:7574435760024613776:0]
2025-11-19T13:59:13.494020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435760024613781:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.494043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:13.494066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:13.493643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:13.493720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435762310086188:2048] [node 1] ICS01 socket: 315 reason# EndOfStream
2025-11-19T13:59:13.493727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435762310086105:2048] [node 1] ICP30 unregister session Session# [5:7574435762310086188:2048] VirtualId# [5:7574435762310086185:0]
2025-11-19T13:59:13.493732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435762310086188:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.493774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:13.493795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:13.493827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:13.493881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435762916029103:2048] [node 1] ICS01 socket: 317 reason# EndOfStream
2025-11-19T13:59:13.493885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435762916029016:2048] [node 1] ICP30 unregister session Session# [4:7574435762916029103:2048] VirtualId# [4:7574435762916029097:0]
2025-11-19T13:59:13.493889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435762916029103:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.493911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:13.493928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:13.495181Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:13.495243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435762873352454:2049] [node 2] ICS01 socket: 354 reason# EndOfStream
2025-11-19T13:59:13.495248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435762873352358:2049] [node 2] ICP30 unregister session Session# [3:7574435762873352454:2049] VirtualId# [3:7574435762873352449:0]
2025-11-19T13:59:13.495252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435762873352454:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.495278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:13.495294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:13.500907Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:13.500993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435762310086196:2049] [node 2] ICS01 socket: 334 reason# EndOfStream
2025-11-19T13:59:13.501001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435762310086106:2049] [node 2] ICP30 unregister session Session# [5:7574435762310086196:2049] VirtualId# [5:7574435762310086187:0]
2025-11-19T13:59:13.501006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435762310086196:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.501055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:13.501077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:13.501257Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:13.501319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435762916029110:2049] [node 2] ICS01 socket: 335 reason# EndOfStream
2025-11-19T13:59:13.501323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435762916029017:2049] [node 2] ICP30 unregister session Session# [4:7574435762916029110:2049] VirtualId# [4:7574435762916029106:0]
2025-11-19T13:59:13.501327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435762916029110:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.501352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:13.501373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:13.501108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:13.501175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435760024613773:2049] [node 2] ICS01 socket: 339 reason# EndOfStream
2025-11-19T13:59:13.501179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435760024613682:2049] [node 2] ICP30 unregister session Session# [6:7574435760024613773:2049] VirtualId# [6:7574435760024613766:0]
2025-11-19T13:59:13.501184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435760024613773:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.501209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:13.501230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:13.512145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:13.512222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435762310086204:2050] [node 3] ICS01 socket: 307 reason# EndOfStream
2025-11-19T13:59:13.512227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435762310086107:2050] [node 3] ICP30 unregister session Session# [5:7574435762310086204:2050] VirtualId# [5:7574435762310086198:0]
2025-11-19T13:59:13.512230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435762310086204:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.512257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:13.512277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:13.511941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:13.512029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435760024613777:2050] [node 3] ICS01 socket: 340 reason# EndOfStream
2025-11-19T13:59:13.512038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435760024613683:2050] [node 3] ICP30 unregister session Session# [6:7574435760024613777:2050] VirtualId# [6:7574435760024613771:0]
2025-11-19T13:59:13.512045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435760024613777:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.512089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:13.512112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:13.529039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:13.529132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435760024613764:2060] [node 4] ICS01 socket: 320 reason# EndOfStream
2025-11-19T13:59:13.529143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435760024613699:2060] [node 4] ICP30 unregister session Session# [6:7574435760024613764:2060] VirtualId# [6:7574435760024613762:0]
2025-11-19T13:59:13.529149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435760024613764:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:13.529213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:13.529239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:13.608129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435760480495422:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:13.608142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435760480495423:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:13.652146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435759143309016:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:13.652160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435759143309017:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:13.696248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435759503973129:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:13.696260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435759503973130:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:13.736050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435759776430912:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:13.736061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435759776430913:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:13.736065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435759776430914:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:13.770323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435758789807299:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:13.770343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435758789807300:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:13.770348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435758789807301:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:13.799988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435759440904383:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:13.799999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435759440904384:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:13.800003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435759440904385:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:13.847640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435759503973184:2097] [node 5] ICP01 ready to work
2025-11-19T13:59:13.847735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435759503973186:2099] [node 6] ICP01 ready to work
2025-11-19T13:59:13.848063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435759776430962:2094] [node 6] ICP01 ready to work
2025-11-19T13:59:13.848172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435759776430963:2096] [node 5] ICP01 ready to work
2025-11-19T13:59:13.848352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435758789807335:2079] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.853669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435760480495487:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:13.855205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435760480495485:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35149
2025-11-19T13:59:13.864218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435758789807364:2087] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.864227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435759440904442:2065] [node 4] ICP01 ready to work
2025-11-19T13:59:13.864324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435759440904443:2067] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.867778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435759776430962:2094] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19641932ns) {Configure:104(19641315ns) {ProcessConfigured:135(16724171ns) {StartInitialHandshake:157(16723632ns) {DropHandshakes:469(564ns) {DropIncomingHandshake:441(82ns) DropOutgoingHandshake:456(44ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:476(1721ns) {SwitchToState:178(829ns)}}}}}
2025-11-19T13:59:13.867955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435759776430978:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47897
2025-11-19T13:59:13.867967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435759776430979:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47897
2025-11-19T13:59:13.867977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435759776430980:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47897
2025-11-19T13:59:13.866397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435759503973184:2097] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(18611022ns) {Configure:104(18609937ns) {ProcessConfigured:135(16148379ns) {StartInitialHandshake:157(16147734ns) {DropHandshakes:469(501ns) {DropIncomingHandshake:441(111ns) DropOutgoingHandshake:456(77ns)} GenerateSessionVirtualId:988(273ns) PrepareNewSessionHandshake:476(600ns) {SwitchToState:178(310ns)}}}}}
2025-11-19T13:59:13.866512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435759503973197:2104] [node 4] ICP01 ready to work
2025-11-19T13:59:13.867647Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435759503973186:2099] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19778848ns) {Configure:104(19777009ns) {ProcessConfigured:135(17323301ns) {StartInitialHandshake:157(17322479ns) {DropHandshakes:469(312ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(120ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(778ns) {SwitchToState:178(386ns)}}}}}
2025-11-19T13:59:13.867742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435759503973208:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38687
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.875284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435760480495497:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.872445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435759440904384:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24318441ns) {Configure:104(24317764ns) {ProcessConfigured:135(23948021ns) {StartInitialHandshake:157(23947478ns) {DropHandshakes:469(347ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:476(1044ns) {SwitchToState:178(554ns)}}}}}
2025-11-19T13:59:13.874799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435759440904461:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50207
2025-11-19T13:59:13.875940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435760480495498:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35149
2025-11-19T13:59:13.876499Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435758789807299:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(28082944ns) {Configure:104(28082197ns) {ProcessConfigured:135(27865474ns) {StartInitialHandshake:157(27865216ns) {DropHandshakes:469(260ns) {DropIncomingHandshake:441(37ns) DropOutgoingHandshake:456(86ns)} GenerateSessionVirtualId:988(150ns) PrepareNewSessionHandshake:476(641ns) {SwitchToState:178(460ns)}}}}}
2025-11-19T13:59:13.879778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435760480495493:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:13.883915Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435760480495485:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.884093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435759143309082:2105] [node 5] ICP01 ready to work
2025-11-19T13:59:13.884187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435759143309083:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:13.885416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435759143309080:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4464
2025-11-19T13:59:13.885499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435759143309081:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4464
2025-11-19T13:59:13.885881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758789807377:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64124
2025-11-19T13:59:13.885939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758789807378:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64124
2025-11-19T13:59:13.885949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758789807379:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64124
2025-11-19T13:59:13.886441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435759143309079:2099] [node 6] ICP01 ready to work
2025-11-19T13:59:13.887601Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435758789807377:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.890596Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435759776430963:2096] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(42370437ns) {Configure:104(42369687ns) {ProcessConfigured:135(39463745ns) {StartInitialHandshake:157(39463422ns) {DropHandshakes:469(657ns) {DropIncomingHandshake:441(42ns) DropOutgoingHandshake:456(464ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(675ns) {SwitchToState:178(528ns)}}}}}
2025-11-19T13:59:13.891602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435759776430985:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47897
2025-11-19T13:59:13.891614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435759776430986:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47897
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.891966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435759503973209:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38687
2025-11-19T13:59:13.892707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435759503973211:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38687
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.895944Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435759776430979:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.895944Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435759503973209:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.896090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435759503973129:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.896221Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435760480495498:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.895917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758789807381:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64124
2025-11-19T13:59:13.896970Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435758789807378:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.898005Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435759776430980:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.898041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435759503973210:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38687
2025-11-19T13:59:13.900699Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759143309080:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.900744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435759143309016:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.904759Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435760480495489:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.904780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435760480495489:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.905923Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435760480495483:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.905928Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435760480495483:2107] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.907964Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435759503973199:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.907975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435759503973199:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.908771Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435759503973199:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:13.908866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435759503973197:2104] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.908877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:13.908902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435759503973197:2104] [node 4] ICP22 created new session: [3:7574435759503973214:2104]
2025-11-19T13:59:13.908908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435759503973214:2104] [node 4] ICS09 handshake done sender: [3:7574435759503973199:2111] self: [3:7574435759503973198:0] peer: [4:7574435759776430996:0] socket: 322 qp: -1
2025-11-19T13:59:13.908916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435759503973214:2104] [node 4] ICS10 traffic start
2025-11-19T13:59:13.908924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:13.909195Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435760480495483:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:13.909279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435760480495422:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.909287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:13.909292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435760480495422:2048] [node 2] ICP22 created new session: [1:7574435760480495506:2048]
2025-11-19T13:59:13.909297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435760480495506:2048] [node 2] ICS09 handshake done sender: [1:7574435760480495483:2107] self: [1:7574435760480495482:0] peer: [2:7574435759143309097:0] socket: 312 qp: -1
2025-11-19T13:59:13.909302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435760480495506:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:13.909320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.911921Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435759503973211:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.911967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435759503973197:2104] [node 4] ICP18 handshake (actor [3:7574435759503973211:2118]) is too old
2025-11-19T13:59:13.907770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435759776430914:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.911792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435759776430997:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47897
2025-11-19T13:59:13.913079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435758789807301:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.913383Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435759503973196:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.913392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435759503973196:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.912130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435759440904462:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50207
2025-11-19T13:59:13.916250Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435759776430985:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.916342Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435759776430980:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.919875Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435759440904462:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.919922Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435759440904461:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.919951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435759440904383:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.919962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435759440904442:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.922707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435759143309098:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4464
2025-11-19T13:59:13.922860Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435759776430977:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.922871Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435759776430977:2108] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.923188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435759776430914:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.923203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:13.923211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435759776430914:2050] [node 3] ICP22 created new session: [4:7574435759776430998:2050]
2025-11-19T13:59:13.923217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435759776430998:2050] [node 3] ICS09 handshake done sender: [4:7574435759776430980:2111] self: [4:7574435759776430996:0] peer: [3:7574435759503973198:0] socket: 327 qp: -1
2025-11-19T13:59:13.923225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435759776430998:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:13.923233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:13.927826Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435760480495500:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.927848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435760480495500:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.927856Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435760480495489:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:13.924829Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435759503973196:2110] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:13.925353Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435759503973208:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.928695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758789807383:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64124
2025-11-19T13:59:13.931999Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435758789807378:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:13.927978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435760480495423:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.928003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:13.928011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435760480495423:2049] [node 3] ICP22 created new session: [1:7574435760480495510:2049]
2025-11-19T13:59:13.928017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435760480495510:2049] [node 3] ICS09 handshake done sender: [1:7574435760480495489:2110] self: [1:7574435760480495488:0] peer: [3:7574435759503973212:0] socket: 332 qp: -1
2025-11-19T13:59:13.928023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435760480495510:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:13.928031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:13.932321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435759776430912:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.932532Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435759776430978:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.933109Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435759776430977:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.936111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435759143309099:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4464
2025-11-19T13:59:13.936281Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435759143309080:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:13.936334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435759503973184:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.936359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:13.936366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435759503973184:2097] [node 5] ICP22 created new session: [3:7574435759503973219:2097]
2025-11-19T13:59:13.936372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435759503973219:2097] [node 5] ICS09 handshake done sender: [3:7574435759503973196:2110] self: [3:7574435759503973189:0] peer: [5:7574435758789807382:0] socket: 321 qp: -1
2025-11-19T13:59:13.936379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435759503973219:2097] [node 5] ICS10 traffic start
2025-11-19T13:59:13.936388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.937476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435759776430962:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.937503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:13.937510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435759776430962:2094] [node 6] ICP22 created new session: [4:7574435759776431004:2094]
2025-11-19T13:59:13.937516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435759776431004:2094] [node 6] ICS09 handshake done sender: [4:7574435759776430977:2108] self: [4:7574435759776430966:0] peer: [6:7574435759440904464:0] socket: 328 qp: -1
2025-11-19T13:59:13.937520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435759776431004:2094] [node 6] ICS10 traffic start
2025-11-19T13:59:13.937528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:13.937655Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435759776430986:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.939220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435758789807301:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.939239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:13.939245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435758789807301:2050] [node 3] ICP22 created new session: [5:7574435758789807384:2050]
2025-11-19T13:59:13.939250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435758789807384:2050] [node 3] ICS09 handshake done sender: [5:7574435758789807378:2116] self: [5:7574435758789807382:0] peer: [3:7574435759503973189:0] socket: 339 qp: -1
2025-11-19T13:59:13.939256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435758789807384:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:13.939264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:13.939409Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435758789807381:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.941266Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435759503973129:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19849) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435759503973209:2116] held: no
2025-11-19T13:59:13.941359Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435759503973210:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.941559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435759503973221:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38687
2025-11-19T13:59:13.938233Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435760480495491:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.938243Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435760480495491:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.939104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435760480495512:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35149
2025-11-19T13:59:13.940422Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435760480495500:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:13.941721Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435760480495491:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.949986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435759776430913:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.950059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435759776431007:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47897
2025-11-19T13:59:13.950372Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759143309081:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.950396Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759143309099:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.950456Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759143309089:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.950459Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435759143309089:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.950856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435759143309016:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.950867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:13.950872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435759143309016:2048] [node 1] ICP22 created new session: [2:7574435759143309101:2048]
2025-11-19T13:59:13.950876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435759143309101:2048] [node 1] ICS09 handshake done sender: [2:7574435759143309080:2110] self: [2:7574435759143309097:0] peer: [1:7574435760480495482:0] socket: 313 qp: -1
2025-11-19T13:59:13.950881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435759143309101:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:13.950887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:13.951336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435759503973130:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.951405Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435759503973209:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:13.948075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435760480495497:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.948102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:13.948111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435760480495497:2113] [node 6] ICP22 created new session: [1:7574435760480495516:2113]
2025-11-19T13:59:13.948118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435760480495516:2113] [node 6] ICS09 handshake done sender: [1:7574435760480495500:2115] self: [1:7574435760480495499:0] peer: [6:7574435759440904463:0] socket: 351 qp: -1
2025-11-19T13:59:13.948125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435760480495516:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:13.948134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:13.948290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435760480495487:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.948296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:13.948300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435760480495487:2109] [node 4] ICP22 created new session: [1:7574435760480495518:2109]
2025-11-19T13:59:13.948306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435760480495518:2109] [node 4] ICS09 handshake done sender: [1:7574435760480495491:2111] self: [1:7574435760480495490:0] peer: [4:7574435759776431001:0] socket: 333 qp: -1
2025-11-19T13:59:13.948311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435760480495518:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:13.948316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.952603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435759503973129:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.952620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:13.952627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435759503973129:2048] [node 1] ICP22 created new session: [3:7574435759503973224:2048]
2025-11-19T13:59:13.952632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435759503973224:2048] [node 1] ICS09 handshake done sender: [3:7574435759503973209:2116] self: [3:7574435759503973212:0] peer: [1:7574435760480495488:0] socket: 334 qp: -1
2025-11-19T13:59:13.952637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435759503973224:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:13.952644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:13.951998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435759143309100:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4464
2025-11-19T13:59:13.952042Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759143309085:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.952045Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435759143309085:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.954076Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435759143309089:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:13.954196Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435759143309085:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.954301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435759776431009:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47897
2025-11-19T13:59:13.954272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435759503973226:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38687
2025-11-19T13:59:13.954448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435759143309083:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.954459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:13.954464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435759143309083:2109] [node 4] ICP22 created new session: [2:7574435759143309108:2109]
2025-11-19T13:59:13.954470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435759143309108:2109] [node 4] ICS09 handshake done sender: [2:7574435759143309089:2114] self: [2:7574435759143309088:0] peer: [4:7574435759776431008:0] socket: 338 qp: -1
2025-11-19T13:59:13.954475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435759143309108:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:13.954492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:13.954603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435759143309017:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.954612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:13.954618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435759143309017:2049] [node 3] ICP22 created new session: [2:7574435759143309110:2049]
2025-11-19T13:59:13.954621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435759143309110:2049] [node 3] ICS09 handshake done sender: [2:7574435759143309085:2112] self: [2:7574435759143309084:0] peer: [3:7574435759503973223:0] socket: 336 qp: -1
2025-11-19T13:59:13.954625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435759143309110:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:13.954629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.954708Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435758789807379:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.954762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435758789807335:2079] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.954903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758789807386:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64124
2025-11-19T13:59:13.954917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435758789807300:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.955046Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435759776430984:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.955055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435759776430984:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.955398Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435759776430986:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:13.957104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435759440904442:2065] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6587) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435759440904461:2115] held: no
2025-11-19T13:59:13.957840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435759776430913:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.957856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:13.957861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435759776430913:2049] [node 2] ICP22 created new session: [4:7574435759776431011:2049]
2025-11-19T13:59:13.957867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435759776431011:2049] [node 2] ICS09 handshake done sender: [4:7574435759776430986:2116] self: [4:7574435759776431008:0] peer: [2:7574435759143309088:0] socket: 342 qp: -1
2025-11-19T13:59:13.957872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435759776431011:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:13.957877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:13.964368Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435759503973130:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28689) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435759503973210:2117] held: no
2025-11-19T13:59:13.964405Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435759503973210:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.964692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758789807390:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64124
2025-11-19T13:59:13.966238Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435758789807335:2079] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6587) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435758789807381:2118] held: no
2025-11-19T13:59:13.966270Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435758789807381:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:13.964006Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435759776430984:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:13.964139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435759776430963:2096] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.964154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:13.964161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435759776430963:2096] [node 5] ICP22 created new session: [4:7574435759776431013:2096]
2025-11-19T13:59:13.964166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435759776431013:2096] [node 5] ICS09 handshake done sender: [4:7574435759776430984:2114] self: [4:7574435759776430967:0] peer: [5:7574435758789807388:0] socket: 343 qp: -1
2025-11-19T13:59:13.964171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435759776431013:2096] [node 5] ICS10 traffic start
2025-11-19T13:59:13.964180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:13.964270Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435759776430985:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:13.964814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435759776430912:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.964823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:13.964828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435759776430912:2048] [node 1] ICP22 created new session: [4:7574435759776431015:2048]
2025-11-19T13:59:13.964832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435759776431015:2048] [node 1] ICS09 handshake done sender: [4:7574435759776430985:2115] self: [4:7574435759776431001:0] peer: [1:7574435760480495490:0] socket: 341 qp: -1
2025-11-19T13:59:13.964842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435759776431015:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:13.964847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:13.968254Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759143309087:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.968265Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435759143309087:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.968749Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759143309100:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.968898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435759143309083:2109] [node 4] ICP18 handshake (actor [2:7574435759143309100:2119]) is too old
2025-11-19T13:59:13.969351Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435760480495512:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.969441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435760480495487:2109] [node 4] ICP18 handshake (actor [1:7574435760480495512:2117]) is too old
2025-11-19T13:59:13.970407Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435759143309087:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:13.970514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435759143309082:2105] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.970532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:13.970538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435759143309082:2105] [node 5] ICP22 created new session: [2:7574435759143309113:2105]
2025-11-19T13:59:13.970543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435759143309113:2105] [node 5] ICS09 handshake done sender: [2:7574435759143309087:2113] self: [2:7574435759143309086:0] peer: [5:7574435758789807389:0] socket: 337 qp: -1
2025-11-19T13:59:13.970548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435759143309113:2105] [node 5] ICS10 traffic start
2025-11-19T13:59:13.970556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:13.979833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435758789807335:2079] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.979855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:13.979862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435758789807335:2079] [node 4] ICP22 created new session: [5:7574435758789807391:2079]
2025-11-19T13:59:13.979868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435758789807391:2079] [node 4] ICS09 handshake done sender: [5:7574435758789807381:2118] self: [5:7574435758789807388:0] peer: [4:7574435759776430967:0] socket: 347 qp: -1
2025-11-19T13:59:13.979876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435758789807391:2079] [node 4] ICS10 traffic start
2025-11-19T13:59:13.979884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:13.980960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435759503973130:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.980978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:13.980982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435759503973130:2049] [node 2] ICP22 created new session: [3:7574435759503973227:2049]
2025-11-19T13:59:13.980987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435759503973227:2049] [node 2] ICS09 handshake done sender: [3:7574435759503973210:2117] self: [3:7574435759503973223:0] peer: [2:7574435759143309084:0] socket: 345 qp: -1
2025-11-19T13:59:13.980991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435759503973227:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:13.980997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.981388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435758789807300:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28689) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435758789807379:2117] held: no
2025-11-19T13:59:13.981440Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435758789807386:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.981579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758789807393:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64124
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.981697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435760480495520:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35149
2025-11-19T13:59:13.981862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435758789807299:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.981877Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435758789807379:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:13.983184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435758789807300:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.983201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:13.983206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435758789807300:2049] [node 2] ICP22 created new session: [5:7574435758789807396:2049]
2025-11-19T13:59:13.983211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435758789807396:2049] [node 2] ICS09 handshake done sender: [5:7574435758789807379:2117] self: [5:7574435758789807389:0] peer: [2:7574435759143309086:0] socket: 340 qp: -1
2025-11-19T13:59:13.983216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435758789807396:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:13.983222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:13.983512Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435760480495502:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.983522Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435760480495502:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.984192Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435760480495502:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:13.984274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435760480495493:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.984288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:13.984293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435760480495493:2112] [node 5] ICP22 created new session: [1:7574435760480495522:2112]
2025-11-19T13:59:13.984297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435760480495522:2112] [node 5] ICS09 handshake done sender: [1:7574435760480495502:2116] self: [1:7574435760480495501:0] peer: [5:7574435758789807395:0] socket: 362 qp: -1
2025-11-19T13:59:13.984303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435760480495522:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:13.984309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.984697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435758789807398:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64124
2025-11-19T13:59:13.984796Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435760480495520:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.984821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435760480495493:2112] [node 5] ICP18 handshake (actor [1:7574435760480495520:2118]) is too old
2025-11-19T13:59:13.985923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435758789807299:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19849) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435758789807386:2120] held: no
2025-11-19T13:59:13.985950Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435758789807386:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.986029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435758789807299:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.986040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:13.986044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435758789807299:2048] [node 1] ICP22 created new session: [5:7574435758789807399:2048]
2025-11-19T13:59:13.986049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435758789807399:2048] [node 1] ICS09 handshake done sender: [5:7574435758789807386:2120] self: [5:7574435758789807395:0] peer: [1:7574435760480495501:0] socket: 316 qp: -1
2025-11-19T13:59:13.986053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435758789807399:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:13.986059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.986439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435759503973229:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38687
2025-11-19T13:59:13.986051Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435759440904383:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19849) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435759440904462:2116] held: no
2025-11-19T13:59:13.986294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435759440904465:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50207
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.986733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435759143309115:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4464
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.986898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435759440904467:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50207
2025-11-19T13:59:13.986943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435759440904468:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50207
2025-11-19T13:59:13.986974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435759440904469:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50207
2025-11-19T13:59:13.987016Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435759440904461:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:13.991856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435759440904466:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50207
2025-11-19T13:59:13.991980Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435759440904468:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.991991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435759440904442:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.992003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:13.992009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435759440904442:2065] [node 4] ICP22 created new session: [6:7574435759440904472:2065]
2025-11-19T13:59:13.992015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435759440904472:2065] [node 4] ICS09 handshake done sender: [6:7574435759440904461:2115] self: [6:7574435759440904464:0] peer: [4:7574435759776430966:0] socket: 330 qp: -1
2025-11-19T13:59:13.992021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435759440904472:2065] [node 4] ICS10 traffic start
2025-11-19T13:59:13.992041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:13.992159Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435759440904467:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.992180Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435759440904469:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.994571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435759440904384:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.994586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435759440904385:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.994597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435759440904443:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:13.995045Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435759503973207:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.995056Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435759503973207:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.995847Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435759440904462:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:13.995863Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435759503973207:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:13.995961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435759440904383:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:13.995974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:13.995979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435759440904383:2048] [node 1] ICP22 created new session: [6:7574435759440904478:2048]
2025-11-19T13:59:13.995985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435759440904478:2048] [node 1] ICS09 handshake done sender: [6:7574435759440904462:2116] self: [6:7574435759440904463:0] peer: [1:7574435760480495499:0] socket: 353 qp: -1
2025-11-19T13:59:13.995990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435759440904478:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:13.995996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:13.996094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435759440904477:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50207
2025-11-19T13:59:13.996245Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435759503973229:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.996262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435759503973186:2099] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.996272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:13.996277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435759503973186:2099] [node 6] ICP22 created new session: [3:7574435759503973231:2099]
2025-11-19T13:59:13.996282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435759503973231:2099] [node 6] ICS09 handshake done sender: [3:7574435759503973207:2114] self: [3:7574435759503973190:0] peer: [6:7574435759440904475:0] socket: 324 qp: -1
2025-11-19T13:59:13.996290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435759503973231:2099] [node 6] ICS10 traffic start
2025-11-19T13:59:13.996298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:13.995974Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759143309096:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:13.995985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435759143309096:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:13.996672Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435759143309096:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:13.996743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435759143309079:2099] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:13.996756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:13.996762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435759143309079:2099] [node 6] ICP22 created new session: [2:7574435759143309117:2099]
2025-11-19T13:59:13.996767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435759143309117:2099] [node 6] ICS09 handshake done sender: [2:7574435759143309096:2116] self: [2:7574435759143309095:0] peer: [6:7574435759440904474:0] socket: 367 qp: -1
2025-11-19T13:59:13.996773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435759143309117:2099] [node 6] ICS10 traffic start
2025-11-19T13:59:13.996780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.000139Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435759440904467:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:14.000228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435759440904480:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50207
2025-11-19T13:59:14.000350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435759503973186:2099] [node 6] ICP18 handshake (actor [3:7574435759503973229:2121]) is too old
2025-11-19T13:59:14.000530Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435759143309115:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.000556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435759143309079:2099] [node 6] ICP18 handshake (actor [2:7574435759143309115:2120]) is too old
2025-11-19T13:59:14.000832Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435759440904468:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:14.001707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435759440904384:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.001724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:14.001729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435759440904384:2049] [node 2] ICP22 created new session: [6:7574435759440904481:2049]
2025-11-19T13:59:14.001734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435759440904481:2049] [node 2] ICS09 handshake done sender: [6:7574435759440904468:2120] self: [6:7574435759440904474:0] peer: [2:7574435759143309095:0] socket: 364 qp: -1
2025-11-19T13:59:14.001739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435759440904481:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:14.001795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:14.001854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435759440904384:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:14.002148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435759440904385:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.002161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:14.002165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435759440904385:2050] [node 3] ICP22 created new session: [6:7574435759440904483:2050]
2025-11-19T13:59:14.002169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435759440904483:2050] [node 3] ICS09 handshake done sender: [6:7574435759440904467:2119] self: [6:7574435759440904475:0] peer: [3:7574435759503973190:0] socket: 357 qp: -1
2025-11-19T13:59:14.002173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435759440904483:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:14.002178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:14.003914Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435758789807367:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.003927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435758789807367:2107] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.008033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435763735871781:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50207
2025-11-19T13:59:14.011879Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435758789807367:2107] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:14.012050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435758789807364:2087] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.012074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:14.012081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435758789807364:2087] [node 6] ICP22 created new session: [5:7574435763084774698:2087]
2025-11-19T13:59:14.012087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435763084774698:2087] [node 6] ICS09 handshake done sender: [5:7574435758789807367:2107] self: [5:7574435758789807366:0] peer: [6:7574435759440904476:0] socket: 369 qp: -1
2025-11-19T13:59:14.012093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435763084774698:2087] [node 6] ICS10 traffic start
2025-11-19T13:59:14.012102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:14.014685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435759440904443:2067] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31994) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435759440904469:2121] held: no
2025-11-19T13:59:14.014733Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435759440904469:2121] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:14.014804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435759440904443:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.014815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:14.014820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435759440904443:2067] [node 5] ICP22 created new session: [6:7574435763735871782:2067]
2025-11-19T13:59:14.014826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435763735871782:2067] [node 5] ICS09 handshake done sender: [6:7574435759440904469:2121] self: [6:7574435759440904476:0] peer: [5:7574435758789807366:0] socket: 370 qp: -1
2025-11-19T13:59:14.014832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435763735871782:2067] [node 5] ICS10 traffic start
2025-11-19T13:59:14.014840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:14.033047Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:14.033130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435759503973224:2048] [node 1] ICS01 socket: 334 reason# EndOfStream
2025-11-19T13:59:14.033138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435759503973129:2048] [node 1] ICP30 unregister session Session# [3:7574435759503973224:2048] VirtualId# [3:7574435759503973212:0]
2025-11-19T13:59:14.033146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435759503973224:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.033178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:14.033196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:14.035880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:14.035957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435759143309101:2048] [node 1] ICS01 socket: 313 reason# EndOfStream
2025-11-19T13:59:14.035964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435759143309016:2048] [node 1] ICP30 unregister session Session# [2:7574435759143309101:2048] VirtualId# [2:7574435759143309097:0]
2025-11-19T13:59:14.035971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435759143309101:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.036025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:14.036047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:14.041577Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:14.041662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435759776431015:2048] [node 1] ICS01 socket: 341 reason# EndOfStream
2025-11-19T13:59:14.041669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435759776430912:2048] [node 1] ICP30 unregister session Session# [4:7574435759776431015:2048] VirtualId# [4:7574435759776431001:0]
2025-11-19T13:59:14.041676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435759776431015:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.041727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:14.041749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:14.041779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:14.041835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435759440904478:2048] [node 1] ICS01 socket: 353 reason# EndOfStream
2025-11-19T13:59:14.041840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435759440904383:2048] [node 1] ICP30 unregister session Session# [6:7574435759440904478:2048] VirtualId# [6:7574435759440904463:0]
2025-11-19T13:59:14.041844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435759440904478:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.041866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:14.041883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:14.043026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:14.043100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435758789807399:2048] [node 1] ICS01 socket: 316 reason# EndOfStream
2025-11-19T13:59:14.043107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435758789807299:2048] [node 1] ICP30 unregister session Session# [5:7574435758789807399:2048] VirtualId# [5:7574435758789807395:0]
2025-11-19T13:59:14.043113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435758789807399:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.043157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:14.043177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:14.045910Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:14.045980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435759503973227:2049] [node 2] ICS01 socket: 345 reason# EndOfStream
2025-11-19T13:59:14.045987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435759503973130:2049] [node 2] ICP30 unregister session Session# [3:7574435759503973227:2049] VirtualId# [3:7574435759503973223:0]
2025-11-19T13:59:14.045991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435759503973227:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.046018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:14.046034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:14.059914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:14.060018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435759440904481:2049] [node 2] ICS01 socket: 364 reason# EndOfStream
2025-11-19T13:59:14.060028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435759440904384:2049] [node 2] ICP30 unregister session Session# [6:7574435759440904481:2049] VirtualId# [6:7574435759440904474:0]
2025-11-19T13:59:14.060035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435759440904481:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.060091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:14.060116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:14.060156Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:14.060219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435759776431011:2049] [node 2] ICS01 socket: 342 reason# EndOfStream
2025-11-19T13:59:14.060224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435759776430913:2049] [node 2] ICP30 unregister session Session# [4:7574435759776431011:2049] VirtualId# [4:7574435759776431008:0]
2025-11-19T13:59:14.060229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435759776431011:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.060255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:14.060273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:14.060227Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:14.060316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435758789807396:2049] [node 2] ICS01 socket: 340 reason# EndOfStream
2025-11-19T13:59:14.060323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435758789807300:2049] [node 2] ICP30 unregister session Session# [5:7574435758789807396:2049] VirtualId# [5:7574435758789807389:0]
2025-11-19T13:59:14.060329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435758789807396:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.060381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:14.060404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:14.088255Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:14.088318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435759440904483:2050] [node 3] ICS01 socket: 357 reason# EndOfStream
2025-11-19T13:59:14.088322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435759440904385:2050] [node 3] ICP30 unregister session Session# [6:7574435759440904483:2050] VirtualId# [6:7574435759440904475:0]
2025-11-19T13:59:14.088326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435759440904483:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.088352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:14.088372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:14.088026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:14.088126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435758789807384:2050] [node 3] ICS01 socket: 339 reason# EndOfStream
2025-11-19T13:59:14.088135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435758789807301:2050] [node 3] ICP30 unregister session Session# [5:7574435758789807384:2050] VirtualId# [5:7574435758789807382:0]
2025-11-19T13:59:14.088142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435758789807384:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.088194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:14.088218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:14.109109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:14.109214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435759440904472:2065] [node 4] ICS01 socket: 330 reason# EndOfStream
2025-11-19T13:59:14.109226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435759440904442:2065] [node 4] ICP30 unregister session Session# [6:7574435759440904472:2065] VirtualId# [6:7574435759440904464:0]
2025-11-19T13:59:14.109234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435759440904472:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.109305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:14.109330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:14.212139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435763712370413:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:14.212154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435763712370414:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:14.255910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435766580348335:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:14.255925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435766580348336:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:14.316236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435765131597278:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:14.316249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435765131597279:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:14.407930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435766776465212:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:14.407944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435766776465213:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:14.407948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435766776465214:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:14.467671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435764665081427:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:14.467685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435764665081428:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:14.467690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435764665081429:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:14.532053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435763235570871:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:14.532077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435763235570872:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:14.532082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435763235570873:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:14.580227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435764665081451:2062] [node 4] ICP01 ready to work
2025-11-19T13:59:14.580237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435763712370474:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:14.580187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435763235570895:2064] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.583321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435763712370480:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14597
2025-11-19T13:59:14.583341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435763712370481:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14597
2025-11-19T13:59:14.584159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435766580348402:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:14.584945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435766580348406:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:14.585019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435766580348407:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:14.584998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435763235570915:2082] [node 5] ICP01 ready to work
2025-11-19T13:59:14.585401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435764665081465:2060] [node 6] ICP01 ready to work
2025-11-19T13:59:14.585009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435763712370487:2113] [node 5] ICP01 ready to work
2025-11-19T13:59:14.587920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435766776465259:2091] [node 5] ICP01 ready to work
2025-11-19T13:59:14.590454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435763712370486:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.602811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435763235570945:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:828
2025-11-19T13:59:14.603834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435766580348402:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19615297ns) {Configure:104(19614353ns) {ProcessConfigured:135(19354243ns) {StartInitialHandshake:157(19353507ns) {DropHandshakes:469(519ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:476(1577ns) {SwitchToState:178(1080ns)}}}}}
2025-11-19T13:59:14.603967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435766580348407:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(18616079ns) {Configure:104(18615620ns) {ProcessConfigured:135(18420858ns) {StartInitialHandshake:157(18420554ns) {DropHandshakes:469(287ns) {DropIncomingHandshake:441(22ns) DropOutgoingHandshake:456(21ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(778ns) {SwitchToState:178(513ns)}}}}}
2025-11-19T13:59:14.606864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435764665081465:2060] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21400880ns) {Configure:104(21400390ns) {ProcessConfigured:135(21146476ns) {StartInitialHandshake:157(21146163ns) {DropHandshakes:469(382ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:476(1161ns) {SwitchToState:178(1018ns)}}}}}
2025-11-19T13:59:14.607829Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435763712370480:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.608134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435763235570915:2082] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(22485233ns) {Configure:104(22484191ns) {ProcessConfigured:135(22281857ns) {StartInitialHandshake:157(22281589ns) {DropHandshakes:469(358ns) {DropIncomingHandshake:441(14ns) DropOutgoingHandshake:456(24ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:476(684ns) {SwitchToState:178(265ns)}}}}}
2025-11-19T13:59:14.608268Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435763235570873:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(23229823ns) {Configure:104(23229504ns) {ProcessConfigured:135(22960576ns) {StartInitialHandshake:157(22960382ns) {DropHandshakes:469(423ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(37ns)} GenerateSessionVirtualId:988(85ns) PrepareNewSessionHandshake:476(454ns) {SwitchToState:178(184ns)}}}}}
2025-11-19T13:59:14.608526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435763235570948:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:828
2025-11-19T13:59:14.608652Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435763712370481:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.610483Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435763235570948:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.610525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435763235570871:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.610671Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435763712370488:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.610677Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435763712370488:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:14.611232Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435763712370488:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:14.616257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435766776465260:2089] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.618244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435765131597346:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:14.618353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435765131597347:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:14.619221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435765131597342:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9502
2025-11-19T13:59:14.619279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435765131597343:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9502
2025-11-19T13:59:14.619317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435765131597344:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9502
2025-11-19T13:59:14.619328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435765131597345:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9502
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.619934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435764665081505:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11592
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.620555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435766580348414:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56307
2025-11-19T13:59:14.620562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435766580348415:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56307
2025-11-19T13:59:14.620593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435766580348416:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56307
2025-11-19T13:59:14.620600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435766580348417:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56307
2025-11-19T13:59:14.619979Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435763712370487:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(29369586ns) {Configure:104(29368912ns) {ProcessConfigured:135(29080822ns) {StartInitialHandshake:157(29080235ns) {DropHandshakes:469(519ns) {DropIncomingHandshake:441(94ns) DropOutgoingHandshake:456(184ns)} GenerateSessionVirtualId:988(241ns) PrepareNewSessionHandshake:476(1196ns) {SwitchToState:178(701ns)}}}}}
2025-11-19T13:59:14.620000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435763712370474:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.620016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:14.620022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435763712370474:2107] [node 6] ICP22 created new session: [1:7574435763712370497:2107]
2025-11-19T13:59:14.620027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435763712370497:2107] [node 6] ICS09 handshake done sender: [1:7574435763712370488:2114] self: [1:7574435763712370479:0] peer: [6:7574435763235570949:0] socket: 335 qp: -1
2025-11-19T13:59:14.620034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435763712370497:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:14.620052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.621183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435763712370499:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14597
2025-11-19T13:59:14.622918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435766580348416:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.622959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435766580348335:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.623121Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435763712370476:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.623128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435763712370476:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:14.623831Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435763235570945:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.624532Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435763235570871:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1337) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435763235570948:2114] held: no
2025-11-19T13:59:14.624570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435763235570872:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.623863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435765131597278:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(31961737ns) {Configure:104(31960910ns) {ProcessConfigured:135(31584079ns) {StartInitialHandshake:157(31583410ns) {DropHandshakes:469(529ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:476(1068ns) {SwitchToState:178(221ns)}}}}}
2025-11-19T13:59:14.623954Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435765131597343:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.623976Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435765131597342:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.624966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435765131597279:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.624781Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435766580348414:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.624815Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435766580348409:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.624818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435766580348409:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:14.628291Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435763712370476:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:14.628441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435763712370413:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.628459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:14.628467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435763712370413:2048] [node 2] ICP22 created new session: [1:7574435763712370502:2048]
2025-11-19T13:59:14.628472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435763712370502:2048] [node 2] ICS09 handshake done sender: [1:7574435763712370476:2108] self: [1:7574435763712370475:0] peer: [2:7574435766580348418:0] socket: 328 qp: -1
2025-11-19T13:59:14.628477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435763712370502:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:14.628485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.631337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435766776465289:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51718
2025-11-19T13:59:14.631392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435766776465290:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51718
2025-11-19T13:59:14.631425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435766776465291:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51718
2025-11-19T13:59:14.632126Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435763712370478:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.632137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435763712370478:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:14.632819Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435763712370478:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:14.632897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435763712370414:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.632911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:14.632917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435763712370414:2049] [node 3] ICP22 created new session: [1:7574435763712370506:2049]
2025-11-19T13:59:14.632923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435763712370506:2049] [node 3] ICS09 handshake done sender: [1:7574435763712370478:2109] self: [1:7574435763712370477:0] peer: [3:7574435765131597361:0] socket: 329 qp: -1
2025-11-19T13:59:14.632929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435763712370506:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:14.632936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:14.635688Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435766776465291:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.635736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435766776465212:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.631872Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435765131597344:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.631908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435765131597278:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.637158Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435766776465290:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.640265Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435766580348335:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1337) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435766580348416:2117] held: no
2025-11-19T13:59:14.640327Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435766580348415:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.643827Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435763712370491:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.643859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435763712370491:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:14.643890Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435766580348396:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.643911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435766580348396:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:14.647915Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435766776465259:2091] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(59917463ns) {Configure:104(59916861ns) {ProcessConfigured:135(59602161ns) {StartInitialHandshake:157(59601653ns) {DropHandshakes:469(407ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:476(1067ns) {SwitchToState:178(588ns)}}}}}
2025-11-19T13:59:14.649828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435763712370413:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(21354443ns) {HandleHandshakeStatus:329(21353274ns) {DropIncomingHandshake:441(3523ns) DropOutgoingHandshake:456(231ns) SwitchToState:178(178ns) ProcessPendingSessionEvents:495(21271686ns) {ForwardSessionEventToSession:612(21258342ns) ForwardSessionEventToSession:612(2124ns) ForwardSessionEventToSession:612(647ns) ForwardSessionEventToSession:612(2274ns) ForwardSessionEventToSession:612(708ns) ForwardSessionEventToSession:612(707ns) ForwardSessionEventToSession:612(747ns) ForwardSessionEventToSession:612(856ns) ForwardSessionEventToSession:612(775ns) ForwardSessionEventToSession:612(1318ns)}}}
2025-11-19T13:59:14.650178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435766580348419:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56307
2025-11-19T13:59:14.650082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435765131597360:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.656861Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435766580348416:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.657439Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435766776465289:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.657520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435766776465295:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51718
2025-11-19T13:59:14.657997Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435766580348409:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:14.658083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435766580348335:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.658099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:14.658106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435766580348335:2048] [node 1] ICP22 created new session: [2:7574435766580348424:2048]
2025-11-19T13:59:14.658112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435766580348424:2048] [node 1] ICS09 handshake done sender: [2:7574435766580348416:2117] self: [2:7574435766580348418:0] peer: [1:7574435763712370475:0] socket: 350 qp: -1
2025-11-19T13:59:14.658118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435766580348424:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:14.658126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:14.658240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435766580348406:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.658250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:14.658254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435766580348406:2110] [node 6] ICP22 created new session: [2:7574435766580348426:2110]
2025-11-19T13:59:14.658257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435766580348426:2110] [node 6] ICS09 handshake done sender: [2:7574435766580348409:2112] self: [2:7574435766580348408:0] peer: [6:7574435763235570952:0] socket: 330 qp: -1
2025-11-19T13:59:14.658260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435766580348426:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:14.658266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.657071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435763235570954:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:828
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.660094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435763235570957:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:828
2025-11-19T13:59:14.659996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435765131597364:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9502
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.660419Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435766580348417:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.661480Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435766776465295:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.661663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435766776465213:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.661680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435766776465297:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51718
2025-11-19T13:59:14.660188Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435765131597344:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:14.660289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435765131597278:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.660303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:14.660309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435765131597278:2048] [node 1] ICP22 created new session: [3:7574435765131597365:2048]
2025-11-19T13:59:14.660314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435765131597365:2048] [node 1] ICS09 handshake done sender: [3:7574435765131597344:2112] self: [3:7574435765131597361:0] peer: [1:7574435763712370477:0] socket: 343 qp: -1
2025-11-19T13:59:14.660321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435765131597365:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:14.660339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:14.661904Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435766580348396:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:14.661905Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435763712370491:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:14.662137Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435766776465291:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:14.662064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435766580348336:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.662080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:14.662086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435766580348336:2049] [node 3] ICP22 created new session: [2:7574435766580348431:2049]
2025-11-19T13:59:14.662092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435766580348431:2049] [node 3] ICS09 handshake done sender: [2:7574435766580348396:2107] self: [2:7574435766580348395:0] peer: [3:7574435765131597359:0] socket: 318 qp: -1
2025-11-19T13:59:14.662098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435766580348431:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:14.662106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:14.662226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435766776465212:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.662238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:14.662243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435766776465212:2048] [node 1] ICP22 created new session: [4:7574435766776465299:2048]
2025-11-19T13:59:14.662248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435766776465299:2048] [node 1] ICS09 handshake done sender: [4:7574435766776465291:2116] self: [4:7574435766776465292:0] peer: [1:7574435763712370490:0] socket: 357 qp: -1
2025-11-19T13:59:14.662254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435766776465299:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:14.662260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:14.662648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435763712370486:2112] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.662661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:14.662667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435763712370486:2112] [node 4] ICP22 created new session: [1:7574435763712370509:2112]
2025-11-19T13:59:14.662671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435763712370509:2112] [node 4] ICS09 handshake done sender: [1:7574435763712370491:2115] self: [1:7574435763712370490:0] peer: [4:7574435766776465292:0] socket: 353 qp: -1
2025-11-19T13:59:14.662676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435763712370509:2112] [node 4] ICS10 traffic start
2025-11-19T13:59:14.662682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:14.664800Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435766580348412:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.664810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435766580348412:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:14.664613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435763235570955:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:828
2025-11-19T13:59:14.664684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435763235570956:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:828
2025-11-19T13:59:14.664731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435763235570872:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435763235570945:2111] held: no
2025-11-19T13:59:14.664750Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435763235570948:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:14.667937Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435763235570945:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:14.668434Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435765131597345:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.668488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435763712370486:2112] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:14.668561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435763235570871:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.668579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:14.668587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435763235570871:2048] [node 1] ICP22 created new session: [6:7574435763235570958:2048]
2025-11-19T13:59:14.668593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435763235570958:2048] [node 1] ICS09 handshake done sender: [6:7574435763235570948:2114] self: [6:7574435763235570949:0] peer: [1:7574435763712370479:0] socket: 336 qp: -1
2025-11-19T13:59:14.668600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435763235570958:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:14.668607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:14.669032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435763235570872:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.669062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:14.669067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435763235570872:2049] [node 2] ICP22 created new session: [6:7574435763235570960:2049]
2025-11-19T13:59:14.669072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435763235570960:2049] [node 2] ICS09 handshake done sender: [6:7574435763235570945:2111] self: [6:7574435763235570952:0] peer: [2:7574435766580348408:0] socket: 334 qp: -1
2025-11-19T13:59:14.669077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435763235570960:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:14.669093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.669078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435765131597279:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435765131597342:2110] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.672401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435766776465302:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51718
2025-11-19T13:59:14.672571Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435764665081505:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.672605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435764665081429:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.672794Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435765131597353:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.672802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435765131597353:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:14.672073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435766580348433:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56307
2025-11-19T13:59:14.672729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435766580348433:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.676353Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435766580348412:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:14.677319Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435763712370496:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.677333Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435763712370496:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.678240Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435763712370496:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:14.678374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435766580348402:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.678390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:14.678394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435766580348402:2109] [node 4] ICP22 created new session: [2:7574435766580348436:2109]
2025-11-19T13:59:14.678400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435766580348436:2109] [node 4] ICS09 handshake done sender: [2:7574435766580348412:2113] self: [2:7574435766580348405:0] peer: [4:7574435766776465298:0] socket: 360 qp: -1
2025-11-19T13:59:14.678404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435766580348436:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:14.678410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:14.678680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435763712370487:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.678693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:14.678697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435763712370487:2113] [node 5] ICP22 created new session: [1:7574435763712370512:2113]
2025-11-19T13:59:14.678702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435763712370512:2113] [node 5] ICS09 handshake done sender: [1:7574435763712370496:2116] self: [1:7574435763712370489:0] peer: [5:7574435764665081511:0] socket: 327 qp: -1
2025-11-19T13:59:14.678706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435763712370512:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:14.678713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:14.674433Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435763235570955:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.674464Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435763235570956:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.674643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435763235570895:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.674653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435763235570873:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.674732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435765131597367:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9502
2025-11-19T13:59:14.675574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435764665081508:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11592
2025-11-19T13:59:14.675614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435764665081509:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11592
2025-11-19T13:59:14.675641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435764665081510:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11592
2025-11-19T13:59:14.676100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435764665081507:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11592
2025-11-19T13:59:14.676139Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435764665081508:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.676158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435764665081427:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.678084Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435764665081507:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.678170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435764665081512:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11592
2025-11-19T13:59:14.678948Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435764665081509:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.678964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435764665081428:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.679111Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435764665081508:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:14.679207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435764665081427:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.679217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:14.679221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435764665081427:2048] [node 1] ICP22 created new session: [5:7574435764665081515:2048]
2025-11-19T13:59:14.679225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435764665081515:2048] [node 1] ICS09 handshake done sender: [5:7574435764665081508:2117] self: [5:7574435764665081511:0] peer: [1:7574435763712370489:0] socket: 369 qp: -1
2025-11-19T13:59:14.679230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435764665081515:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:14.679244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:14.679832Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435766580348413:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.679842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435766580348413:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:14.679659Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435765131597349:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.679668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435765131597349:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:14.680319Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435765131597353:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:14.680992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435765131597347:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.681004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:14.681009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435765131597347:2109] [node 5] ICP22 created new session: [3:7574435765131597371:2109]
2025-11-19T13:59:14.681015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435765131597371:2109] [node 5] ICS09 handshake done sender: [3:7574435765131597353:2116] self: [3:7574435765131597352:0] peer: [5:7574435764665081506:0] socket: 347 qp: -1
2025-11-19T13:59:14.681037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435765131597371:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:14.681043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:14.681795Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435765131597342:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.684182Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435766776465279:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.684200Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435766776465279:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:14.684647Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435766776465302:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.685270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435766776465213:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435766776465295:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.686075Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435764665081505:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:14.687165Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435766580348413:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:14.687293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435766580348407:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.687308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:14.687312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435766580348407:2111] [node 5] ICP22 created new session: [2:7574435766580348439:2111]
2025-11-19T13:59:14.687317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435766580348439:2111] [node 5] ICS09 handshake done sender: [2:7574435766580348413:2114] self: [2:7574435766580348410:0] peer: [5:7574435764665081514:0] socket: 332 qp: -1
2025-11-19T13:59:14.687322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435766580348439:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:14.687328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:14.685632Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435765131597349:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:14.685957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435765131597374:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9502
2025-11-19T13:59:14.686173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435765131597346:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.686182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:14.686187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435765131597346:2108] [node 6] ICP22 created new session: [3:7574435765131597375:2108]
2025-11-19T13:59:14.686193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435765131597375:2108] [node 6] ICS09 handshake done sender: [3:7574435765131597349:2114] self: [3:7574435765131597348:0] peer: [6:7574435763235570963:0] socket: 345 qp: -1
2025-11-19T13:59:14.686198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435765131597375:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:14.686206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:14.687883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435765131597279:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.687893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:14.687899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435765131597279:2049] [node 2] ICP22 created new session: [3:7574435765131597377:2049]
2025-11-19T13:59:14.687905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435765131597377:2049] [node 2] ICS09 handshake done sender: [3:7574435765131597342:2110] self: [3:7574435765131597359:0] peer: [2:7574435766580348395:0] socket: 331 qp: -1
2025-11-19T13:59:14.687910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435765131597377:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:14.687922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.688427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435766776465214:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.687717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435764665081429:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7717) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435764665081505:2115] held: no
2025-11-19T13:59:14.688519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435766776465304:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51718
2025-11-19T13:59:14.687738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435764665081429:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.687758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:14.687762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435764665081429:2050] [node 3] ICP22 created new session: [5:7574435764665081518:2050]
2025-11-19T13:59:14.687767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435764665081518:2050] [node 3] ICS09 handshake done sender: [5:7574435764665081505:2115] self: [5:7574435764665081506:0] peer: [3:7574435765131597352:0] socket: 348 qp: -1
2025-11-19T13:59:14.687772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435764665081518:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:14.687789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:14.688323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435764665081428:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435764665081509:2118] held: no
2025-11-19T13:59:14.688727Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435765131597374:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.688743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435765131597346:2108] [node 6] ICP18 handshake (actor [3:7574435765131597374:2121]) is too old
2025-11-19T13:59:14.688764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435764665081520:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11592
2025-11-19T13:59:14.688791Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435766776465279:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:14.688957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435766776465260:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.688972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:14.688976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435766776465260:2089] [node 6] ICP22 created new session: [4:7574435766776465307:2089]
2025-11-19T13:59:14.688981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435766776465307:2089] [node 6] ICS09 handshake done sender: [4:7574435766776465279:2110] self: [4:7574435766776465278:0] peer: [6:7574435763235570962:0] socket: 339 qp: -1
2025-11-19T13:59:14.688985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435766776465307:2089] [node 6] ICS10 traffic start
2025-11-19T13:59:14.688991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.681917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435763235570964:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:828
2025-11-19T13:59:14.681957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435763235570965:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:828
2025-11-19T13:59:14.688340Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435763235570964:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.688431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435763235570915:2082] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.688681Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435763235570956:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:14.688842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435763235570873:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.688852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:14.688857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435763235570873:2050] [node 3] ICP22 created new session: [6:7574435763235570968:2050]
2025-11-19T13:59:14.688862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435763235570968:2050] [node 3] ICS09 handshake done sender: [6:7574435763235570956:2119] self: [6:7574435763235570963:0] peer: [3:7574435765131597348:0] socket: 362 qp: -1
2025-11-19T13:59:14.688866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435763235570968:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:14.688877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:14.689587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435763235570970:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:828
2025-11-19T13:59:14.689790Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435763235570955:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:14.692065Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435766776465295:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:14.689824Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435765131597363:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.689833Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435765131597363:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:14.699880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435763235570895:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.699915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:14.699925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435763235570895:2064] [node 4] ICP22 created new session: [6:7574435763235570971:2064]
2025-11-19T13:59:14.699935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435763235570971:2064] [node 4] ICS09 handshake done sender: [6:7574435763235570955:2118] self: [6:7574435763235570962:0] peer: [4:7574435766776465278:0] socket: 361 qp: -1
2025-11-19T13:59:14.699943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435763235570971:2064] [node 4] ICS10 traffic start
2025-11-19T13:59:14.699953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:14.701115Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435764665081499:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.701127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435764665081499:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:14.701806Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435764665081509:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.702100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435764665081521:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11592
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.702052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435766776465213:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.702073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:14.702082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435766776465213:2049] [node 2] ICP22 created new session: [4:7574435766776465309:2049]
2025-11-19T13:59:14.702089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435766776465309:2049] [node 2] ICS09 handshake done sender: [4:7574435766776465295:2118] self: [4:7574435766776465298:0] peer: [2:7574435766580348405:0] socket: 364 qp: -1
2025-11-19T13:59:14.702096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435766776465309:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:14.702104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:14.702418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435766776465311:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51718
2025-11-19T13:59:14.702804Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435764665081499:2109] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:14.702231Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435765131597363:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:14.702319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435765131597360:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.702334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:14.702344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435765131597360:2107] [node 4] ICP22 created new session: [3:7574435765131597380:2107]
2025-11-19T13:59:14.702350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435765131597380:2107] [node 4] ICS09 handshake done sender: [3:7574435765131597363:2118] self: [3:7574435765131597362:0] peer: [4:7574435766776465305:0] socket: 346 qp: -1
2025-11-19T13:59:14.702355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435765131597380:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:14.702362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:14.704004Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435766776465302:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.704377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435763235570973:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:828
2025-11-19T13:59:14.704813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435766776465214:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.704828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:14.704834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435766776465214:2050] [node 3] ICP22 created new session: [4:7574435766776465312:2050]
2025-11-19T13:59:14.704839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435766776465312:2050] [node 3] ICS09 handshake done sender: [4:7574435766776465302:2120] self: [4:7574435766776465305:0] peer: [3:7574435765131597362:0] socket: 368 qp: -1
2025-11-19T13:59:14.704843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435766776465312:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:14.704849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:14.703331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435764665081465:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.703350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:14.703356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435764665081465:2060] [node 6] ICP22 created new session: [5:7574435764665081523:2060]
2025-11-19T13:59:14.703361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435764665081523:2060] [node 6] ICS09 handshake done sender: [5:7574435764665081499:2109] self: [5:7574435764665081471:0] peer: [6:7574435763235570967:0] socket: 376 qp: -1
2025-11-19T13:59:14.703366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435764665081523:2060] [node 6] ICS10 traffic start
2025-11-19T13:59:14.703373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:14.703464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435764665081428:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.703469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:14.703472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435764665081428:2049] [node 2] ICP22 created new session: [5:7574435764665081525:2049]
2025-11-19T13:59:14.703476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435764665081525:2049] [node 2] ICS09 handshake done sender: [5:7574435764665081509:2118] self: [5:7574435764665081514:0] peer: [2:7574435766580348410:0] socket: 372 qp: -1
2025-11-19T13:59:14.703479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435764665081525:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:14.703484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:14.706605Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435763235570964:2121] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:14.707638Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435764665081521:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.707676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435764665081451:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:14.707836Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435766776465293:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:14.707847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435766776465293:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:14.708751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435764665081528:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11592
2025-11-19T13:59:14.708835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435763235570915:2082] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18477) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435763235570964:2121] held: no
2025-11-19T13:59:14.708951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435763235570915:2082] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.708963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:14.708970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435763235570915:2082] [node 5] ICP22 created new session: [6:7574435763235570974:2082]
2025-11-19T13:59:14.708975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435763235570974:2082] [node 5] ICS09 handshake done sender: [6:7574435763235570964:2121] self: [6:7574435763235570967:0] peer: [5:7574435764665081471:0] socket: 379 qp: -1
2025-11-19T13:59:14.708981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435763235570974:2082] [node 5] ICS10 traffic start
2025-11-19T13:59:14.708988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:14.711931Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435766776465293:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:14.712186Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435764665081521:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:14.712356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435766776465259:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:14.712376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:14.712383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435766776465259:2091] [node 5] ICP22 created new session: [4:7574435766776465315:2091]
2025-11-19T13:59:14.712388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435766776465315:2091] [node 5] ICS09 handshake done sender: [4:7574435766776465293:2117] self: [4:7574435766776465261:0] peer: [5:7574435764665081527:0] socket: 382 qp: -1
2025-11-19T13:59:14.712396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435766776465315:2091] [node 5] ICS10 traffic start
2025-11-19T13:59:14.712404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:14.712789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435764665081451:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:14.712808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:14.712812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435764665081451:2062] [node 4] ICP22 created new session: [5:7574435764665081529:2062]
2025-11-19T13:59:14.712817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435764665081529:2062] [node 4] ICS09 handshake done sender: [5:7574435764665081521:2122] self: [5:7574435764665081527:0] peer: [4:7574435766776465261:0] socket: 323 qp: -1
2025-11-19T13:59:14.712822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435764665081529:2062] [node 4] ICS10 traffic start
2025-11-19T13:59:14.712830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:14.719201Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:14.719278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435766580348424:2048] [node 1] ICS01 socket: 350 reason# EndOfStream
2025-11-19T13:59:14.719285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435766580348335:2048] [node 1] ICP30 unregister session Session# [2:7574435766580348424:2048] VirtualId# [2:7574435766580348418:0]
2025-11-19T13:59:14.719291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435766580348424:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.719320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:14.719338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:14.720311Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:14.720388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435765131597365:2048] [node 1] ICS07 socket disconnect 343 reason# EndOfStream
2025-11-19T13:59:14.720393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435765131597365:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.720429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:14.720447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435765131597365:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:14.721063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435765131597278:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1337) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:14.721076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435765131597365:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:14.721081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435765131597278:2048] [node 1] ICP30 unregister session Session# [3:7574435765131597365:2048] VirtualId# [3:7574435765131597361:0]
2025-11-19T13:59:14.721094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:14.721099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435765131597278:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:14.721103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:14.723591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:14.723664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435763235570958:2048] [node 1] ICS07 socket disconnect 336 reason# EndOfStream
2025-11-19T13:59:14.723669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435763235570958:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.723705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:14.723723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435763235570958:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:14.724244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435763235570871:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1337) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:14.724257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435763235570958:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:14.724262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435763235570871:2048] [node 1] ICP30 unregister session Session# [6:7574435763235570958:2048] VirtualId# [6:7574435763235570949:0]
2025-11-19T13:59:14.724274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:14.724279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435763235570871:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:14.724281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:14.727808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:14.727890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435766776465299:2048] [node 1] ICS01 socket: 357 reason# EndOfStream
2025-11-19T13:59:14.727901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435766776465212:2048] [node 1] ICP30 unregister session Session# [4:7574435766776465299:2048] VirtualId# [4:7574435766776465292:0]
2025-11-19T13:59:14.727907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435766776465299:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.727962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:14.727990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:14.728026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:14.728086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435764665081515:2048] [node 1] ICS01 socket: 369 reason# EndOfStream
2025-11-19T13:59:14.728091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435764665081427:2048] [node 1] ICP30 unregister session Session# [5:7574435764665081515:2048] VirtualId# [5:7574435764665081511:0]
2025-11-19T13:59:14.728095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435764665081515:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.728124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:14.728147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:14.731868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574435765131597278:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:14.736019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435763235570871:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:14.739864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:14.739957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435765131597377:2049] [node 2] ICS01 socket: 331 reason# EndOfStream
2025-11-19T13:59:14.739965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435765131597279:2049] [node 2] ICP30 unregister session Session# [3:7574435765131597377:2049] VirtualId# [3:7574435765131597359:0]
2025-11-19T13:59:14.739977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435765131597377:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.740021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:14.740043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:14.748054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:14.748144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435764665081525:2049] [node 2] ICS01 socket: 372 reason# EndOfStream
2025-11-19T13:59:14.747972Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:14.748084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435763235570960:2049] [node 2] ICS01 socket: 334 reason# EndOfStream
2025-11-19T13:59:14.748095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435763235570872:2049] [node 2] ICP30 unregister session Session# [6:7574435763235570960:2049] VirtualId# [6:7574435763235570952:0]
2025-11-19T13:59:14.748102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435763235570960:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.748169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:14.748192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:14.748305Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:14.748366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435766776465309:2049] [node 2] ICS01 socket: 364 reason# EndOfStream
2025-11-19T13:59:14.748370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435766776465213:2049] [node 2] ICP30 unregister session Session# [4:7574435766776465309:2049] VirtualId# [4:7574435766776465298:0]
2025-11-19T13:59:14.748374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435766776465309:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.748399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:14.748419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:14.748151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435764665081428:2049] [node 2] ICP30 unregister session Session# [5:7574435764665081525:2049] VirtualId# [5:7574435764665081514:0]
2025-11-19T13:59:14.748157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435764665081525:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.748208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:14.748236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:14.763677Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:14.763784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435764665081518:2050] [node 3] ICS01 socket: 348 reason# EndOfStream
2025-11-19T13:59:14.763795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435764665081429:2050] [node 3] ICP30 unregister session Session# [5:7574435764665081518:2050] VirtualId# [5:7574435764665081506:0]
2025-11-19T13:59:14.763803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435764665081518:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.763866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:14.763892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:14.763961Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:14.764050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435763235570968:2050] [node 3] ICS01 socket: 362 reason# EndOfStream
2025-11-19T13:59:14.764058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435763235570873:2050] [node 3] ICP30 unregister session Session# [6:7574435763235570968:2050] VirtualId# [6:7574435763235570963:0]
2025-11-19T13:59:14.764064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435763235570968:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.764120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:14.764144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:14.775340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:14.775441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435763235570971:2064] [node 4] ICS07 socket disconnect 361 reason# EndOfStream
2025-11-19T13:59:14.775449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435763235570971:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:14.775512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:14.775538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435763235570971:2064] [node 4] ICS15 start handshake
2025-11-19T13:59:14.776436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435763235570895:2064] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1738) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:14.776457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435763235570971:2064] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:14.776463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435763235570895:2064] [node 4] ICP30 unregister session Session# [6:7574435763235570971:2064] VirtualId# [6:7574435763235570962:0]
2025-11-19T13:59:14.776477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:14.776483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435763235570895:2064] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:14.776485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:15.183833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435769121017148:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:15.183842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435769121017147:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:15.200872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435770565237243:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:15.200895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435770565237244:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:15.223929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435770416271626:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:15.223947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435770416271627:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:15.223953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435770416271628:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:15.211248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435770935150096:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:15.211906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435770935150095:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:15.243841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435769594506429:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:15.243857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435769594506430:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:15.243863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435769594506431:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:15.267857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435770721780704:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:15.267874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435770721780705:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:15.267880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435770721780706:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:15.306555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435770565237290:2090] [node 6] ICP01 ready to work
2025-11-19T13:59:15.307536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435770935150153:2101] [node 4] ICP01 ready to work
2025-11-19T13:59:15.307617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435770935150154:2102] [node 5] ICP01 ready to work
2025-11-19T13:59:15.311989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435770416271686:2105] [node 5] ICP01 ready to work
2025-11-19T13:59:15.311974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435769121017205:2099] [node 5] ICP01 ready to work
2025-11-19T13:59:15.312057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435769121017206:2102] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.320313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435770565237303:2099] [node 5] ICP01 ready to work
2025-11-19T13:59:15.320776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435770721780722:2065] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.321672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435770565237307:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8039
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.320337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435770935150160:2103] [node 6] ICP01 ready to work
2025-11-19T13:59:15.321318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435769121017208:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16742
2025-11-19T13:59:15.321727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435769594506489:2090] [node 4] ICP01 ready to work
2025-11-19T13:59:15.322142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435769594506490:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10850
2025-11-19T13:59:15.323899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435770721780729:2071] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-19T13:59:15.324997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435769121017217:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16742
2025-11-19T13:59:15.329477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435769594506495:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10850
2025-11-19T13:59:15.330148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435770565237298:2096] [node 4] ICP01 ready to work
2025-11-19T13:59:15.330607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435769121017198:2097] [node 6] ICP01 ready to work
2025-11-19T13:59:15.332637Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435770565237307:2101] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.332519Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435769121017208:2103] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.331768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435769594506463:2078] [node 6] ICP01 ready to work
2025-11-19T13:59:15.336625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435770565237243:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25065589ns) {Configure:104(25064642ns) {ProcessConfigured:135(24786349ns) {StartInitialHandshake:157(24785769ns) {DropHandshakes:469(682ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(258ns) PrepareNewSessionHandshake:476(834ns) {SwitchToState:178(406ns)}}}}}
2025-11-19T13:59:15.336664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435770565237243:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:15.336823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435770416271627:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(25270819ns) {Configure:104(25270218ns) {ProcessConfigured:135(24922294ns) {StartInitialHandshake:157(24922141ns) {DropHandshakes:469(516ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(26ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(1220ns) {SwitchToState:178(160ns)}}}}}
2025-11-19T13:59:15.337263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435770416271694:2106] [node 6] ICP01 ready to work
2025-11-19T13:59:15.337871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435770416271686:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(25841259ns) {Configure:104(25840778ns) {ProcessConfigured:135(25650067ns) {StartInitialHandshake:157(25649914ns) {DropHandshakes:469(235ns) {DropIncomingHandshake:441(30ns) DropOutgoingHandshake:456(29ns)} GenerateSessionVirtualId:988(37ns) PrepareNewSessionHandshake:476(573ns) {SwitchToState:178(402ns)}}}}}
2025-11-19T13:59:15.339992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435769594506496:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10850
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.348666Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435770721780722:2065] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.349870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435770721780777:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
2025-11-19T13:59:15.351290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435770416271626:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21414281ns) {Configure:104(21413723ns) {ProcessConfigured:135(21107086ns) {StartInitialHandshake:157(21106553ns) {DropHandshakes:469(870ns) {DropIncomingHandshake:441(173ns) DropOutgoingHandshake:456(204ns)} GenerateSessionVirtualId:988(279ns) PrepareNewSessionHandshake:476(1485ns) {SwitchToState:178(1059ns)}}}}}
2025-11-19T13:59:15.350065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435770935150096:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(42409198ns) {Configure:104(42408724ns) {ProcessConfigured:135(42138725ns) {StartInitialHandshake:157(42138268ns) {DropHandshakes:469(556ns) {DropIncomingHandshake:441(155ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(152ns) PrepareNewSessionHandshake:476(1568ns) {SwitchToState:178(1264ns)}}}}}
2025-11-19T13:59:15.355843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435769121017198:2097] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25169505ns) {Configure:104(25168903ns) {ProcessConfigured:135(24900222ns) {StartInitialHandshake:157(24899644ns) {DropHandshakes:469(710ns) {DropIncomingHandshake:441(104ns) DropOutgoingHandshake:456(37ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(1183ns) {SwitchToState:178(798ns)}}}}}
2025-11-19T13:59:15.355998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435769594506431:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(48084442ns) {Configure:104(48083117ns) {ProcessConfigured:135(47554940ns) {StartInitialHandshake:157(47554481ns) {DropHandshakes:469(188ns) {DropIncomingHandshake:441(31ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(736ns) {SwitchToState:178(271ns)}}}}}
2025-11-19T13:59:15.356376Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435769594506490:2104] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.356466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435769594506430:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:15.356677Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435770565237298:2096] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(26359992ns) {Configure:104(26359512ns) {ProcessConfigured:135(24727621ns) {StartInitialHandshake:157(24727204ns) {DropHandshakes:469(842ns) {DropIncomingHandshake:441(185ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(127ns) PrepareNewSessionHandshake:476(582ns) {SwitchToState:178(276ns)}}}}}
2025-11-19T13:59:15.356931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435770565237305:2100] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.356937Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435770565237305:2100] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:15.357484Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435770565237305:2100] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:15.357597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435770565237303:2099] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.357612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:15.357618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435770565237303:2099] [node 5] ICP22 created new session: [2:7574435770565237322:2099]
2025-11-19T13:59:15.357624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435770565237322:2099] [node 5] ICS09 handshake done sender: [2:7574435770565237305:2100] self: [2:7574435770565237304:0] peer: [5:7574435769594506508:0] socket: 330 qp: -1
2025-11-19T13:59:15.357631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435770565237322:2099] [node 5] ICS10 traffic start
2025-11-19T13:59:15.357641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.356834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435770935150160:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(36424073ns) {Configure:104(36423742ns) {ProcessConfigured:135(36143353ns) {StartInitialHandshake:157(36143238ns) {DropHandshakes:469(478ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(98ns)} GenerateSessionVirtualId:988(114ns) PrepareNewSessionHandshake:476(742ns) {SwitchToState:178(163ns)}}}}}
2025-11-19T13:59:15.358955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435769121017228:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16742
2025-11-19T13:59:15.359107Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435769121017228:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.360317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435769594506510:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10850
2025-11-19T13:59:15.360762Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435769594506496:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.360784Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435769594506490:2104] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:15.360929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435769594506430:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.360948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:15.360955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435769594506430:2049] [node 2] ICP22 created new session: [5:7574435769594506511:2049]
2025-11-19T13:59:15.360960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435769594506511:2049] [node 2] ICS09 handshake done sender: [5:7574435769594506490:2104] self: [5:7574435769594506508:0] peer: [2:7574435770565237304:0] socket: 334 qp: -1
2025-11-19T13:59:15.360966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435769594506511:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:15.360982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.363553Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435770721780705:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(31448783ns) {Configure:104(31447952ns) {ProcessConfigured:135(31172165ns) {StartInitialHandshake:157(31171772ns) {DropHandshakes:469(484ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(52ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:476(727ns) {SwitchToState:178(567ns)}}}}}
2025-11-19T13:59:15.363604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435770721780705:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:15.364559Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435770565237292:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.364571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435770565237292:2094] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.364070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435770416271704:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59398
2025-11-19T13:59:15.364127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435770416271705:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59398
2025-11-19T13:59:15.364136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435770416271706:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59398
2025-11-19T13:59:15.365645Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435770416271704:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.365677Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435770416271706:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.365702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435770416271626:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:15.365712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435770416271628:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:15.366497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435770416271709:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59398
2025-11-19T13:59:15.367228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435770416271710:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59398
2025-11-19T13:59:15.367502Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435770416271706:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:15.365832Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435770935150162:2105] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.367586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435770416271628:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.365843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435770935150162:2105] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:15.367603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:15.367349Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435770935150162:2105] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:15.367609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435770416271628:2050] [node 3] ICP22 created new session: [4:7574435770416271711:2050]
2025-11-19T13:59:15.367614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435770416271711:2050] [node 3] ICS09 handshake done sender: [4:7574435770416271706:2117] self: [4:7574435770416271708:0] peer: [3:7574435770935150157:0] socket: 358 qp: -1
2025-11-19T13:59:15.367619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435770416271711:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:15.367647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:15.366606Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435769121017210:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.366616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435769121017210:2104] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:15.368149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435770935150153:2101] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.368165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:15.368170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435770935150153:2101] [node 4] ICP22 created new session: [3:7574435770935150173:2101]
2025-11-19T13:59:15.368175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435770935150173:2101] [node 4] ICS09 handshake done sender: [3:7574435770935150162:2105] self: [3:7574435770935150157:0] peer: [4:7574435770416271708:0] socket: 341 qp: -1
2025-11-19T13:59:15.368179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435770935150173:2101] [node 4] ICS10 traffic start
2025-11-19T13:59:15.368185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.368296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435769594506513:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10850
2025-11-19T13:59:15.368508Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435770565237292:2094] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:15.368638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435770565237290:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.368650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:15.368655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435770565237290:2090] [node 6] ICP22 created new session: [2:7574435770565237327:2090]
2025-11-19T13:59:15.368660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435770565237327:2090] [node 6] ICS09 handshake done sender: [2:7574435770565237292:2094] self: [2:7574435770565237291:0] peer: [6:7574435770721780780:0] socket: 326 qp: -1
2025-11-19T13:59:15.368664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435770565237327:2090] [node 6] ICS10 traffic start
2025-11-19T13:59:15.368669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:15.369175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435770721780754:2061] [node 4] ICP01 ready to work
2025-11-19T13:59:15.370890Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435769594506495:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.371115Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435769121017210:2104] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:15.371873Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435769121017212:2105] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.371885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435769121017212:2105] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.376003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435770565237332:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8039
2025-11-19T13:59:15.376022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435770565237333:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8039
2025-11-19T13:59:15.376069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435770565237334:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8039
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.376499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435770935150178:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
2025-11-19T13:59:15.376545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435770935150179:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
2025-11-19T13:59:15.376555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435770935150180:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.377248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435770935150181:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
2025-11-19T13:59:15.377290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435770935150182:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
2025-11-19T13:59:15.377325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435770935150178:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.377344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435770935150096:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:15.377460Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435770565237294:2095] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.377467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435770565237294:2095] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:15.377972Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435770565237294:2095] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:15.378047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435770565237244:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.378063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:15.378068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435770565237244:2049] [node 3] ICP22 created new session: [2:7574435770565237336:2049]
2025-11-19T13:59:15.378074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435770565237336:2049] [node 3] ICS09 handshake done sender: [2:7574435770565237294:2095] self: [2:7574435770565237293:0] peer: [3:7574435770935150183:0] socket: 325 qp: -1
2025-11-19T13:59:15.378079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435770565237336:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:15.378086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.378313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435770935150184:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
2025-11-19T13:59:15.378443Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435770935150178:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:15.378545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435770935150096:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.378557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:15.378561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435770935150096:2049] [node 2] ICP22 created new session: [3:7574435770935150185:2049]
2025-11-19T13:59:15.378565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435770935150185:2049] [node 2] ICS09 handshake done sender: [3:7574435770935150178:2115] self: [3:7574435770935150183:0] peer: [2:7574435770565237293:0] socket: 329 qp: -1
2025-11-19T13:59:15.378570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435770935150185:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:15.378584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.379423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435770721780783:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
2025-11-19T13:59:15.376704Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435769594506463:2078] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(36602619ns) {Configure:104(36602138ns) {ProcessConfigured:135(36313202ns) {StartInitialHandshake:157(36312417ns) {DropHandshakes:469(560ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(130ns) PrepareNewSessionHandshake:476(826ns) {SwitchToState:178(641ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.380472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435770721780784:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
2025-11-19T13:59:15.380207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435769121017206:2102] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.380227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:15.380233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435769121017206:2102] [node 4] ICP22 created new session: [1:7574435769121017231:2102]
2025-11-19T13:59:15.380240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435769121017231:2102] [node 4] ICS09 handshake done sender: [1:7574435769121017210:2104] self: [1:7574435769121017209:0] peer: [4:7574435770416271707:0] socket: 331 qp: -1
2025-11-19T13:59:15.380245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435769121017231:2102] [node 4] ICS10 traffic start
2025-11-19T13:59:15.380274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:15.380826Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435770565237333:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.380854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435770565237290:2090] [node 6] ICP18 handshake (actor [2:7574435770565237333:2117]) is too old
2025-11-19T13:59:15.380616Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435770935150159:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.380630Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435770935150159:2104] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:15.380762Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435770935150181:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.380784Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435770935150182:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.382065Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435770935150179:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.382091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435770935150095:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:15.393983Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435770416271704:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.396744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435770721780704:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(64684084ns) {Configure:104(64683357ns) {ProcessConfigured:135(64378700ns) {StartInitialHandshake:157(64377871ns) {DropHandshakes:469(411ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:476(820ns) {SwitchToState:178(73ns)}}}}}
2025-11-19T13:59:15.396803Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435770721780722:2065] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:15.396524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435770416271714:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59398
2025-11-19T13:59:15.397560Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435770416271705:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.397798Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435770565237332:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.397654Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435770416271714:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.397718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435770416271627:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:15.397819Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435769121017217:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.397885Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435769594506513:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.397923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435769594506489:2090] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:15.397845Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435770565237320:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.397851Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435770565237320:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:15.398405Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435770565237320:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:15.398491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435770565237298:2096] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.398505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:15.398512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435770565237298:2096] [node 4] ICP22 created new session: [2:7574435770565237340:2096]
2025-11-19T13:59:15.398534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435770565237340:2096] [node 4] ICS09 handshake done sender: [2:7574435770565237320:2112] self: [2:7574435770565237314:0] peer: [4:7574435770416271716:0] socket: 368 qp: -1
2025-11-19T13:59:15.398541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435770565237340:2096] [node 4] ICS10 traffic start
2025-11-19T13:59:15.398550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:15.399680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435770565237244:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:15.399894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435769121017205:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(87791631ns) {Configure:104(87790365ns) {ProcessConfigured:135(87581377ns) {StartInitialHandshake:157(87580743ns) {DropHandshakes:469(212ns) {DropIncomingHandshake:441(43ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:476(471ns) {SwitchToState:178(101ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.400498Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435770416271627:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26399) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435770416271714:2120] held: no
2025-11-19T13:59:15.400519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435770416271626:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.400528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:15.403803Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435769121017215:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.403817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435769121017215:2106] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:15.400532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435770416271626:2048] [node 1] ICP22 created new session: [4:7574435770416271718:2048]
2025-11-19T13:59:15.400536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435770416271718:2048] [node 1] ICS09 handshake done sender: [4:7574435770416271704:2115] self: [4:7574435770416271707:0] peer: [1:7574435769121017209:0] socket: 340 qp: -1
2025-11-19T13:59:15.400541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435770416271718:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:15.400554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:15.400770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435770416271717:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59398
2025-11-19T13:59:15.403851Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435770416271700:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.403856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435770416271700:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:15.403862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435769594506431:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:15.403900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435769594506431:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(32965604ns) {IncomingHandshake:261(32965079ns) {DropIncomingHandshake:441(17ns) PrepareNewSessionHandshake:476(337ns) {SwitchToState:178(84ns)} GenerateSessionVirtualId:988(166ns) IssueIncomingHandshakeReply:193(32961058ns) {SwitchToState:178(563ns)}}}
2025-11-19T13:59:15.405092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435770721780705:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.405109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:15.405117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435770721780705:2049] [node 2] ICP22 created new session: [6:7574435770721780788:2049]
2025-11-19T13:59:15.405123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435770721780788:2049] [node 2] ICS09 handshake done sender: [6:7574435770721780722:2065] self: [6:7574435770721780780:0] peer: [2:7574435770565237291:0] socket: 327 qp: -1
2025-11-19T13:59:15.405128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435770721780788:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:15.405135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:15.405274Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435770721780777:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.404048Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435770416271714:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:15.405969Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435770935150159:2104] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:15.406066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435770935150154:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.406080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:15.406085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435770935150154:2102] [node 5] ICP22 created new session: [3:7574435770935150192:2102]
2025-11-19T13:59:15.406090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435770935150192:2102] [node 5] ICS09 handshake done sender: [3:7574435770935150159:2104] self: [3:7574435770935150156:0] peer: [5:7574435769594506514:0] socket: 337 qp: -1
2025-11-19T13:59:15.406094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435770935150192:2102] [node 5] ICS10 traffic start
2025-11-19T13:59:15.406100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.406443Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435770721780705:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:15.406499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435770721780786:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
2025-11-19T13:59:15.406508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435770721780787:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.406783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435769121017236:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16742
2025-11-19T13:59:15.406919Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435770721780787:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.407877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435770721780706:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:15.408101Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435769594506431:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435769594506495:2106] held: no
2025-11-19T13:59:15.408102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435770416271627:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.408118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:15.408127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435770416271627:2049] [node 2] ICP22 created new session: [4:7574435770416271720:2049]
2025-11-19T13:59:15.408132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435770416271720:2049] [node 2] ICS09 handshake done sender: [4:7574435770416271714:2120] self: [4:7574435770416271716:0] peer: [2:7574435770565237314:0] socket: 369 qp: -1
2025-11-19T13:59:15.408137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435770416271720:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:15.408144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:15.412011Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435769121017236:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.411931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435770721780784:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.412005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435770721780729:2071] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:15.412242Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435769594506515:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.412251Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435769594506515:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:15.414603Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435769594506515:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:15.419947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435769594506463:2078] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.419972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:15.419979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435769594506463:2078] [node 6] ICP22 created new session: [5:7574435769594506520:2078]
2025-11-19T13:59:15.419985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435769594506520:2078] [node 6] ICS09 handshake done sender: [5:7574435769594506515:2119] self: [5:7574435769594506504:0] peer: [6:7574435770721780792:0] socket: 375 qp: -1
2025-11-19T13:59:15.419995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435769594506520:2078] [node 6] ICS10 traffic start
2025-11-19T13:59:15.420020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:15.420287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435770721780754:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(51035079ns) {Configure:104(51033901ns) {ProcessConfigured:135(50742965ns) {StartInitialHandshake:157(50742234ns) {DropHandshakes:469(461ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(174ns) PrepareNewSessionHandshake:476(530ns) {SwitchToState:178(281ns)}}}}}
2025-11-19T13:59:15.420321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435770721780754:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.420586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435770565237342:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8039
2025-11-19T13:59:15.424335Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435769121017212:2105] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:15.424680Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435769121017215:2106] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:15.424742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435769121017148:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.424765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:15.424770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435769121017148:2049] [node 3] ICP22 created new session: [1:7574435769121017240:2049]
2025-11-19T13:59:15.424776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435769121017240:2049] [node 3] ICS09 handshake done sender: [1:7574435769121017215:2106] self: [1:7574435769121017199:0] peer: [3:7574435770935150189:0] socket: 338 qp: -1
2025-11-19T13:59:15.424781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435769121017240:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:15.424788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:15.425412Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435770416271700:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:15.425504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435770416271686:2105] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.425518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:15.425524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435770416271686:2105] [node 5] ICP22 created new session: [4:7574435770416271723:2105]
2025-11-19T13:59:15.425529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435770416271723:2105] [node 5] ICS09 handshake done sender: [4:7574435770416271700:2111] self: [4:7574435770416271687:0] peer: [5:7574435769594506518:0] socket: 365 qp: -1
2025-11-19T13:59:15.425534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435770416271723:2105] [node 5] ICS10 traffic start
2025-11-19T13:59:15.425540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:15.424532Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435770565237307:2101] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:15.431543Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435770935150169:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.431558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435770935150169:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:15.432006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435769121017147:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.432022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:15.432028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435769121017147:2048] [node 2] ICP22 created new session: [1:7574435769121017242:2048]
2025-11-19T13:59:15.432033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435769121017242:2048] [node 2] ICS09 handshake done sender: [1:7574435769121017212:2105] self: [1:7574435769121017211:0] peer: [2:7574435770565237316:0] socket: 332 qp: -1
2025-11-19T13:59:15.432037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435769121017242:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:15.432044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:15.432774Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435770935150095:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26177) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435770935150179:2116] held: no
2025-11-19T13:59:15.431594Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435770416271696:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.431606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435770416271696:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.440527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435770935150194:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
2025-11-19T13:59:15.440709Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435770935150179:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:15.441570Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435770721780729:2071] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25130) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435770721780784:2116] held: no
2025-11-19T13:59:15.441715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435770935150095:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.441732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:15.441739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435770935150095:2048] [node 1] ICP22 created new session: [3:7574435770935150195:2048]
2025-11-19T13:59:15.441745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435770935150195:2048] [node 1] ICS09 handshake done sender: [3:7574435770935150179:2116] self: [3:7574435770935150189:0] peer: [1:7574435769121017199:0] socket: 371 qp: -1
2025-11-19T13:59:15.441752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435770935150195:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:15.441768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:15.444208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435769594506463:2078] [node 6] ICP35 event processing took too much time PendingConnection:278(24232802ns) {HandleHandshakeStatus:329(24232093ns) {DropIncomingHandshake:441(5147ns) DropOutgoingHandshake:456(494ns) SwitchToState:178(62ns) ProcessPendingSessionEvents:495(18751177ns) {ForwardSessionEventToSession:612(18739471ns) ForwardSessionEventToSession:612(1587ns) ForwardSessionEventToSession:612(608ns) ForwardSessionEventToSession:612(625ns) ForwardSessionEventToSession:612(705ns) ForwardSessionEventToSession:612(1628ns) ForwardSessionEventToSession:612(638ns) ForwardSessionEventToSession:612(535ns) ForwardSessionEventToSession:612(536ns) ForwardSessionEventToSession:612(970ns)}}}
2025-11-19T13:59:15.447336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435770721780795:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
2025-11-19T13:59:15.447381Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435770721780786:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.447421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435770721780704:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.448340Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435770721780784:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:15.448396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435770721780798:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
2025-11-19T13:59:15.448494Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435770935150169:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:15.448748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435770721780729:2071] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.448762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:15.448768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435770721780729:2071] [node 5] ICP22 created new session: [6:7574435770721780799:2071]
2025-11-19T13:59:15.448773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435770721780799:2071] [node 5] ICS09 handshake done sender: [6:7574435770721780784:2116] self: [6:7574435770721780792:0] peer: [5:7574435769594506504:0] socket: 377 qp: -1
2025-11-19T13:59:15.448778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435770721780799:2071] [node 5] ICS10 traffic start
2025-11-19T13:59:15.448785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:15.449342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435770935150160:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.449362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:15.449368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435770935150160:2103] [node 6] ICP22 created new session: [3:7574435770935150198:2103]
2025-11-19T13:59:15.449373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435770935150198:2103] [node 6] ICS09 handshake done sender: [3:7574435770935150169:2110] self: [3:7574435770935150161:0] peer: [6:7574435770721780791:0] socket: 348 qp: -1
2025-11-19T13:59:15.449378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435770935150198:2103] [node 6] ICS10 traffic start
2025-11-19T13:59:15.449385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:15.450284Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435770416271696:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:15.450385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435770416271694:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.450396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:15.450401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435770416271694:2106] [node 6] ICP22 created new session: [4:7574435770416271726:2106]
2025-11-19T13:59:15.450405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435770416271726:2106] [node 6] ICS09 handshake done sender: [4:7574435770416271696:2110] self: [4:7574435770416271695:0] peer: [6:7574435770721780794:0] socket: 347 qp: -1
2025-11-19T13:59:15.450410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435770416271726:2106] [node 6] ICS10 traffic start
2025-11-19T13:59:15.450415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:15.447653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435769594506522:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10850
2025-11-19T13:59:15.448288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435769594506523:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10850
2025-11-19T13:59:15.448330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435769594506524:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10850
2025-11-19T13:59:15.448860Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435769594506495:2106] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:15.449932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435769594506431:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.449943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:15.449948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435769594506431:2050] [node 3] ICP22 created new session: [5:7574435769594506525:2050]
2025-11-19T13:59:15.449953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435769594506525:2050] [node 3] ICS09 handshake done sender: [5:7574435769594506495:2106] self: [5:7574435769594506514:0] peer: [3:7574435770935150156:0] socket: 343 qp: -1
2025-11-19T13:59:15.449958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435769594506525:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:15.449964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:15.450071Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435769594506524:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.450102Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435769594506513:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:15.451629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435769594506489:2090] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.451639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:15.451654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435769594506489:2090] [node 4] ICP22 created new session: [5:7574435769594506527:2090]
2025-11-19T13:59:15.451658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435769594506527:2090] [node 4] ICS09 handshake done sender: [5:7574435769594506513:2118] self: [5:7574435769594506518:0] peer: [4:7574435770416271687:0] socket: 366 qp: -1
2025-11-19T13:59:15.451663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435769594506527:2090] [node 4] ICS10 traffic start
2025-11-19T13:59:15.451669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:15.451908Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435769121017234:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.451723Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435769594506489:2090] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:15.451798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435769594506429:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:15.451921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435769121017234:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:15.451945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435770565237243:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.451970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:15.451978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435770565237243:2048] [node 1] ICP22 created new session: [2:7574435770565237343:2048]
2025-11-19T13:59:15.451984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435770565237343:2048] [node 1] ICS09 handshake done sender: [2:7574435770565237307:2101] self: [2:7574435770565237316:0] peer: [1:7574435769121017211:0] socket: 333 qp: -1
2025-11-19T13:59:15.451990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435770565237343:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:15.452003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:15.452182Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435770721780787:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:15.456196Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435770416271686:2105] [node 5] ICP35 event processing took too much time PendingConnection:278(30657503ns) {HandleHandshakeStatus:329(30657039ns) {DropIncomingHandshake:441(3985ns) DropOutgoingHandshake:456(75ns) SwitchToState:178(242ns) ProcessPendingSessionEvents:495(8838788ns) {ForwardSessionEventToSession:612(8828758ns) ForwardSessionEventToSession:612(1285ns) ForwardSessionEventToSession:612(875ns) ForwardSessionEventToSession:612(534ns) ForwardSessionEventToSession:612(575ns) ForwardSessionEventToSession:612(802ns) ForwardSessionEventToSession:612(644ns) ForwardSessionEventToSession:612(523ns) ForwardSessionEventToSession:612(675ns) ForwardSessionEventToSession:612(936ns)}}}
2025-11-19T13:59:15.456126Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435769121017227:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.456141Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435769121017227:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:15.456957Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435769121017227:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:15.457042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435769121017198:2097] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.457056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:15.457062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435769121017198:2097] [node 6] ICP22 created new session: [1:7574435769121017246:2097]
2025-11-19T13:59:15.457068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435769121017246:2097] [node 6] ICS09 handshake done sender: [1:7574435769121017227:2115] self: [1:7574435769121017222:0] peer: [6:7574435770721780797:0] socket: 378 qp: -1
2025-11-19T13:59:15.457073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435769121017246:2097] [node 6] ICS10 traffic start
2025-11-19T13:59:15.457080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.464203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435769594506530:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10850
2025-11-19T13:59:15.463864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435770565237243:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(38313468ns) {HandleHandshakeStatus:329(38312987ns) {DropIncomingHandshake:441(789ns) DropOutgoingHandshake:456(2535ns) SwitchToState:178(287ns) ProcessPendingSessionEvents:495(11796179ns) {ForwardSessionEventToSession:612(11786032ns) ForwardSessionEventToSession:612(1278ns) ForwardSessionEventToSession:612(734ns) ForwardSessionEventToSession:612(929ns) ForwardSessionEventToSession:612(526ns) ForwardSessionEventToSession:612(1509ns) ForwardSessionEventToSession:612(553ns) ForwardSessionEventToSession:612(510ns) ForwardSessionEventToSession:612(652ns) ForwardSessionEventToSession:612(1122ns)}}}
2025-11-19T13:59:15.463895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435770565237243:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:15.463951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435770721780706:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.463970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:15.463978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435770721780706:2050] [node 3] ICP22 created new session: [6:7574435770721780803:2050]
2025-11-19T13:59:15.463984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435770721780803:2050] [node 3] ICS09 handshake done sender: [6:7574435770721780787:2119] self: [6:7574435770721780791:0] peer: [3:7574435770935150161:0] socket: 346 qp: -1
2025-11-19T13:59:15.463988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435770721780803:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:15.463995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:15.464062Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435770721780706:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:15.464165Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435770721780704:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26177) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435770721780786:2118] held: no
2025-11-19T13:59:15.464231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435770721780801:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.464444Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435769121017234:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:15.464419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435770721780802:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
2025-11-19T13:59:15.464748Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435770721780777:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:15.464830Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435769594506524:2122] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:15.465038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435769121017205:2099] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:15.465056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:15.465062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435769121017205:2099] [node 5] ICP22 created new session: [1:7574435769121017249:2099]
2025-11-19T13:59:15.465066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435769121017249:2099] [node 5] ICS09 handshake done sender: [1:7574435769121017234:2117] self: [1:7574435769121017207:0] peer: [5:7574435769594506529:0] socket: 324 qp: -1
2025-11-19T13:59:15.465071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435769121017249:2099] [node 5] ICS10 traffic start
2025-11-19T13:59:15.465077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:15.466341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435769594506429:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.466357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:15.466363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435769594506429:2048] [node 1] ICP22 created new session: [5:7574435769594506531:2048]
2025-11-19T13:59:15.466369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435769594506531:2048] [node 1] ICS09 handshake done sender: [5:7574435769594506524:2122] self: [5:7574435769594506529:0] peer: [1:7574435769121017207:0] socket: 351 qp: -1
2025-11-19T13:59:15.466373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435769594506531:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:15.466378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:15.464554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435770416271728:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59398
2025-11-19T13:59:15.464821Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435770416271728:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:15.464849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435770416271694:2106] [node 6] ICP18 handshake (actor [4:7574435770416271728:2122]) is too old
2025-11-19T13:59:15.467659Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435770721780786:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:15.467781Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435770721780754:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1768) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435770721780777:2113] held: no
2025-11-19T13:59:15.467799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435770721780754:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.467806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:15.467810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435770721780754:2061] [node 4] ICP22 created new session: [6:7574435770721780805:2061]
2025-11-19T13:59:15.467819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435770721780805:2061] [node 4] ICS09 handshake done sender: [6:7574435770721780777:2113] self: [6:7574435770721780794:0] peer: [4:7574435770416271695:0] socket: 349 qp: -1
2025-11-19T13:59:15.467824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435770721780805:2061] [node 4] ICS10 traffic start
2025-11-19T13:59:15.467830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:15.472302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435770721780704:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:15.472315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:15.472320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435770721780704:2048] [node 1] ICP22 created new session: [6:7574435770721780807:2048]
2025-11-19T13:59:15.472327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435770721780807:2048] [node 1] ICS09 handshake done sender: [6:7574435770721780786:2118] self: [6:7574435770721780797:0] peer: [1:7574435769121017222:0] socket: 379 qp: -1
2025-11-19T13:59:15.472332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435770721780807:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:15.472340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:15.485061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:15.485158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435770935150195:2048] [node 1] ICS01 socket: 371 reason# EndOfStream
2025-11-19T13:59:15.485168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435770935150095:2048] [node 1] ICP30 unregister session Session# [3:7574435770935150195:2048] VirtualId# [3:7574435770935150189:0]
2025-11-19T13:59:15.485176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435770935150195:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:15.485204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:15.485223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:15.486366Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:15.486397Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:15.486477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574435770565237343:2048] [node 1] ICS07 socket disconnect 333 reason# EndOfStream
2025-11-19T13:59:15.486482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435770565237343:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:15.486535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:15.486554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435770565237343:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:15.486799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435770721780807:2048] [node 1] ICS01 socket: 379 reason# EndOfStream
2025-11-19T13:59:15.486808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435770721780704:2048] [node 1] ICP30 unregister session Session# [6:7574435770721780807:2048] VirtualId# [6:7574435770721780797:0]
2025-11-19T13:59:15.486821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435770721780807:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:15.486851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:15.486870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:15.486954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:15.487031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435769594506531:2048] [node 1] ICS01 socket: 351 reason# EndOfStream
2025-11-19T13:59:15.487035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435769594506429:2048] [node 1] ICP30 unregister session Session# [5:7574435769594506531:2048] VirtualId# [5:7574435769594506529:0]
2025-11-19T13:59:15.487039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435769594506531:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:15.487060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:15.487075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:15.490738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435770565237243:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26177) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:15.490754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435770565237343:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:15.490759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435770565237243:2048] [node 1] ICP30 unregister session Session# [2:7574435770565237343:2048] VirtualId# [2:7574435770565237316:0]
2025-11-19T13:59:15.490776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:15.490782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574435770565237243:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26177) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:15.490786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26177) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:15.491896Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:15.491978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435770416271718:2048] [node 1] ICS01 socket: 340 reason# EndOfStream
2025-11-19T13:59:15.491984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435770416271626:2048] [node 1] ICP30 unregister session Session# [4:7574435770416271718:2048] VirtualId# [4:7574435770416271707:0]
2025-11-19T13:59:15.491989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435770416271718:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:15.492030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:15.492055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:15.500765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:15.500838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435770935150185:2049] [node 2] ICS01 socket: 329 reason# EndOfStream
2025-11-19T13:59:15.500844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435770935150096:2049] [node 2] ICP30 unregister session Session# [3:7574435770935150185:2049] VirtualId# [3:7574435770935150183:0]
2025-11-19T13:59:15.500850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435770935150185:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:15.500884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:15.500911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:15.512486Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:15.512587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435770721780788:2049] [node 2] ICS01 socket: 327 reason# EndOfStream
2025-11-19T13:59:15.512595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435770721780705:2049] [node 2] ICP30 unregister session Session# [6:7574435770721780788:2049] VirtualId# [6:7574435770721780780:0]
2025-11-19T13:59:15.512601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435770721780788:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:15.512642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:15.512664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:15.512783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:15.512844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435770416271720:2049] [node 2] ICS01 socket: 369 reason# EndOfStream
2025-11-19T13:59:15.512848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435770416271627:2049] [node 2] ICP30 unregister session Session# [4:7574435770416271720:2049] VirtualId# [4:7574435770416271716:0]
2025-11-19T13:59:15.512852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435770416271720:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:15.512876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:15.512895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:15.512920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:15.512983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435769594506511:2049] [node 2] ICS01 socket: 334 reason# EndOfStream
2025-11-19T13:59:15.512986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435769594506430:2049] [node 2] ICP30 unregister session Session# [5:7574435769594506511:2049] VirtualId# [5:7574435769594506508:0]
2025-11-19T13:59:15.512989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435769594506511:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:15.513011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:15.513027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:15.526225Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:15.526329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435770721780803:2050] [node 3] ICS01 socket: 346 reason# EndOfStream
2025-11-19T13:59:15.526341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435770721780706:2050] [node 3] ICP30 unregister session Session# [6:7574435770721780803:2050] VirtualId# [6:7574435770721780791:0]
2025-11-19T13:59:15.526350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435770721780803:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:15.526413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:15.526440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:15.526566Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:15.526633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435769594506525:2050] [node 3] ICS01 socket: 343 reason# EndOfStream
2025-11-19T13:59:15.526639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435769594506431:2050] [node 3] ICP30 unregister session Session# [5:7574435769594506525:2050] VirtualId# [5:7574435769594506514:0]
2025-11-19T13:59:15.526643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435769594506525:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:15.526674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:15.526695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:15.543831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:15.543942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435770721780805:2061] [node 4] ICS01 socket: 349 reason# EndOfStream
2025-11-19T13:59:15.543953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435770721780754:2061] [node 4] ICP30 unregister session Session# [6:7574435770721780805:2061] VirtualId# [6:7574435770721780794:0]
2025-11-19T13:59:15.543961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435770721780805:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:15.544023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:15.544048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:15.901705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435768000425563:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:15.901718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435768000425564:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:15.939933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435770002922825:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:15.939987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435770002922826:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:15.995485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435770846502462:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:15.995498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435770846502463:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:16.021564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435775452425717:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:16.021575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435775452425718:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:16.021580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435775452425719:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:16.068348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435772120863973:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:16.068362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435772120863974:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:16.068366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435772120863975:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:16.111940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435773529043420:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:16.111953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435773529043421:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:16.111958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435773529043422:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:16.176412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435774297890182:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:16.176048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435772295392922:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:16.176148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435772295392923:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:16.176238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435772295392924:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:16.177890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435772120864018:2091] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.181601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435774297890185:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63039
2025-11-19T13:59:16.178900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435772120864024:2094] [node 6] ICP01 ready to work
2025-11-19T13:59:16.182514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435772295392927:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26480
2025-11-19T13:59:16.182357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435775141469818:2106] [node 5] ICP01 ready to work
2025-11-19T13:59:16.188134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435775141469822:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:16.188477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435774297890188:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.190887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435772295392937:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26480
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.196476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435772120864038:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
2025-11-19T13:59:16.196665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435774297890197:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.201074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435772295392927:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.205601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435774297890200:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63039
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.208740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435768000425564:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(32542534ns) {Configure:104(32541316ns) {ProcessConfigured:135(28139014ns) {StartInitialHandshake:157(28138335ns) {DropHandshakes:469(433ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:476(1135ns) {SwitchToState:178(648ns)}}}}}
2025-11-19T13:59:16.209755Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435774297890200:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.208933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773529043488:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
2025-11-19T13:59:16.208989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773529043489:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
2025-11-19T13:59:16.209347Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435773529043488:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.212568Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435768000425563:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(32250199ns) {Configure:104(32249317ns) {ProcessConfigured:135(29779480ns) {StartInitialHandshake:157(29778497ns) {DropHandshakes:469(322ns) {DropIncomingHandshake:441(33ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:476(1177ns) {SwitchToState:178(738ns)}}}}}
2025-11-19T13:59:16.213007Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435774297890185:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.214851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435770002922826:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(32172579ns) {Configure:104(32171694ns) {ProcessConfigured:135(31881009ns) {StartInitialHandshake:157(31880478ns) {DropHandshakes:469(417ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:476(923ns) {SwitchToState:178(593ns)}}}}}
2025-11-19T13:59:16.212691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435775141469818:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(30264027ns) {Configure:104(30263573ns) {ProcessConfigured:135(29855932ns) {StartInitialHandshake:157(29855563ns) {DropHandshakes:469(448ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(36ns)} GenerateSessionVirtualId:988(113ns) PrepareNewSessionHandshake:476(993ns) {SwitchToState:178(267ns)}}}}}
2025-11-19T13:59:16.217844Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435772295392937:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.216913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435775452425778:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:16.220625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435775141469828:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1045
2025-11-19T13:59:16.223864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435772120864039:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
2025-11-19T13:59:16.224260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435773529043422:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(47457118ns) {Configure:104(47456406ns) {ProcessConfigured:135(47165260ns) {StartInitialHandshake:157(47164829ns) {DropHandshakes:469(1012ns) {DropIncomingHandshake:441(154ns) DropOutgoingHandshake:456(280ns)} GenerateSessionVirtualId:988(342ns) PrepareNewSessionHandshake:476(708ns) {SwitchToState:178(110ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.224513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435775141469833:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1045
2025-11-19T13:59:16.225394Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435772120864039:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.225456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435772120863974:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.228108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435773529043486:2072] [node 4] ICP01 ready to work
2025-11-19T13:59:16.228261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435772120863975:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(32248725ns) {Configure:104(32247562ns) {ProcessConfigured:135(31933786ns) {StartInitialHandshake:157(31933210ns) {DropHandshakes:469(695ns) {DropIncomingHandshake:441(116ns) DropOutgoingHandshake:456(153ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:476(807ns) {SwitchToState:178(383ns)}}}}}
2025-11-19T13:59:16.228450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435775141469827:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:16.229300Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775141469833:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.232036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435773529043487:2079] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.233146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775452425792:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46651
2025-11-19T13:59:16.233203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775452425793:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46651
2025-11-19T13:59:16.233214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775452425794:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46651
2025-11-19T13:59:16.233225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775452425795:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46651
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.233615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435772295392942:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26480
2025-11-19T13:59:16.233639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435772295392943:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26480
2025-11-19T13:59:16.233824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435770846502463:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16183459ns) {Configure:104(16182233ns) {ProcessConfigured:135(15895337ns) {StartInitialHandshake:157(15894752ns) {DropHandshakes:469(758ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(81ns)} GenerateSessionVirtualId:988(280ns) PrepareNewSessionHandshake:476(615ns) {SwitchToState:178(97ns)}}}}}
2025-11-19T13:59:16.234162Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775452425792:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.234187Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775452425795:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.235925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435775452425718:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17467924ns) {Configure:104(17467194ns) {ProcessConfigured:135(17183351ns) {StartInitialHandshake:157(17182819ns) {DropHandshakes:469(481ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(349ns) PrepareNewSessionHandshake:476(851ns) {SwitchToState:178(481ns)}}}}}
2025-11-19T13:59:16.235970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435775452425718:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.236145Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435772120863973:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16441890ns) {Configure:104(16441347ns) {ProcessConfigured:135(16083057ns) {StartInitialHandshake:157(16082763ns) {DropHandshakes:469(373ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(406ns) PrepareNewSessionHandshake:476(1134ns) {SwitchToState:178(271ns)}}}}}
2025-11-19T13:59:16.240055Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435774297890194:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.240068Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435774297890194:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.241082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775452425799:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46651
2025-11-19T13:59:16.241237Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435774297890194:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:16.241419Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435775452425795:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:16.241531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435775452425718:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.241550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:16.241556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435775452425718:2049] [node 2] ICP22 created new session: [4:7574435775452425800:2049]
2025-11-19T13:59:16.241562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435775452425800:2049] [node 2] ICS09 handshake done sender: [4:7574435775452425795:2116] self: [4:7574435775452425798:0] peer: [2:7574435774297890184:0] socket: 361 qp: -1
2025-11-19T13:59:16.241568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435775452425800:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:16.241600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:16.241960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435774297890182:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.241987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:16.241992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435774297890182:2107] [node 4] ICP22 created new session: [2:7574435774297890205:2107]
2025-11-19T13:59:16.241996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435774297890205:2107] [node 4] ICS09 handshake done sender: [2:7574435774297890194:2112] self: [2:7574435774297890184:0] peer: [4:7574435775452425798:0] socket: 345 qp: -1
2025-11-19T13:59:16.242001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435774297890205:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:16.242010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.242706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435772120864054:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
2025-11-19T13:59:16.242726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435772120864055:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
2025-11-19T13:59:16.243821Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435774297890190:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.243845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435774297890190:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.248789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435773529043486:2072] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(20534536ns) {Configure:104(20533714ns) {ProcessConfigured:135(20174292ns) {StartInitialHandshake:157(20173742ns) {DropHandshakes:469(628ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(141ns)} GenerateSessionVirtualId:988(371ns) PrepareNewSessionHandshake:476(1331ns) {SwitchToState:178(1181ns)}}}}}
2025-11-19T13:59:16.249114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435773529043487:2079] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.249151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435773529043487:2079] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17026627ns) {Configure:104(17026396ns) {ProcessConfigured:135(16675816ns) {StartInitialHandshake:157(16651853ns) {DropHandshakes:469(722ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(119ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:476(671ns) {SwitchToState:178(410ns)}} IncomingHandshake:261(17086ns) {DropIncomingHandshake:441(260ns) PrepareNewSessionHandshake:476(180ns) {SwitchToState:178(143ns)} GenerateSessionVirtualId:988(370ns) IssueIncomingHandshakeReply:193(13933ns) {SwitchToState:178(35ns)}}}}}
2025-11-19T13:59:16.248981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435775141469827:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20471208ns) {Configure:104(20470474ns) {ProcessConfigured:135(20138096ns) {StartInitialHandshake:157(20137347ns) {DropHandshakes:469(502ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(84ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(936ns) {SwitchToState:178(397ns)}}}}}
2025-11-19T13:59:16.247920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435775452425786:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:16.255267Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435774297890190:2110] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:16.255326Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435772120864026:2095] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.255342Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435772120864026:2095] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.256444Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435772120864026:2095] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:16.256736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435774297890188:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.256761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:16.256768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435774297890188:2109] [node 5] ICP22 created new session: [2:7574435774297890209:2109]
2025-11-19T13:59:16.256774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435774297890209:2109] [node 5] ICS09 handshake done sender: [2:7574435774297890190:2110] self: [2:7574435774297890189:0] peer: [5:7574435772120864051:0] socket: 343 qp: -1
2025-11-19T13:59:16.256780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435774297890209:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:16.256790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.256254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773529043501:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
2025-11-19T13:59:16.256300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773529043502:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
2025-11-19T13:59:16.256311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773529043503:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
2025-11-19T13:59:16.258603Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435773529043501:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.260882Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775141469828:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.258722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435773529043421:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.258916Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435773529043488:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:16.259022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435773529043487:2079] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.259037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:16.259045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435773529043487:2079] [node 5] ICP22 created new session: [6:7574435773529043505:2079]
2025-11-19T13:59:16.259050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435773529043505:2079] [node 5] ICS09 handshake done sender: [6:7574435773529043488:2108] self: [6:7574435773529043500:0] peer: [5:7574435772120864025:0] socket: 334 qp: -1
2025-11-19T13:59:16.259056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435773529043505:2079] [node 5] ICS10 traffic start
2025-11-19T13:59:16.259078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:16.260829Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435773529043489:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.260057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775452425805:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46651
2025-11-19T13:59:16.260865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435773529043420:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.260708Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775452425793:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.260735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435775452425717:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.261049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435772120864054:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.261073Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435772120864038:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.261089Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435772120864055:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.261111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435772120863975:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.261120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435772120863973:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.261129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435772120864018:2091] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.261311Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435772295392942:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.261389Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435772295392943:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.261224Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775452425789:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.260192Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435774297890199:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.261245Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435775452425789:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.260200Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435774297890199:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.261942Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435775452425789:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:16.262016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435775452425778:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.262045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:16.262056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435775452425778:2107] [node 5] ICP22 created new session: [4:7574435775452425808:2107]
2025-11-19T13:59:16.262062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435775452425808:2107] [node 5] ICS09 handshake done sender: [4:7574435775452425789:2112] self: [4:7574435775452425787:0] peer: [5:7574435772120864060:0] socket: 358 qp: -1
2025-11-19T13:59:16.262067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435775452425808:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:16.262074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.262894Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435774297890199:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:16.264073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435774297890197:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.264091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:16.264096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435774297890197:2113] [node 6] ICP22 created new session: [2:7574435774297890213:2113]
2025-11-19T13:59:16.264101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435774297890213:2113] [node 6] ICS09 handshake done sender: [2:7574435774297890199:2114] self: [2:7574435774297890198:0] peer: [6:7574435773529043504:0] socket: 367 qp: -1
2025-11-19T13:59:16.264106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435774297890213:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:16.264113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:16.262791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773529043509:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
2025-11-19T13:59:16.264005Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435773529043501:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:16.265467Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775141469829:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.265481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435775141469829:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.265767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435775141469840:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1045
2025-11-19T13:59:16.266110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435773529043421:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.266130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:16.266138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435773529043421:2049] [node 2] ICP22 created new session: [6:7574435773529043510:2049]
2025-11-19T13:59:16.266144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435773529043510:2049] [node 2] ICS09 handshake done sender: [6:7574435773529043501:2117] self: [6:7574435773529043504:0] peer: [2:7574435774297890198:0] socket: 369 qp: -1
2025-11-19T13:59:16.266151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435773529043510:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:16.266157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:16.266791Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775452425805:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.266659Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435773529043502:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.266692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435773529043486:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.265215Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435772295392930:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.265232Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435772295392930:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.265394Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435772295392926:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.265402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435772295392926:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.265440Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435772295392932:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.265445Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435772295392932:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.265352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435772120864057:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
2025-11-19T13:59:16.269474Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775141469840:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.269568Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775452425794:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.269609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435775452425719:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.270030Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435772120864018:2091] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15286) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435772120864055:2118] held: no
2025-11-19T13:59:16.270060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435772120864024:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.270071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:16.270076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435772120864024:2094] [node 6] ICP22 created new session: [5:7574435772120864061:2094]
2025-11-19T13:59:16.270082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435772120864061:2094] [node 6] ICS09 handshake done sender: [5:7574435772120864026:2095] self: [5:7574435772120864025:0] peer: [6:7574435773529043500:0] socket: 333 qp: -1
2025-11-19T13:59:16.270089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435772120864061:2094] [node 6] ICS10 traffic start
2025-11-19T13:59:16.270098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:16.269796Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775141469824:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.269803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435775141469824:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.270784Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435775141469824:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:16.270871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435775141469822:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.270888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:16.270893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435775141469822:2108] [node 4] ICP22 created new session: [3:7574435775141469844:2108]
2025-11-19T13:59:16.270899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435775141469844:2108] [node 4] ICS09 handshake done sender: [3:7574435775141469824:2109] self: [3:7574435775141469823:0] peer: [4:7574435775452425811:0] socket: 342 qp: -1
2025-11-19T13:59:16.270904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435775141469844:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:16.270921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:16.271282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435775452425719:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5380) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435775452425794:2115] held: no
2025-11-19T13:59:16.272430Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775452425803:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.272442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435775452425803:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.272486Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435774297890203:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.272496Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435774297890203:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.271943Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435772295392930:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:16.272688Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435772295392932:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:16.274693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435772295392923:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.274713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:16.274722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435772295392923:2108] [node 5] ICP22 created new session: [1:7574435772295392948:2108]
2025-11-19T13:59:16.274728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435772295392948:2108] [node 5] ICS09 handshake done sender: [1:7574435772295392930:2112] self: [1:7574435772295392929:0] peer: [5:7574435772120864059:0] socket: 338 qp: -1
2025-11-19T13:59:16.274736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435772295392948:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:16.274744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.277277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435774297890215:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63039
2025-11-19T13:59:16.277385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435774297890188:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(20616905ns) {HandleHandshakeStatus:329(20615693ns) {DropIncomingHandshake:441(8005ns) DropOutgoingHandshake:456(238ns) SwitchToState:178(243ns) ProcessPendingSessionEvents:495(11687934ns) {ForwardSessionEventToSession:612(11675953ns) ForwardSessionEventToSession:612(1686ns) ForwardSessionEventToSession:612(1136ns) ForwardSessionEventToSession:612(500ns) ForwardSessionEventToSession:612(727ns) ForwardSessionEventToSession:612(817ns) ForwardSessionEventToSession:612(581ns) ForwardSessionEventToSession:612(917ns) ForwardSessionEventToSession:612(676ns) ForwardSessionEventToSession:612(972ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.278136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775452425812:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46651
2025-11-19T13:59:16.277087Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435772120864039:2106] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:16.277898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435772120863974:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16374) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435772120864039:2106] held: no
2025-11-19T13:59:16.278011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435772120864063:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.279685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773529043513:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.280338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435772120864064:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
2025-11-19T13:59:16.282264Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435772120864038:2105] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.280110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435775141469847:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1045
2025-11-19T13:59:16.280684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435770846502463:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.281918Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435775141469829:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:16.282456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435775141469818:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.282467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:16.282476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435775141469818:2106] [node 5] ICP22 created new session: [3:7574435775141469851:2106]
2025-11-19T13:59:16.282482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435775141469851:2106] [node 5] ICS09 handshake done sender: [3:7574435775141469829:2112] self: [3:7574435775141469819:0] peer: [5:7574435772120864058:0] socket: 353 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.282905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435772120863974:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.282920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:16.282925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435772120863974:2049] [node 2] ICP22 created new session: [5:7574435772120864068:2049]
2025-11-19T13:59:16.282929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435772120864068:2049] [node 2] ICS09 handshake done sender: [5:7574435772120864039:2106] self: [5:7574435772120864051:0] peer: [2:7574435774297890189:0] socket: 347 qp: -1
2025-11-19T13:59:16.282935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435772120864068:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:16.282953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:16.283060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435772120864067:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14441
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.283166Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435774297890203:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:16.283283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435774297890217:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63039
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.283501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773529043514:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
2025-11-19T13:59:16.282488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435775141469851:2106] [node 5] ICS10 traffic start
2025-11-19T13:59:16.282496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:16.282665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435775141469849:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1045
2025-11-19T13:59:16.282676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435775141469850:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1045
2025-11-19T13:59:16.285374Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435775141469840:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:16.280949Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435775452425794:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:16.281659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775452425813:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46651
2025-11-19T13:59:16.285763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435770002922826:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.285781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:16.285788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435770002922826:2049] [node 3] ICP22 created new session: [2:7574435774297890218:2049]
2025-11-19T13:59:16.285793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435774297890218:2049] [node 3] ICS09 handshake done sender: [2:7574435774297890203:2116] self: [2:7574435774297890186:0] peer: [3:7574435775141469842:0] socket: 376 qp: -1
2025-11-19T13:59:16.285799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435774297890218:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:16.285822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:16.286145Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435773529043489:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:16.286173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435775452425719:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.286194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:16.286201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435775452425719:2050] [node 3] ICP22 created new session: [4:7574435775452425814:2050]
2025-11-19T13:59:16.286207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435775452425814:2050] [node 3] ICS09 handshake done sender: [4:7574435775452425794:2115] self: [4:7574435775452425811:0] peer: [3:7574435775141469823:0] socket: 360 qp: -1
2025-11-19T13:59:16.286214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435775452425814:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:16.286226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:16.286266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435773529043420:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.286284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:16.286289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435773529043420:2048] [node 1] ICP22 created new session: [6:7574435773529043515:2048]
2025-11-19T13:59:16.286294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435773529043515:2048] [node 1] ICS09 handshake done sender: [6:7574435773529043489:2109] self: [6:7574435773529043508:0] peer: [1:7574435772295392931:0] socket: 352 qp: -1
2025-11-19T13:59:16.286298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435773529043515:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:16.286308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:16.286644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435772120863973:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.286661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:16.286665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435772120863973:2048] [node 1] ICP22 created new session: [5:7574435772120864070:2048]
2025-11-19T13:59:16.286669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435772120864070:2048] [node 1] ICS09 handshake done sender: [5:7574435772120864038:2105] self: [5:7574435772120864059:0] peer: [1:7574435772295392929:0] socket: 341 qp: -1
2025-11-19T13:59:16.286675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435772120864070:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:16.286680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:16.287137Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435773529043514:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.287168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435773529043422:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.285812Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435772295392926:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:16.287297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435772295392953:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26480
2025-11-19T13:59:16.286872Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775141469850:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.286885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435770846502463:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.286894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:16.286899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435770846502463:2049] [node 2] ICP22 created new session: [3:7574435775141469854:2049]
2025-11-19T13:59:16.286904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435775141469854:2049] [node 2] ICS09 handshake done sender: [3:7574435775141469840:2117] self: [3:7574435775141469842:0] peer: [2:7574435774297890186:0] socket: 377 qp: -1
2025-11-19T13:59:16.286909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435775141469854:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:16.286915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:16.287049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435775141469818:2106] [node 5] ICP18 handshake (actor [3:7574435775141469850:2120]) is too old
2025-11-19T13:59:16.287429Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435772120864024:2094] [node 6] ICP35 event processing took too much time PendingConnection:278(17339296ns) {HandleHandshakeStatus:329(17338935ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(103ns) SwitchToState:178(81ns) ProcessPendingSessionEvents:495(14568180ns) {ForwardSessionEventToSession:612(14554279ns) ForwardSessionEventToSession:612(1927ns) ForwardSessionEventToSession:612(1118ns) ForwardSessionEventToSession:612(804ns) ForwardSessionEventToSession:612(1021ns) ForwardSessionEventToSession:612(1091ns) ForwardSessionEventToSession:612(518ns) ForwardSessionEventToSession:612(496ns) ForwardSessionEventToSession:612(549ns) ForwardSessionEventToSession:612(1264ns)}}}
2025-11-19T13:59:16.287446Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435772120864055:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:16.288039Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435772120864054:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:16.288151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435772120864018:2091] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.288160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:16.288166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435772120864018:2091] [node 4] ICP22 created new session: [5:7574435772120864072:2091]
2025-11-19T13:59:16.288171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435772120864072:2091] [node 4] ICS09 handshake done sender: [5:7574435772120864055:2118] self: [5:7574435772120864060:0] peer: [4:7574435775452425787:0] socket: 368 qp: -1
2025-11-19T13:59:16.288176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435772120864072:2091] [node 4] ICS10 traffic start
2025-11-19T13:59:16.288181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:16.288107Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435775452425793:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:16.288244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435775452425717:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.288261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:16.288267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435775452425717:2048] [node 1] ICP22 created new session: [4:7574435775452425816:2048]
2025-11-19T13:59:16.288272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435775452425816:2048] [node 1] ICS09 handshake done sender: [4:7574435775452425793:2114] self: [4:7574435775452425806:0] peer: [1:7574435772295392925:0] socket: 359 qp: -1
2025-11-19T13:59:16.288277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435775452425816:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:16.288295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:16.288388Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775141469847:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.288421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435770846502462:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.288528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435772120863975:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.288539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:16.288543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435772120863975:2050] [node 3] ICP22 created new session: [5:7574435772120864074:2050]
2025-11-19T13:59:16.288547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435772120864074:2050] [node 3] ICS09 handshake done sender: [5:7574435772120864054:2117] self: [5:7574435772120864058:0] peer: [3:7574435775141469819:0] socket: 354 qp: -1
2025-11-19T13:59:16.288551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435772120864074:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:16.288555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:16.289103Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435775452425803:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:16.289194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435775452425786:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.289206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:16.289210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435775452425786:2109] [node 6] ICP22 created new session: [4:7574435775452425819:2109]
2025-11-19T13:59:16.289216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435775452425819:2109] [node 6] ICS09 handshake done sender: [4:7574435775452425803:2119] self: [4:7574435775452425802:0] peer: [6:7574435773529043512:0] socket: 370 qp: -1
2025-11-19T13:59:16.289220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435775452425819:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:16.289115Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435772295392939:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.289124Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435772295392939:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.289149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435772295392924:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.289164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:16.289171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435772295392924:2109] [node 6] ICP22 created new session: [1:7574435772295392954:2109]
2025-11-19T13:59:16.289176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435772295392954:2109] [node 6] ICS09 handshake done sender: [1:7574435772295392932:2113] self: [1:7574435772295392931:0] peer: [6:7574435773529043508:0] socket: 339 qp: -1
2025-11-19T13:59:16.289181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435772295392954:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:16.289198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:16.289225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:16.289054Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775141469838:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.289068Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435775141469838:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.293269Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435774297890215:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.293331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435770002922826:2049] [node 3] ICP18 handshake (actor [2:7574435774297890215:2117]) is too old
2025-11-19T13:59:16.292597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435772295392922:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.292617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:16.292622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435772295392922:2107] [node 4] ICP22 created new session: [1:7574435772295392956:2107]
2025-11-19T13:59:16.292627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435772295392956:2107] [node 4] ICS09 handshake done sender: [1:7574435772295392926:2110] self: [1:7574435772295392925:0] peer: [4:7574435775452425806:0] socket: 337 qp: -1
2025-11-19T13:59:16.292633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435772295392956:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:16.292651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.298304Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435775141469838:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:16.297979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773529043518:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
2025-11-19T13:59:16.298029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773529043519:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34401
2025-11-19T13:59:16.298825Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435773529043486:2072] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15286) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435773529043502:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.299643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435775141469858:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1045
2025-11-19T13:59:16.299814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435775141469827:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.299838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:16.299843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435775141469827:2110] [node 6] ICP22 created new session: [3:7574435775141469859:2110]
2025-11-19T13:59:16.299849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435775141469859:2110] [node 6] ICS09 handshake done sender: [3:7574435775141469838:2116] self: [3:7574435775141469834:0] peer: [6:7574435773529043517:0] socket: 396 qp: -1
2025-11-19T13:59:16.299855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435775141469859:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:16.299865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:16.301100Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435774297890217:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.301149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435770002922825:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.301516Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435772295392939:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:16.301717Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435775141469847:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:16.301814Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435773529043502:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:16.301860Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435773529043422:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5380) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435773529043514:2122] held: no
2025-11-19T13:59:16.302004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435770846502462:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.302034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:16.302039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435770846502462:2048] [node 1] ICP22 created new session: [3:7574435775141469861:2048]
2025-11-19T13:59:16.302044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435775141469861:2048] [node 1] ICS09 handshake done sender: [3:7574435775141469847:2118] self: [3:7574435775141469856:0] peer: [1:7574435772295392928:0] socket: 387 qp: -1
2025-11-19T13:59:16.302049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435775141469861:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:16.302058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:16.306208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435772295392924:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(33401168ns) {HandleHandshakeStatus:329(33400209ns) {DropIncomingHandshake:441(3904ns) DropOutgoingHandshake:456(87ns) SwitchToState:178(62ns) ProcessPendingSessionEvents:495(10077180ns) {ForwardSessionEventToSession:612(10064473ns) ForwardSessionEventToSession:612(1465ns) ForwardSessionEventToSession:612(943ns) ForwardSessionEventToSession:612(1112ns) ForwardSessionEventToSession:612(875ns) ForwardSessionEventToSession:612(869ns) ForwardSessionEventToSession:612(810ns) ForwardSessionEventToSession:612(597ns) ForwardSessionEventToSession:612(580ns) ForwardSessionEventToSession:612(988ns)}}}
2025-11-19T13:59:16.306227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435768000425564:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.306244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:16.306251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435768000425564:2049] [node 3] ICP22 created new session: [1:7574435772295392959:2049]
2025-11-19T13:59:16.306255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435772295392959:2049] [node 3] ICS09 handshake done sender: [1:7574435772295392939:2115] self: [1:7574435772295392928:0] peer: [3:7574435775141469856:0] socket: 385 qp: -1
2025-11-19T13:59:16.306261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435772295392959:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:16.306268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:16.306462Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435772295392940:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.306469Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435772295392940:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.307488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435773529043486:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.307503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:16.307511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435773529043486:2072] [node 4] ICP22 created new session: [6:7574435773529043520:2072]
2025-11-19T13:59:16.307516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435773529043520:2072] [node 4] ICS09 handshake done sender: [6:7574435773529043502:2118] self: [6:7574435773529043512:0] peer: [4:7574435775452425802:0] socket: 374 qp: -1
2025-11-19T13:59:16.307521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435773529043520:2072] [node 4] ICS10 traffic start
2025-11-19T13:59:16.307528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:16.307601Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435773529043514:2122] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:16.310454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435773529043422:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.310469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:16.310474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435773529043422:2050] [node 3] ICP22 created new session: [6:7574435773529043522:2050]
2025-11-19T13:59:16.310479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435773529043522:2050] [node 3] ICS09 handshake done sender: [6:7574435773529043514:2122] self: [6:7574435773529043517:0] peer: [3:7574435775141469834:0] socket: 398 qp: -1
2025-11-19T13:59:16.310484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435773529043522:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:16.310493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.311444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435774297890221:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63039
2025-11-19T13:59:16.311710Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435772295392940:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:16.311843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435768000425563:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.311864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:16.311870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435768000425563:2048] [node 2] ICP22 created new session: [1:7574435772295392962:2048]
2025-11-19T13:59:16.311875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435772295392962:2048] [node 2] ICS09 handshake done sender: [1:7574435772295392940:2116] self: [1:7574435772295392936:0] peer: [2:7574435774297890220:0] socket: 386 qp: -1
2025-11-19T13:59:16.311881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435772295392962:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:16.311888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:16.312111Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435774297890217:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:16.312209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435770002922825:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.312224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:16.312229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435770002922825:2048] [node 1] ICP22 created new session: [2:7574435774297890222:2048]
2025-11-19T13:59:16.312235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435774297890222:2048] [node 1] ICS09 handshake done sender: [2:7574435774297890217:2118] self: [2:7574435774297890220:0] peer: [1:7574435772295392936:0] socket: 397 qp: -1
2025-11-19T13:59:16.312241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435774297890222:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:16.312246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:16.322977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435772295392923:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:16.331040Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:16.331124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574435774297890222:2048] [node 1] ICS07 socket disconnect 397 reason# EndOfStream
2025-11-19T13:59:16.331131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435774297890222:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:16.331163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:16.331177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435774297890222:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:16.331832Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:16.331906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435775141469861:2048] [node 1] ICS01 socket: 387 reason# EndOfStream
2025-11-19T13:59:16.331913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435770846502462:2048] [node 1] ICP30 unregister session Session# [3:7574435775141469861:2048] VirtualId# [3:7574435775141469856:0]
2025-11-19T13:59:16.331919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435775141469861:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:16.331946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:16.331968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:16.336476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435770002922825:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28775) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:16.336491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435774297890222:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:16.336888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435770002922825:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28775) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:16.336899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435774297890222:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:16.336905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435770002922825:2048] [node 1] ICP30 unregister session Session# [2:7574435774297890222:2048] VirtualId# [2:7574435774297890220:0]
2025-11-19T13:59:16.336921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:16.336926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574435770002922825:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28775) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:16.336942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28775) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:16.343969Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:16.344055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435772120864070:2048] [node 1] ICS07 socket disconnect 341 reason# EndOfStream
2025-11-19T13:59:16.344061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435772120864070:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:16.344114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:16.344136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435772120864070:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:16.344624Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:16.344680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435775452425816:2048] [node 1] ICS01 socket: 359 reason# EndOfStream
2025-11-19T13:59:16.344685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435775452425717:2048] [node 1] ICP30 unregister session Session# [4:7574435775452425816:2048] VirtualId# [4:7574435775452425806:0]
2025-11-19T13:59:16.344689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435775452425816:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:16.344713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:16.344732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:16.344350Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:16.344412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435773529043515:2048] [node 1] ICS01 socket: 352 reason# EndOfStream
2025-11-19T13:59:16.344418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435773529043420:2048] [node 1] ICP30 unregister session Session# [6:7574435773529043515:2048] VirtualId# [6:7574435773529043508:0]
2025-11-19T13:59:16.344424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435773529043515:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:16.344449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:16.344472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:16.347514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435772120863973:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28775) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:16.347527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435772120864070:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:16.347532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435772120863973:2048] [node 1] ICP30 unregister session Session# [5:7574435772120864070:2048] VirtualId# [5:7574435772120864059:0]
2025-11-19T13:59:16.347546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:16.347551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435772120863973:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28775) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:16.347553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28775) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:16.352547Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:16.352628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435775141469854:2049] [node 2] ICS01 socket: 377 reason# EndOfStream
2025-11-19T13:59:16.352636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435770846502463:2049] [node 2] ICP30 unregister session Session# [3:7574435775141469854:2049] VirtualId# [3:7574435775141469842:0]
2025-11-19T13:59:16.352641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435775141469854:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:16.352674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:16.352692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:16.356511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435772120863973:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:16.362161Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:16.362254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435773529043510:2049] [node 2] ICS01 socket: 369 reason# EndOfStream
2025-11-19T13:59:16.362262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435773529043421:2049] [node 2] ICP30 unregister session Session# [6:7574435773529043510:2049] VirtualId# [6:7574435773529043504:0]
2025-11-19T13:59:16.362268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435773529043510:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:16.362325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:16.362349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:16.362431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:16.362495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435775452425800:2049] [node 2] ICS01 socket: 361 reason# EndOfStream
2025-11-19T13:59:16.362500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435775452425718:2049] [node 2] ICP30 unregister session Session# [4:7574435775452425800:2049] VirtualId# [4:7574435775452425798:0]
2025-11-19T13:59:16.362504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435775452425800:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:16.362545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:16.362567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:16.366749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:16.366830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435772120864068:2049] [node 2] ICS01 socket: 347 reason# EndOfStream
2025-11-19T13:59:16.366836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435772120863974:2049] [node 2] ICP30 unregister session Session# [5:7574435772120864068:2049] VirtualId# [5:7574435772120864051:0]
2025-11-19T13:59:16.366842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435772120864068:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:16.366887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:16.366909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:16.378445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:16.378530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435773529043522:2050] [node 3] ICS01 socket: 398 reason# EndOfStream
2025-11-19T13:59:16.378548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435773529043422:2050] [node 3] ICP30 unregister session Session# [6:7574435773529043522:2050] VirtualId# [6:7574435773529043517:0]
2025-11-19T13:59:16.378555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435773529043522:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:16.378606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:16.378628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:16.378701Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:16.378767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435772120864074:2050] [node 3] ICS07 socket disconnect 354 reason# EndOfStream
2025-11-19T13:59:16.378771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435772120864074:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:16.378801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:16.378820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435772120864074:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:16.382088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435772120863975:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5380) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:16.382102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435772120864074:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:16.382107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435772120863975:2050] [node 3] ICP30 unregister session Session# [5:7574435772120864074:2050] VirtualId# [5:7574435772120864058:0]
2025-11-19T13:59:16.382120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:16.382125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435772120863975:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5380) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:16.382128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5380) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:16.390631Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:16.390711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435773529043520:2072] [node 4] ICS01 socket: 374 reason# EndOfStream
2025-11-19T13:59:16.390718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435773529043486:2072] [node 4] ICP30 unregister session Session# [6:7574435773529043520:2072] VirtualId# [6:7574435773529043512:0]
2025-11-19T13:59:16.390724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435773529043520:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:16.390774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:16.390795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:16.515912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435773017641984:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:16.515932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435773017641985:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:16.576219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435771901526616:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:16.576241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435771901526617:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:16.624151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435775740297464:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:16.624170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435775740297465:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:16.671950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435775732952511:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:16.671964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435775732952512:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:16.671970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435775732952513:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:16.720075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435774882486199:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:16.720089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435774882486200:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:16.720093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435774882486201:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:16.796194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435773484288809:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:16.796215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435773484288810:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:16.796219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435773484288811:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:16.827998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435775732952561:2089] [node 5] ICP01 ready to work
2025-11-19T13:59:16.828123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435775732952564:2094] [node 6] ICP01 ready to work
2025-11-19T13:59:16.828211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435773484288828:2059] [node 5] ICP01 ready to work
2025-11-19T13:59:16.828317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435773484288829:2064] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.829516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435773017642051:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.835232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435775740297524:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40245
2025-11-19T13:59:16.835434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435774882486235:2063] [node 4] ICP01 ready to work
2025-11-19T13:59:16.835516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435774882486237:2073] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.839343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435771901526675:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35677
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.843873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435775740297525:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40245
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.847232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435775740297526:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40245
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.848467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435771901526681:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:16.848565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435771901526682:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:16.849139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435771901526685:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:16.849999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435771901526680:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35677
2025-11-19T13:59:16.851253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435775740297527:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40245
2025-11-19T13:59:16.851425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435775740297531:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:16.851509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435775740297532:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:16.851574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435775740297533:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:16.859853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435773017642051:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(30254181ns) {Configure:104(30253416ns) {ProcessConfigured:135(30012230ns) {StartInitialHandshake:157(30011580ns) {DropHandshakes:469(362ns) {DropIncomingHandshake:441(31ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(75ns) PrepareNewSessionHandshake:476(1487ns) {SwitchToState:178(1024ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.880292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773484288887:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-11-19T13:59:16.880316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773484288888:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-11-19T13:59:16.881558Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435775740297464:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(29935792ns) {Configure:104(29935047ns) {ProcessConfigured:135(29576941ns) {StartInitialHandshake:157(29575810ns) {DropHandshakes:469(519ns) {DropIncomingHandshake:441(101ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(181ns) PrepareNewSessionHandshake:476(1345ns) {SwitchToState:178(676ns)}}}}}
2025-11-19T13:59:16.898199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435773017642056:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:16.898308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435773017642057:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.919844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435774882486277:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-19T13:59:16.917191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435773017642058:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35651
2025-11-19T13:59:16.923783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435773017642059:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35651
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.923883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435773017642060:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35651
2025-11-19T13:59:16.923914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435773017642061:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35651
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.925519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435774882486278:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-19T13:59:16.925728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435774882486279:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-19T13:59:16.925757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435774882486280:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-19T13:59:16.926231Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435773017642058:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.929849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435771901526697:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35677
2025-11-19T13:59:16.934564Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435774882486280:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.934604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435774882486199:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.936282Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435773017642055:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.936294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435773017642055:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.937049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435774882486282:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-19T13:59:16.937164Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435773017642055:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:16.937369Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435774882486280:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:16.937436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435774882486199:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.937456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:16.937463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435774882486199:2048] [node 1] ICP22 created new session: [5:7574435774882486283:2048]
2025-11-19T13:59:16.937469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435774882486283:2048] [node 1] ICS09 handshake done sender: [5:7574435774882486280:2118] self: [5:7574435774882486281:0] peer: [1:7574435773017642052:0] socket: 371 qp: -1
2025-11-19T13:59:16.937476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435774882486283:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:16.937487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:16.938765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435773017642051:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.938787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:16.938793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435773017642051:2109] [node 5] ICP22 created new session: [1:7574435773017642069:2109]
2025-11-19T13:59:16.938799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435773017642069:2109] [node 5] ICS09 handshake done sender: [1:7574435773017642055:2112] self: [1:7574435773017642052:0] peer: [5:7574435774882486281:0] socket: 370 qp: -1
2025-11-19T13:59:16.938805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435773017642069:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:16.938812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:16.939286Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775740297524:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.939361Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775740297525:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.940649Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435771901526680:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.940719Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435771901526675:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.940743Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435771901526697:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.940814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435771901526616:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.940988Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435773484288888:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.941021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435773484288810:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.941991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775732952585:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29963
2025-11-19T13:59:16.942052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775732952586:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29963
2025-11-19T13:59:16.942265Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435771901526689:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.942278Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435771901526689:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.943092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773484288890:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-11-19T13:59:16.939517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435775740297464:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.940038Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775740297527:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.942158Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775740297526:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.948152Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775732952585:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.948189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435775732952513:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.948360Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435771901526689:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:16.950310Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775740297537:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.950324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435775740297537:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.951102Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435773484288887:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.951319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775732952592:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29963
2025-11-19T13:59:16.951435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435771901526682:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.951456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:16.951462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435771901526682:2109] [node 6] ICP22 created new session: [2:7574435771901526702:2109]
2025-11-19T13:59:16.951468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435771901526702:2109] [node 6] ICS09 handshake done sender: [2:7574435771901526689:2114] self: [2:7574435771901526688:0] peer: [6:7574435773484288889:0] socket: 356 qp: -1
2025-11-19T13:59:16.951476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435771901526702:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:16.951487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:16.951898Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435773484288888:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:16.952057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435773484288829:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.952070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435773484288810:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.952081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:16.952087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435773484288810:2049] [node 2] ICP22 created new session: [6:7574435773484288892:2049]
2025-11-19T13:59:16.952091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435773484288892:2049] [node 2] ICS09 handshake done sender: [6:7574435773484288888:2116] self: [6:7574435773484288889:0] peer: [2:7574435771901526688:0] socket: 357 qp: -1
2025-11-19T13:59:16.952097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435773484288892:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:16.952104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:16.953098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435773017642061:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.953141Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435773017642045:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.953144Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435773017642045:2107] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.952779Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775732952577:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.952789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435775732952577:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.953533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775732952593:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29963
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.956309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773484288895:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-11-19T13:59:16.956685Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435775740297537:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:16.956869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435775740297531:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.956890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:16.956897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435775740297531:2111] [node 4] ICP22 created new session: [3:7574435775740297553:2111]
2025-11-19T13:59:16.956903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435775740297553:2111] [node 4] ICS09 handshake done sender: [3:7574435775740297537:2115] self: [3:7574435775740297536:0] peer: [4:7574435775732952591:0] socket: 359 qp: -1
2025-11-19T13:59:16.956908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435775740297553:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:16.956916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:16.957142Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435775732952577:2109] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:16.956773Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435773017642059:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.956808Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435773017642060:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.957683Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435773017642047:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.957698Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435773017642047:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.958130Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435774882486278:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.958160Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435774882486279:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.958178Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435774882486277:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.958238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435774882486200:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.958249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435774882486235:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.960352Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435773017642045:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:16.959994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435775732952564:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.960012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:16.960019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435775732952564:2094] [node 6] ICP22 created new session: [4:7574435775732952595:2094]
2025-11-19T13:59:16.960028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435775732952595:2094] [node 6] ICS09 handshake done sender: [4:7574435775732952577:2109] self: [4:7574435775732952576:0] peer: [6:7574435773484288891:0] socket: 344 qp: -1
2025-11-19T13:59:16.960034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435775732952595:2094] [node 6] ICS10 traffic start
2025-11-19T13:59:16.960042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:16.960201Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775732952575:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.960205Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435775732952575:2108] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.962033Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435771901526687:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.962047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435771901526687:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.962735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435771901526705:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35677
2025-11-19T13:59:16.963095Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435773484288887:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:16.963204Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435771901526687:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:16.963295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435771901526681:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.963313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:16.963320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435771901526681:2108] [node 5] ICP22 created new session: [2:7574435771901526707:2108]
2025-11-19T13:59:16.963325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435771901526707:2108] [node 5] ICS09 handshake done sender: [2:7574435771901526687:2113] self: [2:7574435771901526686:0] peer: [5:7574435774882486287:0] socket: 355 qp: -1
2025-11-19T13:59:16.963330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435771901526707:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:16.963337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.963618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435773484288829:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.963630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:16.963636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435773484288829:2064] [node 4] ICP22 created new session: [6:7574435773484288897:2064]
2025-11-19T13:59:16.963641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435773484288897:2064] [node 4] ICS09 handshake done sender: [6:7574435773484288887:2115] self: [6:7574435773484288891:0] peer: [4:7574435775732952576:0] socket: 353 qp: -1
2025-11-19T13:59:16.963646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435773484288897:2064] [node 4] ICS10 traffic start
2025-11-19T13:59:16.963651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:16.963836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773484288896:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-11-19T13:59:16.963941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435775740297552:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40245
2025-11-19T13:59:16.966342Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435775732952585:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:16.967858Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435773017642047:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:16.968434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435775740297555:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40245
2025-11-19T13:59:16.969759Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775740297552:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.969787Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435775740297524:2107] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.969808Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435775732952513:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13725) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435775732952585:2113] held: no
2025-11-19T13:59:16.969832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435775732952513:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.969844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:16.969851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435775732952513:2050] [node 3] ICP22 created new session: [4:7574435775732952599:2050]
2025-11-19T13:59:16.969856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435775732952599:2050] [node 3] ICS09 handshake done sender: [4:7574435775732952585:2113] self: [4:7574435775732952591:0] peer: [3:7574435775740297536:0] socket: 366 qp: -1
2025-11-19T13:59:16.969862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435775732952599:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:16.969868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:16.969965Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775732952593:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.969994Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775732952586:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.970273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435775732952564:2094] [node 6] ICP18 handshake (actor [4:7574435775732952593:2118]) is too old
2025-11-19T13:59:16.970360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775732952601:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29963
2025-11-19T13:59:16.970024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435773017641985:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.970044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:16.970049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435773017641985:2049] [node 3] ICP22 created new session: [1:7574435773017642077:2049]
2025-11-19T13:59:16.970054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435773017642077:2049] [node 3] ICS09 handshake done sender: [1:7574435773017642045:2107] self: [1:7574435773017642044:0] peer: [3:7574435775740297547:0] socket: 336 qp: -1
2025-11-19T13:59:16.970059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435773017642077:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:16.970066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:16.970164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435773017641984:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.970172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:16.970176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435773017641984:2048] [node 2] ICP22 created new session: [1:7574435773017642079:2048]
2025-11-19T13:59:16.970180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435773017642079:2048] [node 2] ICS09 handshake done sender: [1:7574435773017642047:2108] self: [1:7574435773017642046:0] peer: [2:7574435771901526699:0] socket: 337 qp: -1
2025-11-19T13:59:16.970183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435773017642079:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:16.970188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.973509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435774882486200:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23947) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435774882486279:2117] held: no
2025-11-19T13:59:16.973631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435774882486289:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-19T13:59:16.973508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435775740297464:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.973540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:16.973551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435775740297464:2048] [node 1] ICP22 created new session: [3:7574435775740297557:2048]
2025-11-19T13:59:16.973559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435775740297557:2048] [node 1] ICS09 handshake done sender: [3:7574435775740297524:2107] self: [3:7574435775740297547:0] peer: [1:7574435773017642044:0] socket: 339 qp: -1
2025-11-19T13:59:16.973567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435775740297557:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:16.973575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:16.973742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435775740297465:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.974776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435774882486290:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-19T13:59:16.974867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435774882486291:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-19T13:59:16.974966Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435774882486279:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.976889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435773017642081:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35651
2025-11-19T13:59:16.979034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435774882486200:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.979057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:16.979065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435774882486200:2049] [node 2] ICP22 created new session: [5:7574435774882486293:2049]
2025-11-19T13:59:16.979072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435774882486293:2049] [node 2] ICS09 handshake done sender: [5:7574435774882486279:2117] self: [5:7574435774882486287:0] peer: [2:7574435771901526686:0] socket: 369 qp: -1
2025-11-19T13:59:16.979079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435774882486293:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:16.979098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.976556Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435775732952575:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:16.977135Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775732952601:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.977195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435775732952561:2089] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.977211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:16.977218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435775732952561:2089] [node 5] ICP22 created new session: [4:7574435775732952605:2089]
2025-11-19T13:59:16.977224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435775732952605:2089] [node 5] ICS09 handshake done sender: [4:7574435775732952575:2108] self: [4:7574435775732952569:0] peer: [5:7574435774882486288:0] socket: 343 qp: -1
2025-11-19T13:59:16.977231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435775732952605:2089] [node 5] ICS10 traffic start
2025-11-19T13:59:16.977240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:16.979298Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435773484288896:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.979372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773484288899:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-11-19T13:59:16.979392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435773484288809:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.981349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435773017642063:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.983502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435773484288901:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-11-19T13:59:16.981364Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435773017642063:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.983674Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435773017642063:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:16.983706Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435774882486277:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:16.983998Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435773484288896:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:16.984093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435773484288809:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.984114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:16.984122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435773484288809:2048] [node 1] ICP22 created new session: [6:7574435773484288902:2048]
2025-11-19T13:59:16.984130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435773484288902:2048] [node 1] ICS09 handshake done sender: [6:7574435773484288896:2119] self: [6:7574435773484288900:0] peer: [1:7574435773017642062:0] socket: 388 qp: -1
2025-11-19T13:59:16.984138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435773484288902:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:16.984150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:16.985145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435773017642056:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:16.985168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:16.985175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435773017642056:2110] [node 6] ICP22 created new session: [1:7574435773017642084:2110]
2025-11-19T13:59:16.985180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435773017642084:2110] [node 6] ICS09 handshake done sender: [1:7574435773017642063:2117] self: [1:7574435773017642062:0] peer: [6:7574435773484288900:0] socket: 384 qp: -1
2025-11-19T13:59:16.985185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435773017642084:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:16.985194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.987970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775732952604:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29963
2025-11-19T13:59:16.988036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435775732952512:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.993239Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435771901526616:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17291) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435771901526675:2106] held: no
2025-11-19T13:59:16.993414Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435775732952604:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.993459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435775732952511:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:16.993435Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435771901526693:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.993448Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435771901526693:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:16.993598Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435773017642081:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.993642Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435773017642065:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:16.993646Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435773017642065:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:16.994407Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435771901526675:2106] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:16.995345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435771901526616:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:16.995360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:16.995369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435771901526616:2048] [node 1] ICP22 created new session: [2:7574435771901526712:2048]
2025-11-19T13:59:16.995377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435771901526712:2048] [node 1] ICS09 handshake done sender: [2:7574435771901526675:2106] self: [2:7574435771901526699:0] peer: [1:7574435773017642046:0] socket: 338 qp: -1
2025-11-19T13:59:16.995386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435771901526712:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:16.995396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:16.995568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435771901526710:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35677
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.001456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775732952609:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29963
2025-11-19T13:59:17.001478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435775732952610:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29963
2025-11-19T13:59:17.001685Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435773017642065:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:17.001911Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435775732952604:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:17.001988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435775732952511:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.002005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:17.002011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435775732952511:2048] [node 1] ICP22 created new session: [4:7574435775732952611:2048]
2025-11-19T13:59:17.002018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435775732952611:2048] [node 1] ICS09 handshake done sender: [4:7574435775732952604:2120] self: [4:7574435775732952608:0] peer: [1:7574435773017642064:0] socket: 396 qp: -1
2025-11-19T13:59:17.002024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435775732952611:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:17.002033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:17.003068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435773017642057:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.003086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:17.003092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435773017642057:2111] [node 4] ICP22 created new session: [1:7574435773017642088:2111]
2025-11-19T13:59:17.003097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435773017642088:2111] [node 4] ICS09 handshake done sender: [1:7574435773017642065:2118] self: [1:7574435773017642064:0] peer: [4:7574435775732952608:0] socket: 395 qp: -1
2025-11-19T13:59:17.003102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435773017642088:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:17.003110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:17.007837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435774882486235:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.007873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:17.007888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435774882486235:2063] [node 4] ICP22 created new session: [5:7574435779177453591:2063]
2025-11-19T13:59:17.007897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435779177453591:2063] [node 4] ICS09 handshake done sender: [5:7574435774882486277:2115] self: [5:7574435774882486288:0] peer: [4:7574435775732952569:0] socket: 365 qp: -1
2025-11-19T13:59:17.007907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435779177453591:2063] [node 4] ICS10 traffic start
2025-11-19T13:59:17.007921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:17.008039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435774882486235:2063] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:17.008620Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435773484288899:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.008667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435773484288828:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.009601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435777779256201:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-11-19T13:59:17.009709Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435771901526693:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:17.010874Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435771901526710:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.010923Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435771901526691:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.010927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435771901526691:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.011675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435771901526685:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.011684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:17.011690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435771901526685:2110] [node 4] ICP22 created new session: [2:7574435776196494010:2110]
2025-11-19T13:59:17.011697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435776196494010:2110] [node 4] ICS09 handshake done sender: [2:7574435771901526693:2116] self: [2:7574435771901526692:0] peer: [4:7574435775732952607:0] socket: 352 qp: -1
2025-11-19T13:59:17.011703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435776196494010:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:17.011709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:17.009918Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435775732952601:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:17.009985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435775732952512:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.009995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:17.009999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435775732952512:2049] [node 2] ICP22 created new session: [4:7574435780027919909:2049]
2025-11-19T13:59:17.010004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435780027919909:2049] [node 2] ICS09 handshake done sender: [4:7574435775732952601:2119] self: [4:7574435775732952607:0] peer: [2:7574435771901526692:0] socket: 393 qp: -1
2025-11-19T13:59:17.010009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435780027919909:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:17.010016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:17.011981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435771901526685:2110] [node 4] ICP18 handshake (actor [2:7574435771901526710:2119]) is too old
2025-11-19T13:59:17.012317Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435774882486246:2087] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.012327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435774882486246:2087] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.012891Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435774882486290:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.012993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435774882486201:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.013255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435777779256202:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-11-19T13:59:17.013419Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775740297541:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.013435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435775740297541:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.015456Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435774882486246:2087] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:17.015687Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435773484288899:2120] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:17.015082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435780035264857:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40245
2025-11-19T13:59:17.015839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435773484288828:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.015856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:17.015861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435773484288828:2059] [node 5] ICP22 created new session: [6:7574435777779256203:2059]
2025-11-19T13:59:17.015866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435777779256203:2059] [node 5] ICS09 handshake done sender: [6:7574435773484288899:2120] self: [6:7574435777779256200:0] peer: [5:7574435774882486245:0] socket: 397 qp: -1
2025-11-19T13:59:17.015872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435777779256203:2059] [node 5] ICS10 traffic start
2025-11-19T13:59:17.015879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:17.028465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435774882486237:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.028496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:17.028505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435774882486237:2073] [node 6] ICP22 created new session: [5:7574435779177453596:2073]
2025-11-19T13:59:17.028511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435779177453596:2073] [node 6] ICS09 handshake done sender: [5:7574435774882486246:2087] self: [5:7574435774882486245:0] peer: [6:7574435777779256200:0] socket: 363 qp: -1
2025-11-19T13:59:17.028518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435779177453596:2073] [node 6] ICS10 traffic start
2025-11-19T13:59:17.028530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.029396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435779177453595:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7477
2025-11-19T13:59:17.029213Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435771901526691:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:17.029516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435771901526617:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.029529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:17.029533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435771901526617:2049] [node 3] ICP22 created new session: [2:7574435776196494013:2049]
2025-11-19T13:59:17.029538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435776196494013:2049] [node 3] ICS09 handshake done sender: [2:7574435771901526691:2115] self: [2:7574435771901526690:0] peer: [3:7574435775740297559:0] socket: 378 qp: -1
2025-11-19T13:59:17.029543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435776196494013:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:17.029550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:17.032053Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435774882486290:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:17.032122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435774882486201:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.032135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:17.032140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435774882486201:2050] [node 3] ICP22 created new session: [5:7574435779177453598:2050]
2025-11-19T13:59:17.032144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435779177453598:2050] [node 3] ICS09 handshake done sender: [5:7574435774882486290:2121] self: [5:7574435779177453594:0] peer: [3:7574435775740297540:0] socket: 345 qp: -1
2025-11-19T13:59:17.032149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435779177453598:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:17.032155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:17.031870Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435775740297541:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:17.045816Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435777779256201:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.045871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435773484288811:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.052038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435775740297532:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.052076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:17.052091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435775740297532:2112] [node 5] ICP22 created new session: [3:7574435780035264859:2112]
2025-11-19T13:59:17.052100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435780035264859:2112] [node 5] ICS09 handshake done sender: [3:7574435775740297541:2117] self: [3:7574435775740297540:0] peer: [5:7574435779177453594:0] socket: 390 qp: -1
2025-11-19T13:59:17.052111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435780035264859:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:17.052125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:17.052363Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435775740297543:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.052376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435775740297543:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.052856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435775740297465:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23947) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435775740297552:2119] held: no
2025-11-19T13:59:17.063447Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435775740297552:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.066845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435777779256206:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-11-19T13:59:17.067220Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435777779256201:2122] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:17.067304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435773484288811:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.067320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:17.067324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435773484288811:2050] [node 3] ICP22 created new session: [6:7574435777779256207:2050]
2025-11-19T13:59:17.067329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435777779256207:2050] [node 3] ICS09 handshake done sender: [6:7574435777779256201:2122] self: [6:7574435777779256205:0] peer: [3:7574435775740297542:0] socket: 362 qp: -1
2025-11-19T13:59:17.067334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435777779256207:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:17.067340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:17.066469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435775740297465:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.066496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:17.066507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435775740297465:2049] [node 2] ICP22 created new session: [3:7574435780035264862:2049]
2025-11-19T13:59:17.066515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435780035264862:2049] [node 2] ICS09 handshake done sender: [3:7574435775740297552:2119] self: [3:7574435775740297559:0] peer: [2:7574435771901526690:0] socket: 383 qp: -1
2025-11-19T13:59:17.066523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435780035264862:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:17.066529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:17.067042Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435775740297543:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:17.068641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435775740297533:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.068654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:17.068659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435775740297533:2113] [node 6] ICP22 created new session: [3:7574435780035264864:2113]
2025-11-19T13:59:17.068664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435780035264864:2113] [node 6] ICS09 handshake done sender: [3:7574435775740297543:2118] self: [3:7574435775740297542:0] peer: [6:7574435777779256205:0] socket: 358 qp: -1
2025-11-19T13:59:17.068667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435780035264864:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:17.068673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:17.090437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:17.090559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435771901526712:2048] [node 1] ICS01 socket: 338 reason# EndOfStream
2025-11-19T13:59:17.090573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435771901526616:2048] [node 1] ICP30 unregister session Session# [2:7574435771901526712:2048] VirtualId# [2:7574435771901526699:0]
2025-11-19T13:59:17.090582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435771901526712:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.090638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:17.090663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:17.091274Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:17.091351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435775740297557:2048] [node 1] ICS01 socket: 339 reason# EndOfStream
2025-11-19T13:59:17.091357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435775740297464:2048] [node 1] ICP30 unregister session Session# [3:7574435775740297557:2048] VirtualId# [3:7574435775740297547:0]
2025-11-19T13:59:17.091362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435775740297557:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.091396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:17.091424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:17.112031Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:17.112143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435774882486283:2048] [node 1] ICS01 socket: 371 reason# EndOfStream
2025-11-19T13:59:17.112155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435774882486199:2048] [node 1] ICP30 unregister session Session# [5:7574435774882486283:2048] VirtualId# [5:7574435774882486281:0]
2025-11-19T13:59:17.112165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435774882486283:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.112225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:17.112257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:17.112297Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:17.112361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435773484288902:2048] [node 1] ICS01 socket: 388 reason# EndOfStream
2025-11-19T13:59:17.112366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435773484288809:2048] [node 1] ICP30 unregister session Session# [6:7574435773484288902:2048] VirtualId# [6:7574435773484288900:0]
2025-11-19T13:59:17.112370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435773484288902:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.112398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:17.112419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:17.112448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:17.112507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435775732952611:2048] [node 1] ICS01 socket: 396 reason# EndOfStream
2025-11-19T13:59:17.112512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435775732952511:2048] [node 1] ICP30 unregister session Session# [4:7574435775732952611:2048] VirtualId# [4:7574435775732952608:0]
2025-11-19T13:59:17.112515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435775732952611:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.112541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:17.112562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:17.116425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:17.116507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435780035264862:2049] [node 2] ICS01 socket: 383 reason# EndOfStream
2025-11-19T13:59:17.116515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435775740297465:2049] [node 2] ICP30 unregister session Session# [3:7574435780035264862:2049] VirtualId# [3:7574435775740297559:0]
2025-11-19T13:59:17.116522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435780035264862:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.116568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:17.116589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:17.119846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:17.119920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435774882486293:2049] [node 2] ICS01 socket: 369 reason# EndOfStream
2025-11-19T13:59:17.119926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435774882486200:2049] [node 2] ICP30 unregister session Session# [5:7574435774882486293:2049] VirtualId# [5:7574435774882486287:0]
2025-11-19T13:59:17.119931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435774882486293:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.119980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:17.120006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:17.120031Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:17.120088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435780027919909:2049] [node 2] ICS01 socket: 393 reason# EndOfStream
2025-11-19T13:59:17.120092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435775732952512:2049] [node 2] ICP30 unregister session Session# [4:7574435780027919909:2049] VirtualId# [4:7574435775732952607:0]
2025-11-19T13:59:17.120096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435780027919909:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.120120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:17.120138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:17.120160Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:17.120215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435773484288892:2049] [node 2] ICS01 socket: 357 reason# EndOfStream
2025-11-19T13:59:17.120219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435773484288810:2049] [node 2] ICP30 unregister session Session# [6:7574435773484288892:2049] VirtualId# [6:7574435773484288889:0]
2025-11-19T13:59:17.120222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435773484288892:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.120246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:17.120269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:17.132078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:17.132198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435779177453598:2050] [node 3] ICS01 socket: 345 reason# EndOfStream
2025-11-19T13:59:17.132208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435774882486201:2050] [node 3] ICP30 unregister session Session# [5:7574435779177453598:2050] VirtualId# [5:7574435779177453594:0]
2025-11-19T13:59:17.132215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435779177453598:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.132279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:17.132307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:17.132436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:17.132494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435777779256207:2050] [node 3] ICS01 socket: 362 reason# EndOfStream
2025-11-19T13:59:17.132498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435773484288811:2050] [node 3] ICP30 unregister session Session# [6:7574435777779256207:2050] VirtualId# [6:7574435777779256205:0]
2025-11-19T13:59:17.132503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435777779256207:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.132528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:17.132547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:17.146992Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:17.147082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435773484288897:2064] [node 4] ICS01 socket: 353 reason# EndOfStream
2025-11-19T13:59:17.147091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435773484288829:2064] [node 4] ICP30 unregister session Session# [6:7574435773484288897:2064] VirtualId# [6:7574435773484288891:0]
2025-11-19T13:59:17.147099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435773484288897:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.147164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:17.147190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:17.217289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435777695732538:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:17.217300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435777695732539:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:17.256043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435775889825089:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:17.256054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435775889825090:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:17.300086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435778354935954:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:17.300100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435778354935955:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:17.331941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435777639257022:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:17.331961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435777639257023:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:17.331965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435777639257024:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:17.376491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435778859746198:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:17.376505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435778859746199:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:17.376510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435778859746200:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:17.425728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435779282816188:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:17.425777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435779282816190:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:17.425861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435779282816189:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:17.451320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435775889825150:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:17.452018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435779282816210:2068] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.452725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435775889825151:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30484
2025-11-19T13:59:17.449877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435777695732599:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:17.459503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435775889825156:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.460208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435779282816223:2070] [node 5] ICP01 ready to work
2025-11-19T13:59:17.464090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435778859746260:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63739
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.466198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435777695732610:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:17.466729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435777695732611:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12364
2025-11-19T13:59:17.468177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435777695732615:2114] [node 6] ICP01 ready to work
2025-11-19T13:59:17.468693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435775889825164:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:17.468751Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435775889825151:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.469400Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435778354935955:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17296533ns) {Configure:104(17294794ns) {ProcessConfigured:135(16927317ns) {StartInitialHandshake:157(16926411ns) {DropHandshakes:469(564ns) {DropIncomingHandshake:441(108ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(279ns) PrepareNewSessionHandshake:476(993ns) {SwitchToState:178(802ns)}}}}}
2025-11-19T13:59:17.469536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435778354936018:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:17.471813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435778859746259:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63739
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.476126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435779282816189:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24258058ns) {Configure:104(24256493ns) {ProcessConfigured:135(23295622ns) {StartInitialHandshake:157(23294905ns) {DropHandshakes:469(577ns) {DropIncomingHandshake:441(142ns) DropOutgoingHandshake:456(119ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:476(915ns) {SwitchToState:178(494ns)}}}}}
2025-11-19T13:59:17.479851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435778354936014:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20527
2025-11-19T13:59:17.484320Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435778354936014:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.484514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435778354936020:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20527
2025-11-19T13:59:17.493269Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777695732611:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.496065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435775889825089:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.496421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435775889825170:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30484
2025-11-19T13:59:17.496048Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435778859746260:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.496237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435778859746262:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:17.496652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435778859746262:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.496845Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435778859746259:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.497117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435778859746199:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.496582Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777695732601:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.496592Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435777695732601:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.498116Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435778354936020:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.498195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435778354935955:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.498627Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435777695732601:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:17.501963Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435775889825160:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.501986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435775889825160:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.504079Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435775889825170:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.504107Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435775889825158:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.504110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435775889825158:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.504652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435775889825171:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30484
2025-11-19T13:59:17.504869Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435775889825151:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.502055Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435779282816225:2079] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.502076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435779282816225:2079] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.507471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435777695732538:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.507496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:17.507509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435777695732538:2048] [node 2] ICP22 created new session: [1:7574435777695732621:2048]
2025-11-19T13:59:17.507516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435777695732621:2048] [node 2] ICS09 handshake done sender: [1:7574435777695732601:2108] self: [1:7574435777695732600:0] peer: [2:7574435775889825169:0] socket: 342 qp: -1
2025-11-19T13:59:17.507524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435777695732621:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:17.507535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:17.507814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435777695732620:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12364
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.508504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435778859746265:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63739
2025-11-19T13:59:17.508765Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435779282816225:2079] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:17.509025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435779282816223:2070] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.509042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:17.509049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435779282816223:2070] [node 5] ICP22 created new session: [6:7574435779282816267:2070]
2025-11-19T13:59:17.509056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435779282816267:2070] [node 5] ICS09 handshake done sender: [6:7574435779282816225:2079] self: [6:7574435779282816224:0] peer: [5:7574435778859746263:0] socket: 352 qp: -1
2025-11-19T13:59:17.509061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435779282816267:2070] [node 5] ICS10 traffic start
2025-11-19T13:59:17.509069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:17.509514Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435778859746260:2109] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.510424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435777695732624:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12364
2025-11-19T13:59:17.509662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435778859746272:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63739
2025-11-19T13:59:17.512254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435778859746262:2110] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.512279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:17.512295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435778859746262:2110] [node 6] ICP22 created new session: [5:7574435778859746274:2110]
2025-11-19T13:59:17.512306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435778859746274:2110] [node 6] ICS09 handshake done sender: [5:7574435778859746260:2109] self: [5:7574435778859746263:0] peer: [6:7574435779282816224:0] socket: 355 qp: -1
2025-11-19T13:59:17.512315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435778859746274:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:17.512343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.515365Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435775889825158:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:17.515432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435775889825089:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.515447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:17.515454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435775889825089:2048] [node 1] ICP22 created new session: [2:7574435775889825174:2048]
2025-11-19T13:59:17.515461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435775889825174:2048] [node 1] ICS09 handshake done sender: [2:7574435775889825151:2108] self: [2:7574435775889825169:0] peer: [1:7574435777695732600:0] socket: 345 qp: -1
2025-11-19T13:59:17.515467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435775889825174:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:17.515475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:17.516164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435777639257083:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2872
2025-11-19T13:59:17.516400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435777639257086:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:17.516487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435777639257087:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:17.516525Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435777639257083:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.513844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435778354936030:2113] [node 5] ICP01 ready to work
2025-11-19T13:59:17.514026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435778354936034:2114] [node 6] ICP01 ready to work
2025-11-19T13:59:17.514650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435778354936031:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20527
2025-11-19T13:59:17.514663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435778354936032:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20527
2025-11-19T13:59:17.514674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435778354936033:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20527
2025-11-19T13:59:17.516975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435778354935954:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18336388ns) {Configure:104(18334670ns) {ProcessConfigured:135(17972390ns) {StartInitialHandshake:157(17971718ns) {DropHandshakes:469(888ns) {DropIncomingHandshake:441(122ns) DropOutgoingHandshake:456(135ns)} GenerateSessionVirtualId:988(382ns) PrepareNewSessionHandshake:476(396ns) {SwitchToState:178(171ns)}}}}}
2025-11-19T13:59:17.517006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435778354935954:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.522210Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435778354936033:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.522492Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435778354936020:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:17.522767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435778354935955:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.522803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:17.522810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435778354935955:2049] [node 2] ICP22 created new session: [3:7574435778354936041:2049]
2025-11-19T13:59:17.522817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435778354936041:2049] [node 2] ICS09 handshake done sender: [3:7574435778354936020:2110] self: [3:7574435778354936025:0] peer: [2:7574435775889825157:0] socket: 357 qp: -1
2025-11-19T13:59:17.522825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435778354936041:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:17.522861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:17.523920Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435775889825160:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:17.524083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435775889825090:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.524106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:17.524113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435775889825090:2049] [node 3] ICP22 created new session: [2:7574435775889825178:2049]
2025-11-19T13:59:17.524119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435775889825178:2049] [node 3] ICS09 handshake done sender: [2:7574435775889825158:2111] self: [2:7574435775889825157:0] peer: [3:7574435778354936025:0] socket: 349 qp: -1
2025-11-19T13:59:17.524125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435775889825178:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:17.524143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:17.524671Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777695732620:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.527794Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777695732603:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.527817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435777695732603:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.531168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435778859746277:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63739
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.531588Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435777695732603:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:17.531880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435777695732539:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.531905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:17.531911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435777695732539:2049] [node 3] ICP22 created new session: [1:7574435777695732627:2049]
2025-11-19T13:59:17.531918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435777695732627:2049] [node 3] ICS09 handshake done sender: [1:7574435777695732603:2109] self: [1:7574435777695732602:0] peer: [3:7574435778354936039:0] socket: 343 qp: -1
2025-11-19T13:59:17.531923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435777695732627:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:17.531930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.532070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435778354936044:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20527
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.533243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435777639257022:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.533275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435777639257089:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2872
2025-11-19T13:59:17.533325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435777639257090:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2872
2025-11-19T13:59:17.533357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435777639257091:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2872
2025-11-19T13:59:17.533734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435775889825156:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.533751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:17.533756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435775889825156:2110] [node 5] ICP22 created new session: [2:7574435775889825180:2110]
2025-11-19T13:59:17.533762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435775889825180:2110] [node 5] ICS09 handshake done sender: [2:7574435775889825160:2112] self: [2:7574435775889825159:0] peer: [5:7574435778859746264:0] socket: 350 qp: -1
2025-11-19T13:59:17.533767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435775889825180:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:17.533774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:17.533982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435775889825182:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30484
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.534869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435779282816270:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39785
2025-11-19T13:59:17.534915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435779282816271:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39785
2025-11-19T13:59:17.534939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435779282816272:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39785
2025-11-19T13:59:17.531503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435778859746276:2115] [node 4] ICP01 ready to work
2025-11-19T13:59:17.535009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435777639257024:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18454048ns) {Configure:104(18453342ns) {ProcessConfigured:135(18069855ns) {StartInitialHandshake:157(18069182ns) {DropHandshakes:469(921ns) {DropIncomingHandshake:441(435ns) DropOutgoingHandshake:456(83ns)} GenerateSessionVirtualId:988(150ns) PrepareNewSessionHandshake:476(888ns) {SwitchToState:178(344ns)}}}}}
2025-11-19T13:59:17.535043Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435777639257089:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.535100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435777639257023:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.540253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435778859746278:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63739
2025-11-19T13:59:17.540884Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435779282816270:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.540922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435779282816189:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.541581Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435777639257090:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.548344Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777695732605:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.548366Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435777695732605:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.548492Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435775889825166:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.548508Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435775889825166:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.548539Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435775889825153:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.548545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435775889825153:2109] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.552653Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435778859746272:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.552801Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435778859746259:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:17.553244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435778859746198:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.553747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435778859746199:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.553760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:17.553768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435778859746199:2049] [node 2] ICP22 created new session: [5:7574435778859746283:2049]
2025-11-19T13:59:17.553775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435778859746283:2049] [node 2] ICS09 handshake done sender: [5:7574435778859746259:2108] self: [5:7574435778859746264:0] peer: [2:7574435775889825159:0] socket: 354 qp: -1
2025-11-19T13:59:17.553781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435778859746283:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:17.553789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:17.552427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435777695732630:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12364
2025-11-19T13:59:17.552995Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435778354935954:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19504) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435778354936014:2107] held: no
2025-11-19T13:59:17.553029Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435778354936014:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:17.555919Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435779282816271:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.556005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435779282816188:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.560233Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777695732624:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.560273Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777695732613:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.560276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435777695732613:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.560930Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777695732617:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.560943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435777695732617:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.565206Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435777695732613:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:17.566219Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435777695732617:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.567036Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435778354936032:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.567107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435778354935954:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.567121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:17.567127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435778354935954:2048] [node 1] ICP22 created new session: [3:7574435778354936045:2048]
2025-11-19T13:59:17.567133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435778354936045:2048] [node 1] ICS09 handshake done sender: [3:7574435778354936014:2107] self: [3:7574435778354936039:0] peer: [1:7574435777695732602:0] socket: 346 qp: -1
2025-11-19T13:59:17.567140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435778354936045:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:17.567147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:17.568928Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435775889825166:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:17.569084Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435775889825153:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:17.569130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435775889825164:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.569143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:17.569159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435775889825164:2113] [node 6] ICP22 created new session: [2:7574435775889825185:2113]
2025-11-19T13:59:17.569166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435775889825185:2113] [node 6] ICS09 handshake done sender: [2:7574435775889825166:2114] self: [2:7574435775889825165:0] peer: [6:7574435779282816273:0] socket: 379 qp: -1
2025-11-19T13:59:17.569171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435775889825185:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:17.569180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:17.569420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435775889825150:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.569432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:17.569437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435775889825150:2107] [node 4] ICP22 created new session: [2:7574435775889825187:2107]
2025-11-19T13:59:17.569441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435775889825187:2107] [node 4] ICS09 handshake done sender: [2:7574435775889825153:2109] self: [2:7574435775889825152:0] peer: [4:7574435777639257105:0] socket: 347 qp: -1
2025-11-19T13:59:17.569445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435775889825187:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:17.569451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:17.567473Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435778859746278:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.568678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435778859746285:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63739
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.581558Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435777639257091:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.581744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435777639257024:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.581777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435777639257107:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2872
2025-11-19T13:59:17.581827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435777639257108:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2872
2025-11-19T13:59:17.581839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435777639257109:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2872
2025-11-19T13:59:17.582644Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435779282816272:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.582679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435779282816210:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.582790Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435775889825182:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.582825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435775889825150:2107] [node 4] ICP18 handshake (actor [2:7574435775889825182:2118]) is too old
2025-11-19T13:59:17.582763Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435778354936022:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.582776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435778354936022:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.583613Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435778354936022:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:17.583885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435778354936018:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.583903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:17.583911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435778354936018:2108] [node 4] ICP22 created new session: [3:7574435778354936050:2108]
2025-11-19T13:59:17.583917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435778354936050:2108] [node 4] ICS09 handshake done sender: [3:7574435778354936022:2111] self: [3:7574435778354936021:0] peer: [4:7574435777639257110:0] socket: 359 qp: -1
2025-11-19T13:59:17.583936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435778354936050:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:17.583945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:17.591040Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435777639257093:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.591060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435777639257093:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.588564Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435778859746199:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(34777143ns) {HandleHandshakeStatus:329(34776486ns) {DropIncomingHandshake:441(213ns) DropOutgoingHandshake:456(82ns) SwitchToState:178(69ns) ProcessPendingSessionEvents:495(34699044ns) {ForwardSessionEventToSession:612(34687643ns) ForwardSessionEventToSession:612(2279ns) ForwardSessionEventToSession:612(943ns) ForwardSessionEventToSession:612(762ns) ForwardSessionEventToSession:612(886ns) ForwardSessionEventToSession:612(501ns) ForwardSessionEventToSession:612(771ns) ForwardSessionEventToSession:612(580ns)}}}
2025-11-19T13:59:17.588603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435778859746200:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.588862Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435778859746272:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:17.592238Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435777695732605:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:17.595991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435777695732615:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.596014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:17.596024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435777695732615:2114] [node 6] ICP22 created new session: [1:7574435777695732635:2114]
2025-11-19T13:59:17.596032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435777695732635:2114] [node 6] ICS09 handshake done sender: [1:7574435777695732617:2115] self: [1:7574435777695732616:0] peer: [6:7574435779282816274:0] socket: 363 qp: -1
2025-11-19T13:59:17.596038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435777695732635:2114] [node 6] ICS10 traffic start
2025-11-19T13:59:17.596047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:17.596730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435777695732610:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.596747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:17.596754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435777695732610:2111] [node 5] ICP22 created new session: [1:7574435777695732637:2111]
2025-11-19T13:59:17.596760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435777695732637:2111] [node 5] ICS09 handshake done sender: [1:7574435777695732613:2113] self: [1:7574435777695732612:0] peer: [5:7574435778859746281:0] socket: 369 qp: -1
2025-11-19T13:59:17.596767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435777695732637:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:17.596774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:17.597189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435777695732599:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.597200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:17.597207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435777695732599:2107] [node 4] ICP22 created new session: [1:7574435777695732639:2107]
2025-11-19T13:59:17.597211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435777695732639:2107] [node 4] ICS09 handshake done sender: [1:7574435777695732605:2110] self: [1:7574435777695732604:0] peer: [4:7574435777639257100:0] socket: 344 qp: -1
2025-11-19T13:59:17.597215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435777695732639:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:17.597221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:17.595942Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435778354936036:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.595958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435778354936036:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.596854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435778859746198:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.596887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:17.596895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435778859746198:2048] [node 1] ICP22 created new session: [5:7574435778859746287:2048]
2025-11-19T13:59:17.596901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435778859746287:2048] [node 1] ICS09 handshake done sender: [5:7574435778859746272:2114] self: [5:7574435778859746281:0] peer: [1:7574435777695732612:0] socket: 372 qp: -1
2025-11-19T13:59:17.596908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435778859746287:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:17.596915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:17.598029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435778859746289:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63739
2025-11-19T13:59:17.597852Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435778354936036:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:17.598772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435778354936030:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.598791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:17.598796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435778354936030:2113] [node 5] ICP22 created new session: [3:7574435778354936053:2113]
2025-11-19T13:59:17.598802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435778354936053:2113] [node 5] ICS09 handshake done sender: [3:7574435778354936036:2118] self: [3:7574435778354936035:0] peer: [5:7574435778859746286:0] socket: 382 qp: -1
2025-11-19T13:59:17.598807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435778354936053:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:17.598815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:17.599302Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435778859746278:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:17.599395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435778859746200:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.599408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:17.599413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435778859746200:2050] [node 3] ICP22 created new session: [5:7574435778859746290:2050]
2025-11-19T13:59:17.599417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435778859746290:2050] [node 3] ICS09 handshake done sender: [5:7574435778859746278:2117] self: [5:7574435778859746286:0] peer: [3:7574435778354936035:0] socket: 360 qp: -1
2025-11-19T13:59:17.599422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435778859746290:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:17.599429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:17.600567Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435777639257107:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.604207Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435779282816188:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19504) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435779282816271:2116] held: no
2025-11-19T13:59:17.604299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435779282816276:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39785
2025-11-19T13:59:17.604792Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435777639257089:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.604926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435779282816277:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39785
2025-11-19T13:59:17.605010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435777639257023:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5239) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435777639257089:2111] held: no
2025-11-19T13:59:17.605029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435777639257023:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.605038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:17.605045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435777639257023:2049] [node 2] ICP22 created new session: [4:7574435777639257113:2049]
2025-11-19T13:59:17.605050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435777639257113:2049] [node 2] ICS09 handshake done sender: [4:7574435777639257089:2111] self: [4:7574435777639257105:0] peer: [2:7574435775889825152:0] socket: 367 qp: -1
2025-11-19T13:59:17.605057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435777639257113:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:17.605062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:17.605294Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435777639257083:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:17.606841Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777695732630:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.606878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435777695732599:2107] [node 4] ICP18 handshake (actor [1:7574435777695732630:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.608077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435779282816278:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39785
2025-11-19T13:59:17.608179Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435779282816271:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.608716Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435777639257093:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:17.608799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435777639257111:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2872
2025-11-19T13:59:17.608923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435777639257022:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.608938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:17.608943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435777639257022:2048] [node 1] ICP22 created new session: [4:7574435777639257116:2048]
2025-11-19T13:59:17.608949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435777639257116:2048] [node 1] ICS09 handshake done sender: [4:7574435777639257083:2108] self: [4:7574435777639257100:0] peer: [1:7574435777695732604:0] socket: 348 qp: -1
2025-11-19T13:59:17.608954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435777639257116:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:17.608960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:17.610252Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435777639257091:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:17.610312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435777639257086:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.610327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:17.610332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435777639257086:2109] [node 6] ICP22 created new session: [4:7574435777639257118:2109]
2025-11-19T13:59:17.610337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435777639257118:2109] [node 6] ICS09 handshake done sender: [4:7574435777639257093:2114] self: [4:7574435777639257092:0] peer: [6:7574435779282816275:0] socket: 386 qp: -1
2025-11-19T13:59:17.610341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435777639257118:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:17.610346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:17.610456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435777639257024:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.610462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:17.610465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435777639257024:2050] [node 3] ICP22 created new session: [4:7574435777639257121:2050]
2025-11-19T13:59:17.610469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435777639257121:2050] [node 3] ICS09 handshake done sender: [4:7574435777639257091:2113] self: [4:7574435777639257110:0] peer: [3:7574435778354936021:0] socket: 385 qp: -1
2025-11-19T13:59:17.610472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435777639257121:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:17.610475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:17.610869Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435779282816270:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.611319Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435779282816278:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.611334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435779282816188:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.611344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:17.611350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435779282816188:2048] [node 1] ICP22 created new session: [6:7574435779282816280:2048]
2025-11-19T13:59:17.611355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435779282816280:2048] [node 1] ICS09 handshake done sender: [6:7574435779282816271:2116] self: [6:7574435779282816274:0] peer: [1:7574435777695732616:0] socket: 390 qp: -1
2025-11-19T13:59:17.611360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435779282816280:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:17.611365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:17.611490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435779282816279:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39785
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.611879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435778859746292:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63739
2025-11-19T13:59:17.615382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435779282816189:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.615404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:17.615412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435779282816189:2049] [node 2] ICP22 created new session: [6:7574435779282816282:2049]
2025-11-19T13:59:17.615417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435779282816282:2049] [node 2] ICS09 handshake done sender: [6:7574435779282816270:2115] self: [6:7574435779282816273:0] peer: [2:7574435775889825165:0] socket: 381 qp: -1
2025-11-19T13:59:17.615423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435779282816282:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:17.615431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:17.615569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435779282816190:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.615584Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435779282816210:2068] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14347) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435779282816272:2117] held: no
2025-11-19T13:59:17.615792Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435779282816272:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:17.617038Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435778354936038:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.617050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435778354936038:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.619555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435779282816210:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.619574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:17.619582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435779282816210:2068] [node 4] ICP22 created new session: [6:7574435779282816286:2068]
2025-11-19T13:59:17.619587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435779282816286:2068] [node 4] ICS09 handshake done sender: [6:7574435779282816272:2117] self: [6:7574435779282816275:0] peer: [4:7574435777639257092:0] socket: 391 qp: -1
2025-11-19T13:59:17.619594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435779282816286:2068] [node 4] ICS10 traffic start
2025-11-19T13:59:17.619602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.619997Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435778354936038:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:17.619998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435779282816288:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39785
2025-11-19T13:59:17.620220Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435778859746292:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.620267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435778859746276:2115] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.620353Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435779282816278:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:17.620112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435778354936034:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.620130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:17.620137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435778354936034:2114] [node 6] ICP22 created new session: [3:7574435778354936056:2114]
2025-11-19T13:59:17.620142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435778354936056:2114] [node 6] ICS09 handshake done sender: [3:7574435778354936038:2119] self: [3:7574435778354936037:0] peer: [6:7574435779282816285:0] socket: 362 qp: -1
2025-11-19T13:59:17.620147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435778354936056:2114] [node 6] ICS10 traffic start
2025-11-19T13:59:17.620155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:17.620437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435779282816190:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.620450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:17.620459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435779282816190:2050] [node 3] ICP22 created new session: [6:7574435779282816289:2050]
2025-11-19T13:59:17.620464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435779282816289:2050] [node 3] ICS09 handshake done sender: [6:7574435779282816278:2120] self: [6:7574435779282816285:0] peer: [3:7574435778354936037:0] socket: 376 qp: -1
2025-11-19T13:59:17.620469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435779282816289:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:17.620475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:17.620459Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435777639257099:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.620471Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435777639257099:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.621430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435778859746294:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63739
2025-11-19T13:59:17.621547Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435777639257099:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:17.621632Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435778859746292:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:17.621704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435778859746276:2115] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.621741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:17.621746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435778859746276:2115] [node 4] ICP22 created new session: [5:7574435778859746295:2115]
2025-11-19T13:59:17.621752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435778859746295:2115] [node 4] ICS09 handshake done sender: [5:7574435778859746292:2121] self: [5:7574435778859746293:0] peer: [4:7574435777639257098:0] socket: 353 qp: -1
2025-11-19T13:59:17.621757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435778859746295:2115] [node 4] ICS10 traffic start
2025-11-19T13:59:17.621763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:17.621937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435777639257087:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.621955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:17.621960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435777639257087:2110] [node 5] ICP22 created new session: [4:7574435777639257125:2110]
2025-11-19T13:59:17.621965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435777639257125:2110] [node 5] ICS09 handshake done sender: [4:7574435777639257099:2117] self: [4:7574435777639257098:0] peer: [5:7574435778859746293:0] socket: 370 qp: -1
2025-11-19T13:59:17.621969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435777639257125:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:17.621976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:17.638234Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:17.638340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435775889825174:2048] [node 1] ICS01 socket: 345 reason# EndOfStream
2025-11-19T13:59:17.638349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435775889825089:2048] [node 1] ICP30 unregister session Session# [2:7574435775889825174:2048] VirtualId# [2:7574435775889825169:0]
2025-11-19T13:59:17.638356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435775889825174:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.638397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:17.638419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:17.646951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:17.647044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435778354936045:2048] [node 1] ICS01 socket: 346 reason# EndOfStream
2025-11-19T13:59:17.647054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435778354935954:2048] [node 1] ICP30 unregister session Session# [3:7574435778354936045:2048] VirtualId# [3:7574435778354936039:0]
2025-11-19T13:59:17.647061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435778354936045:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.647096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:17.647116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:17.653012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:17.653109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435778859746287:2048] [node 1] ICS01 socket: 372 reason# EndOfStream
2025-11-19T13:59:17.653118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435778859746198:2048] [node 1] ICP30 unregister session Session# [5:7574435778859746287:2048] VirtualId# [5:7574435778859746281:0]
2025-11-19T13:59:17.653125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435778859746287:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.653182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:17.653208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:17.653675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:17.653737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435777639257116:2048] [node 1] ICS01 socket: 348 reason# EndOfStream
2025-11-19T13:59:17.653742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435777639257022:2048] [node 1] ICP30 unregister session Session# [4:7574435777639257116:2048] VirtualId# [4:7574435777639257100:0]
2025-11-19T13:59:17.653747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435777639257116:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.653771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:17.653793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:17.653246Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:17.653306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435779282816280:2048] [node 1] ICS01 socket: 390 reason# EndOfStream
2025-11-19T13:59:17.653311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435779282816188:2048] [node 1] ICP30 unregister session Session# [6:7574435779282816280:2048] VirtualId# [6:7574435779282816274:0]
2025-11-19T13:59:17.653315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435779282816280:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.653348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:17.653369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:17.664418Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:17.664512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435778354936041:2049] [node 2] ICS01 socket: 357 reason# EndOfStream
2025-11-19T13:59:17.664522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435778354935955:2049] [node 2] ICP30 unregister session Session# [3:7574435778354936041:2049] VirtualId# [3:7574435778354936025:0]
2025-11-19T13:59:17.664530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435778354936041:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.664571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:17.664591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:17.670980Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:17.671073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435777639257113:2049] [node 2] ICS01 socket: 367 reason# EndOfStream
2025-11-19T13:59:17.671082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435777639257023:2049] [node 2] ICP30 unregister session Session# [4:7574435777639257113:2049] VirtualId# [4:7574435777639257105:0]
2025-11-19T13:59:17.671090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435777639257113:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.671137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:17.671160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:17.671195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:17.671258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435779282816282:2049] [node 2] ICS01 socket: 381 reason# EndOfStream
2025-11-19T13:59:17.671264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435779282816189:2049] [node 2] ICP30 unregister session Session# [6:7574435779282816282:2049] VirtualId# [6:7574435779282816273:0]
2025-11-19T13:59:17.671269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435779282816282:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.671292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:17.671312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:17.673394Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:17.673478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435778859746283:2049] [node 2] ICS07 socket disconnect 354 reason# EndOfStream
2025-11-19T13:59:17.673484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435778859746283:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.673540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:17.673562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435778859746283:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:17.674341Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435778859746199:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5239) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:17.674354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435778859746283:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:17.674360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435778859746199:2049] [node 2] ICP30 unregister session Session# [5:7574435778859746283:2049] VirtualId# [5:7574435778859746264:0]
2025-11-19T13:59:17.674374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:17.674379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435778859746199:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5239) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:17.674382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5239) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:17.684629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435778859746199:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:17.684764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:17.684852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435778859746290:2050] [node 3] ICS01 socket: 360 reason# EndOfStream
2025-11-19T13:59:17.684860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435778859746200:2050] [node 3] ICP30 unregister session Session# [5:7574435778859746290:2050] VirtualId# [5:7574435778859746286:0]
2025-11-19T13:59:17.684866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435778859746290:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.684951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:17.684976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:17.689961Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:17.690055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435779282816289:2050] [node 3] ICS07 socket disconnect 376 reason# EndOfStream
2025-11-19T13:59:17.690061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435779282816289:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.690129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:17.690151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435779282816289:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:17.691004Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435779282816190:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12112) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:17.691017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435779282816289:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:17.691023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435779282816190:2050] [node 3] ICP30 unregister session Session# [6:7574435779282816289:2050] VirtualId# [6:7574435779282816285:0]
2025-11-19T13:59:17.691037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:17.691042Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435779282816190:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12112) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:17.691045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12112) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:17.703807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435779282816190:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:59:17.712385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:17.712477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435779282816286:2068] [node 4] ICS01 socket: 391 reason# EndOfStream
2025-11-19T13:59:17.712486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435779282816210:2068] [node 4] ICP30 unregister session Session# [6:7574435779282816286:2068] VirtualId# [6:7574435779282816275:0]
2025-11-19T13:59:17.712493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435779282816286:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:17.712552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:17.712579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:17.759910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435777989279027:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:17.759925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435777989279028:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:17.781054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435779931324461:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:17.781069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435779931324462:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:17.820616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435776018417859:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:17.820629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435776018417860:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:17.855995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435776337633840:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:17.856010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435776337633841:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:17.856015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435776337633842:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:17.888181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435777785703336:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:17.888194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435777785703337:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:17.888200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435777785703338:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:17.927568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435776455549636:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:17.927581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435776455549637:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:17.927586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435776455549638:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:17.948963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435779931324516:2099] [node 5] ICP01 ready to work
2025-11-19T13:59:17.949374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435776018417915:2102] [node 4] ICP01 ready to work
2025-11-19T13:59:17.952016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435776455549660:2067] [node 4] ICP01 ready to work
2025-11-19T13:59:17.952108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435776455549661:2069] [node 5] ICP01 ready to work
2025-11-19T13:59:17.957359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435777989279090:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:17.957491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435777989279095:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:17.956274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435777785703363:2067] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.962780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435779931324531:2104] [node 6] ICP01 ready to work
2025-11-19T13:59:17.964736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435779931324532:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34891
2025-11-19T13:59:17.964749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435779931324533:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34891
2025-11-19T13:59:17.967949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435776337633840:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18733745ns) {Configure:104(18732830ns) {ProcessConfigured:135(18364695ns) {StartInitialHandshake:157(18364417ns) {DropHandshakes:469(496ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:476(1297ns) {SwitchToState:178(659ns)}}}}}
2025-11-19T13:59:17.970330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435776337633907:2100] [node 6] ICP01 ready to work
2025-11-19T13:59:17.970471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435776337633908:2104] [node 5] ICP01 ready to work
2025-11-19T13:59:17.971682Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435776455549660:2067] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19529984ns) {Configure:104(19529131ns) {ProcessConfigured:135(19256385ns) {StartInitialHandshake:157(19255995ns) {DropHandshakes:469(507ns) {DropIncomingHandshake:441(91ns) DropOutgoingHandshake:456(207ns)} GenerateSessionVirtualId:988(80ns) PrepareNewSessionHandshake:476(2059ns) {SwitchToState:178(1277ns)}}}}}
2025-11-19T13:59:17.976559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435777989279093:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:17.979814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435779931324524:2101] [node 4] ICP01 ready to work
2025-11-19T13:59:17.980326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435776018417927:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.985271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435777989279097:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15666
2025-11-19T13:59:17.985328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435777989279098:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15666
2025-11-19T13:59:17.985338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435777989279099:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15666
2025-11-19T13:59:17.985367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435777989279100:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15666
2025-11-19T13:59:17.986054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435776018417860:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(28098172ns) {Configure:104(28097022ns) {ProcessConfigured:135(27805034ns) {StartInitialHandshake:157(27804758ns) {DropHandshakes:469(847ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(449ns)} GenerateSessionVirtualId:988(149ns) PrepareNewSessionHandshake:476(934ns) {SwitchToState:178(177ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.983724Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435779931324532:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.986174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435779931324541:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34891
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.987550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435779931324542:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34891
2025-11-19T13:59:17.985779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435776455549713:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12577
2025-11-19T13:59:17.985799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435776455549714:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12577
2025-11-19T13:59:17.985838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435776455549715:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12577
2025-11-19T13:59:17.985865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435776455549716:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12577
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.988311Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435776337633910:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.988322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435776337633910:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.988052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435776455549714:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.988089Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435776455549715:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.988144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435776455549717:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12577
2025-11-19T13:59:17.988171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435776455549637:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.988183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435776455549660:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:17.989140Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435776337633910:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.989234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435776337633907:2100] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:17.989247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:17.989253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435776337633907:2100] [node 6] ICP22 created new session: [4:7574435776337633919:2100]
2025-11-19T13:59:17.989259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435776337633919:2100] [node 6] ICS09 handshake done sender: [4:7574435776337633910:2111] self: [4:7574435776337633909:0] peer: [6:7574435776455549719:0] socket: 366 qp: -1
2025-11-19T13:59:17.989265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435776337633919:2100] [node 6] ICS10 traffic start
2025-11-19T13:59:17.989272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:17.989321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435776455549720:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12577
2025-11-19T13:59:17.989395Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435776455549638:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(30708964ns) {Configure:104(30708363ns) {ProcessConfigured:135(30439936ns) {StartInitialHandshake:157(30439648ns) {DropHandshakes:469(313ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:476(565ns) {SwitchToState:178(34ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:17.989681Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435777785703338:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17759973ns) {Configure:104(17759192ns) {ProcessConfigured:135(17489968ns) {StartInitialHandshake:157(17489132ns) {DropHandshakes:469(420ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(46ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:476(989ns) {SwitchToState:178(682ns)}}}}}
2025-11-19T13:59:17.989639Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435776455549715:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:17.989799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435776455549660:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:17.989808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:17.989814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435776455549660:2067] [node 4] ICP22 created new session: [6:7574435776455549722:2067]
2025-11-19T13:59:17.989818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435776455549722:2067] [node 4] ICS09 handshake done sender: [6:7574435776455549715:2116] self: [6:7574435776455549719:0] peer: [4:7574435776337633909:0] socket: 375 qp: -1
2025-11-19T13:59:17.989822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435776455549722:2067] [node 4] ICS10 traffic start
2025-11-19T13:59:17.989827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:17.986223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435776018417931:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:17.992355Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435779931324535:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.992368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435779931324535:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:17.993990Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435779931324542:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.991823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435777785703400:2079] [node 4] ICP01 ready to work
2025-11-19T13:59:17.996663Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777989279099:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.996688Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777989279100:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.996704Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777989279097:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:17.999649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435777989279028:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(51245214ns) {Configure:104(51243838ns) {ProcessConfigured:135(50988009ns) {StartInitialHandshake:157(50987234ns) {DropHandshakes:469(370ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:476(699ns) {SwitchToState:178(502ns)}}}}}
2025-11-19T13:59:18.000661Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435779931324541:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.000694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435779931324461:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.000847Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777989279092:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.000859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435777989279092:2109] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.001897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435779931324545:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34891
2025-11-19T13:59:18.001959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435779931324524:2101] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(22073001ns) {Configure:104(22072209ns) {ProcessConfigured:135(21673096ns) {StartInitialHandshake:157(21672807ns) {DropHandshakes:469(619ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:476(630ns) {SwitchToState:178(200ns)}}}}}
2025-11-19T13:59:18.002509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435777785703351:2061] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9505
2025-11-19T13:59:18.001733Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435777989279090:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(25111363ns) {Configure:104(25111065ns) {ProcessConfigured:135(24780237ns) {StartInitialHandshake:157(24779929ns) {DropHandshakes:469(417ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(46ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:476(436ns) {SwitchToState:178(208ns)}}}}}
2025-11-19T13:59:18.001792Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435777989279092:2109] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:18.003687Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435779931324541:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:18.001521Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435776018417859:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25265209ns) {Configure:104(25263293ns) {ProcessConfigured:135(24949272ns) {StartInitialHandshake:157(24948869ns) {DropHandshakes:469(1371ns) {DropIncomingHandshake:441(95ns) DropOutgoingHandshake:456(122ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:476(631ns) {SwitchToState:178(434ns)}}}}}
2025-11-19T13:59:18.004418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435777989279027:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.004437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:18.004443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435777989279027:2048] [node 2] ICP22 created new session: [1:7574435782284246410:2048]
2025-11-19T13:59:18.004448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435782284246410:2048] [node 2] ICS09 handshake done sender: [1:7574435777989279092:2109] self: [1:7574435777989279087:0] peer: [2:7574435779931324544:0] socket: 371 qp: -1
2025-11-19T13:59:18.004454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435782284246410:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:18.004469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.009554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435780632601218:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64088
2025-11-19T13:59:18.009626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435780632601219:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64088
2025-11-19T13:59:18.009658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435780632601220:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64088
2025-11-19T13:59:18.010425Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435776455549717:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.010483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435776455549638:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.010741Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435777785703400:2079] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18817384ns) {Configure:104(18815896ns) {ProcessConfigured:135(18490545ns) {StartInitialHandshake:157(18489903ns) {DropHandshakes:469(454ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(31ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:476(703ns) {SwitchToState:178(63ns)}}}}}
2025-11-19T13:59:18.010795Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435777785703351:2061] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.010822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435777785703337:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.010737Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435776018417933:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.010749Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435776018417933:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.011854Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435780632601218:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.011971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435776337633842:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.013648Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435776455549660:2067] [node 4] ICP35 event processing took too much time PendingConnection:278(23812642ns) {HandleHandshakeStatus:329(23812291ns) {DropIncomingHandshake:441(161ns) DropOutgoingHandshake:456(1158ns) SwitchToState:178(65ns) ProcessPendingSessionEvents:495(11218491ns) {ForwardSessionEventToSession:612(11205183ns) ForwardSessionEventToSession:612(1683ns) ForwardSessionEventToSession:612(1179ns) ForwardSessionEventToSession:612(615ns) ForwardSessionEventToSession:612(709ns) ForwardSessionEventToSession:612(833ns) ForwardSessionEventToSession:612(1293ns) ForwardSessionEventToSession:612(643ns) ForwardSessionEventToSession:612(747ns) ForwardSessionEventToSession:612(1024ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.017650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435779931324461:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.017670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:18.017679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435779931324461:2048] [node 1] ICP22 created new session: [2:7574435784226291844:2048]
2025-11-19T13:59:18.017685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435784226291844:2048] [node 1] ICS09 handshake done sender: [2:7574435779931324541:2116] self: [2:7574435779931324544:0] peer: [1:7574435777989279087:0] socket: 377 qp: -1
2025-11-19T13:59:18.017692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435784226291844:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:18.017700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:18.017892Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435779931324518:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.017896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435779931324518:2102] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.019191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782080670712:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9505
2025-11-19T13:59:18.019248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782080670713:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9505
2025-11-19T13:59:18.020633Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777989279102:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.020647Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435777989279102:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.021371Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435782080670712:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.021601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435779931324461:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.019918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435776455549716:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.020005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435776455549636:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.022327Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435780632601220:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.022412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435776337633840:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.022553Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435782080670713:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.022589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435777785703400:2079] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.022684Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435776337633912:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.022690Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435776337633912:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.023438Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435776337633912:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:18.023522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435776337633908:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.023536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:18.023543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435776337633908:2104] [node 5] ICP22 created new session: [4:7574435780632601224:2104]
2025-11-19T13:59:18.023548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435780632601224:2104] [node 5] ICS09 handshake done sender: [4:7574435776337633912:2112] self: [4:7574435776337633911:0] peer: [5:7574435782080670715:0] socket: 384 qp: -1
2025-11-19T13:59:18.023553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435780632601224:2104] [node 5] ICS10 traffic start
2025-11-19T13:59:18.023559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:18.022129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435780313385233:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44831
2025-11-19T13:59:18.026186Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435777989279102:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:18.026425Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435779931324533:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.026788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435784226291846:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34891
2025-11-19T13:59:18.027048Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435779931324518:2102] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:18.027223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435779931324516:2099] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.027234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:18.027239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435779931324516:2099] [node 5] ICP22 created new session: [2:7574435784226291850:2099]
2025-11-19T13:59:18.027243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435784226291850:2099] [node 5] ICS09 handshake done sender: [2:7574435779931324518:2102] self: [2:7574435779931324517:0] peer: [5:7574435782080670711:0] socket: 348 qp: -1
2025-11-19T13:59:18.027246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435784226291850:2099] [node 5] ICS10 traffic start
2025-11-19T13:59:18.027257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:18.025966Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435776018417927:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(45578840ns) {Configure:104(45577653ns) {ProcessConfigured:135(45270723ns) {StartInitialHandshake:157(45270002ns) {DropHandshakes:469(596ns) {DropIncomingHandshake:441(45ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(335ns) PrepareNewSessionHandshake:476(705ns) {SwitchToState:178(143ns)}}}}}
2025-11-19T13:59:18.026119Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435776018417933:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:18.027129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435776018417931:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.027138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:18.027144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435776018417931:2110] [node 6] ICP22 created new session: [3:7574435780313385236:2110]
2025-11-19T13:59:18.027149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435780313385236:2110] [node 6] ICS09 handshake done sender: [3:7574435776018417933:2112] self: [3:7574435776018417932:0] peer: [6:7574435780750517021:0] socket: 379 qp: -1
2025-11-19T13:59:18.027154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435780313385236:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:18.027171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:18.027837Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435776455549713:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.027876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435776455549661:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.029324Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435776018417917:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.029341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435776018417917:2103] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.035869Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777989279098:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.036016Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777989279104:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.036020Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435777989279104:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.036734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435777989279095:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.036753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:18.036761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435777989279095:2110] [node 6] ICP22 created new session: [1:7574435782284246414:2110]
2025-11-19T13:59:18.036768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435782284246414:2110] [node 6] ICS09 handshake done sender: [1:7574435777989279102:2115] self: [1:7574435777989279101:0] peer: [6:7574435780750517022:0] socket: 372 qp: -1
2025-11-19T13:59:18.036777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435782284246414:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:18.036789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:18.037303Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435777989279027:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(32850917ns) {HandleHandshakeStatus:329(32850229ns) {DropIncomingHandshake:441(3918ns) DropOutgoingHandshake:456(308ns) SwitchToState:178(71ns) ProcessPendingSessionEvents:495(25539080ns) {ForwardSessionEventToSession:612(25529208ns) ForwardSessionEventToSession:612(1626ns) ForwardSessionEventToSession:612(959ns) ForwardSessionEventToSession:612(602ns) ForwardSessionEventToSession:612(487ns) ForwardSessionEventToSession:612(1097ns) ForwardSessionEventToSession:612(972ns) ForwardSessionEventToSession:612(446ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.038934Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435777989279104:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.038446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782080670716:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9505
2025-11-19T13:59:18.039885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435776337633907:2100] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:18.039919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [4:7574435776337633907:2100] [node 6] ICP35 event processing took too much time StateWork:293(27982044ns) {HandleHandshakeStatus:395(27980973ns) {LogHandshakeFail:472(4623ns) {UpdateErrorStateLog:544(2505ns)}}}
2025-11-19T13:59:18.042158Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435777785703365:2072] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.042176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435777785703365:2072] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.042301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435776455549636:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435776455549716:2117] held: no
2025-11-19T13:59:18.042669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435777989279093:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.042690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:18.042697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435777989279093:2108] [node 4] ICP22 created new session: [1:7574435782284246419:2108]
2025-11-19T13:59:18.042703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435782284246419:2108] [node 4] ICS09 handshake done sender: [1:7574435777989279104:2116] self: [1:7574435777989279103:0] peer: [4:7574435780632601222:0] socket: 373 qp: -1
2025-11-19T13:59:18.042708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435782284246419:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:18.042715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.045829Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435780313385233:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.045970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435776018417859:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.046165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782080670717:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9505
2025-11-19T13:59:18.046316Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435776337633840:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435780632601220:2117] held: no
2025-11-19T13:59:18.046471Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435776018417917:2103] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:18.048790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435776018417931:2110] [node 6] ICP35 event processing took too much time PendingConnection:278(21631070ns) {HandleHandshakeStatus:329(21630505ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(224ns) SwitchToState:178(45ns) ProcessPendingSessionEvents:495(9997772ns) {ForwardSessionEventToSession:612(9984265ns) ForwardSessionEventToSession:612(1960ns) ForwardSessionEventToSession:612(1074ns) ForwardSessionEventToSession:612(634ns) ForwardSessionEventToSession:612(785ns) ForwardSessionEventToSession:612(1331ns) ForwardSessionEventToSession:612(574ns) ForwardSessionEventToSession:612(751ns) ForwardSessionEventToSession:612(635ns) ForwardSessionEventToSession:612(698ns)}}}
2025-11-19T13:59:18.048815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435776018417915:2102] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.048841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:18.048849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435776018417915:2102] [node 4] ICP22 created new session: [3:7574435780313385241:2102]
2025-11-19T13:59:18.048855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435780313385241:2102] [node 4] ICS09 handshake done sender: [3:7574435776018417917:2103] self: [3:7574435776018417916:0] peer: [4:7574435780632601221:0] socket: 349 qp: -1
2025-11-19T13:59:18.048861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435780313385241:2102] [node 4] ICS10 traffic start
2025-11-19T13:59:18.048868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:18.048305Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435779931324516:2099] [node 5] ICP35 event processing took too much time PendingConnection:278(21047761ns) {HandleHandshakeStatus:329(21047474ns) {DropIncomingHandshake:441(2477ns) DropOutgoingHandshake:456(88ns) SwitchToState:178(63ns) ProcessPendingSessionEvents:495(9572009ns) {ForwardSessionEventToSession:612(9560339ns) ForwardSessionEventToSession:612(2064ns) ForwardSessionEventToSession:612(904ns) ForwardSessionEventToSession:612(547ns) ForwardSessionEventToSession:612(515ns) ForwardSessionEventToSession:612(1082ns) ForwardSessionEventToSession:612(748ns) ForwardSessionEventToSession:612(477ns) ForwardSessionEventToSession:612(657ns) ForwardSessionEventToSession:612(867ns)}}}
2025-11-19T13:59:18.047863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782080670718:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9505
2025-11-19T13:59:18.050589Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435777989279111:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.050603Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435777989279111:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.050830Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435779931324535:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.051212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435782284246421:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15666
2025-11-19T13:59:18.052251Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435777989279111:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:18.052324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435777989279028:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.052339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:18.052347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435777989279028:2049] [node 3] ICP22 created new session: [1:7574435782284246423:2049]
2025-11-19T13:59:18.052353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435782284246423:2049] [node 3] ICS09 handshake done sender: [1:7574435777989279111:2117] self: [1:7574435777989279089:0] peer: [3:7574435780313385240:0] socket: 369 qp: -1
2025-11-19T13:59:18.052359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435782284246423:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:18.052367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:18.053438Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435784226291846:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.055505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435777785703337:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19336) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435777785703351:2061] held: no
2025-11-19T13:59:18.053465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435779931324531:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.053476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:18.053485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435779931324531:2104] [node 6] ICP22 created new session: [2:7574435784226291854:2104]
2025-11-19T13:59:18.053490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435784226291854:2104] [node 6] ICS09 handshake done sender: [2:7574435779931324535:2113] self: [2:7574435779931324534:0] peer: [6:7574435776455549718:0] socket: 364 qp: -1
2025-11-19T13:59:18.053496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435784226291854:2104] [node 6] ICS10 traffic start
2025-11-19T13:59:18.053505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:18.054892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435779931324531:2104] [node 6] ICP18 handshake (actor [2:7574435784226291846:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.056026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435780750517024:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12577
2025-11-19T13:59:18.056416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435780750517025:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12577
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.059386Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435782284246421:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.059615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435776337633842:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8111) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435780632601218:2115] held: no
2025-11-19T13:59:18.057295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435780313385244:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44831
2025-11-19T13:59:18.057340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435780313385245:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44831
2025-11-19T13:59:18.057370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435780313385246:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44831
2025-11-19T13:59:18.058093Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435780313385244:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.058116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435776018417915:2102] [node 4] ICP18 handshake (actor [3:7574435780313385244:2116]) is too old
2025-11-19T13:59:18.058352Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435780313385233:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:18.059464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435776018417859:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.059476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:18.059483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435776018417859:2048] [node 1] ICP22 created new session: [3:7574435780313385247:2048]
2025-11-19T13:59:18.059488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435780313385247:2048] [node 1] ICS09 handshake done sender: [3:7574435780313385233:2114] self: [3:7574435780313385240:0] peer: [1:7574435777989279089:0] socket: 393 qp: -1
No RDMA context found for GID: 2025-11-19T13:59:18.059496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435780313385247:2048] [node 1] ICS10 traffic start
::ffff:127.1.0.0
2025-11-19T13:59:18.059503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.060322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435780750517026:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12577
2025-11-19T13:59:18.060407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435777989279028:2049] [node 3] ICP18 handshake (actor [1:7574435782284246421:2119]) is too old
2025-11-19T13:59:18.060518Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435777785703351:2061] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:18.059900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435780632601227:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64088
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.064473Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435782080670713:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:18.064970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435780632601228:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64088
2025-11-19T13:59:18.065509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435776455549637:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19336) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435776455549714:2115] held: no
2025-11-19T13:59:18.065541Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435776455549717:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:18.065713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435777785703337:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.065729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:18.065751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435777785703337:2049] [node 2] ICP22 created new session: [5:7574435782080670721:2049]
2025-11-19T13:59:18.065757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435782080670721:2049] [node 2] ICS09 handshake done sender: [5:7574435777785703351:2061] self: [5:7574435782080670711:0] peer: [2:7574435779931324517:0] socket: 351 qp: -1
2025-11-19T13:59:18.065763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435782080670721:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:18.065781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:18.065606Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435779931324538:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.065620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435779931324538:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.066951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435780632601229:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64088
2025-11-19T13:59:18.067001Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435780632601220:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:18.066401Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435782080670718:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.066412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435777785703400:2079] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.066421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:18.066426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435777785703400:2079] [node 4] ICP22 created new session: [5:7574435782080670723:2079]
2025-11-19T13:59:18.066431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435782080670723:2079] [node 4] ICS09 handshake done sender: [5:7574435782080670713:2117] self: [5:7574435782080670715:0] peer: [4:7574435776337633911:0] socket: 390 qp: -1
2025-11-19T13:59:18.066436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435782080670723:2079] [node 4] ICS10 traffic start
2025-11-19T13:59:18.066443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:18.066618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782080670719:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9505
2025-11-19T13:59:18.066844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435777785703336:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.067168Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435780632601218:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:18.067539Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435779931324538:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:18.065184Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435780313385245:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.065217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435776018417860:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.067883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435779931324462:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.067905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:18.067913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435779931324462:2049] [node 3] ICP22 created new session: [2:7574435784226291857:2049]
2025-11-19T13:59:18.067919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435784226291857:2049] [node 3] ICS09 handshake done sender: [2:7574435779931324538:2114] self: [2:7574435779931324523:0] peer: [3:7574435780313385249:0] socket: 398 qp: -1
2025-11-19T13:59:18.067924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435784226291857:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:18.067931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:18.067589Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435782284246409:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.067599Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435782284246409:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.067299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435776337633840:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.067310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:18.067316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435776337633840:2048] [node 1] ICP22 created new session: [4:7574435780632601230:2048]
2025-11-19T13:59:18.067320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435780632601230:2048] [node 1] ICS09 handshake done sender: [4:7574435780632601220:2117] self: [4:7574435780632601222:0] peer: [1:7574435777989279103:0] socket: 388 qp: -1
2025-11-19T13:59:18.067325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435780632601230:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:18.067331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:18.067386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435776337633842:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.067392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:18.067395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435776337633842:2050] [node 3] ICP22 created new session: [4:7574435780632601232:2050]
2025-11-19T13:59:18.067398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435780632601232:2050] [node 3] ICS09 handshake done sender: [4:7574435780632601218:2115] self: [4:7574435780632601221:0] peer: [3:7574435776018417916:0] socket: 352 qp: -1
2025-11-19T13:59:18.067402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435780632601232:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:18.067406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:18.068106Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435780632601229:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.068129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435776337633841:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.068535Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435782284246409:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:18.068646Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435777785703365:2072] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:18.068707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435777989279090:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.068720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:18.068725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435777989279090:2107] [node 5] ICP22 created new session: [1:7574435782284246426:2107]
2025-11-19T13:59:18.068729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435782284246426:2107] [node 5] ICS09 handshake done sender: [1:7574435782284246409:2118] self: [1:7574435777989279096:0] peer: [5:7574435782080670726:0] socket: 400 qp: -1
2025-11-19T13:59:18.068733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435782284246426:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:18.068738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:18.075869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435776018417859:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16363271ns) {HandleHandshakeStatus:329(16362418ns) {DropIncomingHandshake:441(280ns) DropOutgoingHandshake:456(956ns) SwitchToState:178(138ns) ProcessPendingSessionEvents:495(11021929ns) {ForwardSessionEventToSession:612(11006234ns) ForwardSessionEventToSession:612(2700ns) ForwardSessionEventToSession:612(1134ns) ForwardSessionEventToSession:612(800ns) ForwardSessionEventToSession:612(563ns) ForwardSessionEventToSession:612(1099ns) ForwardSessionEventToSession:612(725ns) ForwardSessionEventToSession:612(546ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(1244ns)}}}
2025-11-19T13:59:18.068493Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435779931324546:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.068502Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435779931324546:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.078166Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435776018417860:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19336) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435780313385245:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.078495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435780632601235:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64088
2025-11-19T13:59:18.078832Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435779931324546:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:18.079043Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435780632601229:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:18.079147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435776337633841:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.079165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:18.079172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435776337633841:2049] [node 2] ICP22 created new session: [4:7574435780632601236:2049]
2025-11-19T13:59:18.079178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435780632601236:2049] [node 2] ICS09 handshake done sender: [4:7574435780632601229:2120] self: [4:7574435780632601234:0] peer: [2:7574435779931324540:0] socket: 397 qp: -1
2025-11-19T13:59:18.079185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435780632601236:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:18.079194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:18.083870Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435776455549716:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:18.084012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435777785703337:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(18275246ns) {HandleHandshakeStatus:329(18274756ns) {DropIncomingHandshake:441(205ns) DropOutgoingHandshake:456(54ns) SwitchToState:178(51ns) ProcessPendingSessionEvents:495(484ns)}}
2025-11-19T13:59:18.085376Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435776455549714:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.085528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435776455549638:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.085548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:18.085556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435776455549638:2050] [node 3] ICP22 created new session: [6:7574435780750517027:2050]
2025-11-19T13:59:18.085563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435780750517027:2050] [node 3] ICS09 handshake done sender: [6:7574435776455549717:2118] self: [6:7574435780750517021:0] peer: [3:7574435776018417932:0] socket: 381 qp: -1
2025-11-19T13:59:18.085570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435780750517027:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:18.085578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:18.085810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435776455549637:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.085817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:18.085822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435776455549637:2049] [node 2] ICP22 created new session: [6:7574435780750517031:2049]
2025-11-19T13:59:18.085826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435780750517031:2049] [node 2] ICS09 handshake done sender: [6:7574435776455549714:2115] self: [6:7574435776455549718:0] peer: [2:7574435779931324534:0] socket: 374 qp: -1
2025-11-19T13:59:18.085831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435780750517031:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:18.085837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:18.086110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435780750517029:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12577
2025-11-19T13:59:18.086550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435776455549636:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.086574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:18.086580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435776455549636:2048] [node 1] ICP22 created new session: [6:7574435780750517033:2048]
2025-11-19T13:59:18.086587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435780750517033:2048] [node 1] ICS09 handshake done sender: [6:7574435776455549716:2117] self: [6:7574435780750517022:0] peer: [1:7574435777989279101:0] socket: 376 qp: -1
2025-11-19T13:59:18.086592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435780750517033:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:18.086598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:18.086921Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435776455549713:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:18.087014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435776455549661:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.087025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:18.087030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435776455549661:2069] [node 5] ICP22 created new session: [6:7574435780750517035:2069]
2025-11-19T13:59:18.087035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435780750517035:2069] [node 5] ICS09 handshake done sender: [6:7574435776455549713:2114] self: [6:7574435780750517023:0] peer: [5:7574435777785703364:0] socket: 359 qp: -1
2025-11-19T13:59:18.087040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435780750517035:2069] [node 5] ICS10 traffic start
2025-11-19T13:59:18.087051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:18.088172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435777785703363:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.088197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:18.088205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435777785703363:2067] [node 6] ICP22 created new session: [5:7574435782080670727:2067]
2025-11-19T13:59:18.088215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435782080670727:2067] [node 6] ICS09 handshake done sender: [5:7574435777785703365:2072] self: [5:7574435777785703364:0] peer: [6:7574435780750517023:0] socket: 356 qp: -1
2025-11-19T13:59:18.088223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435782080670727:2067] [node 6] ICS10 traffic start
2025-11-19T13:59:18.088240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:18.088422Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435782080670719:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.088620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435779931324524:2101] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.088639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:18.088645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435779931324524:2101] [node 4] ICP22 created new session: [2:7574435784226291860:2101]
2025-11-19T13:59:18.088650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435784226291860:2101] [node 4] ICS09 handshake done sender: [2:7574435779931324546:2119] self: [2:7574435779931324540:0] peer: [4:7574435780632601234:0] socket: 404 qp: -1
2025-11-19T13:59:18.088654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435784226291860:2101] [node 4] ICS10 traffic start
2025-11-19T13:59:18.088660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:18.091523Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435779931324462:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(23602838ns) {HandleHandshakeStatus:329(23601894ns) {DropIncomingHandshake:441(3376ns) DropOutgoingHandshake:456(151ns) SwitchToState:178(171ns) ProcessPendingSessionEvents:495(23519687ns) {ForwardSessionEventToSession:612(23508732ns) ForwardSessionEventToSession:612(1671ns) ForwardSessionEventToSession:612(740ns) ForwardSessionEventToSession:612(1360ns) ForwardSessionEventToSession:612(586ns) ForwardSessionEventToSession:612(673ns) ForwardSessionEventToSession:612(890ns) ForwardSessionEventToSession:612(501ns) ForwardSessionEventToSession:612(608ns) ForwardSessionEventToSession:612(1134ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.092048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435780313385250:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44831
2025-11-19T13:59:18.092193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435780313385251:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44831
2025-11-19T13:59:18.094181Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435777989279090:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(25436654ns) {HandleHandshakeStatus:329(25435843ns) {DropIncomingHandshake:441(2929ns) DropOutgoingHandshake:456(158ns) SwitchToState:178(60ns) ProcessPendingSessionEvents:495(25384370ns) {ForwardSessionEventToSession:612(25373185ns) ForwardSessionEventToSession:612(1952ns) ForwardSessionEventToSession:612(656ns) ForwardSessionEventToSession:612(1430ns) ForwardSessionEventToSession:612(599ns) ForwardSessionEventToSession:612(574ns) ForwardSessionEventToSession:612(583ns) ForwardSessionEventToSession:612(815ns) ForwardSessionEventToSession:612(628ns) ForwardSessionEventToSession:612(977ns)}}}
2025-11-19T13:59:18.094779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435777785703336:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435782080670718:2120] held: no
2025-11-19T13:59:18.094869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435777785703338:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.095068Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435780313385251:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.095104Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435780313385235:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.095108Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435780313385235:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.095947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782080670730:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9505
2025-11-19T13:59:18.096399Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435780313385235:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:18.096775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435776018417927:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.096789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:18.096793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435776018417927:2109] [node 5] ICP22 created new session: [3:7574435780313385254:2109]
2025-11-19T13:59:18.096797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435780313385254:2109] [node 5] ICS09 handshake done sender: [3:7574435780313385235:2115] self: [3:7574435776018417928:0] peer: [5:7574435782080670729:0] socket: 405 qp: -1
2025-11-19T13:59:18.096801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435780313385254:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:18.096806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:18.096889Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435780313385245:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:18.097263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435776018417860:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.097275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:18.097280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435776018417860:2049] [node 2] ICP22 created new session: [3:7574435780313385256:2049]
2025-11-19T13:59:18.097284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435780313385256:2049] [node 2] ICS09 handshake done sender: [3:7574435780313385245:2117] self: [3:7574435780313385249:0] peer: [2:7574435779931324523:0] socket: 389 qp: -1
2025-11-19T13:59:18.097288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435780313385256:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:18.097292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.096236Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435782080670718:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:18.096475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435777785703336:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.096486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:18.096492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435777785703336:2048] [node 1] ICP22 created new session: [5:7574435782080670731:2048]
2025-11-19T13:59:18.096498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435782080670731:2048] [node 1] ICS09 handshake done sender: [5:7574435782080670718:2120] self: [5:7574435782080670726:0] peer: [1:7574435777989279096:0] socket: 402 qp: -1
2025-11-19T13:59:18.096504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435782080670731:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:18.096510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:18.098753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435777785703338:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8111) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435782080670719:2121] held: no
2025-11-19T13:59:18.099899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782080670733:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9505
2025-11-19T13:59:18.111978Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435782080670719:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:18.112238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435777785703338:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.112258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:18.112268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435777785703338:2050] [node 3] ICP22 created new session: [5:7574435782080670734:2050]
2025-11-19T13:59:18.112275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435782080670734:2050] [node 3] ICS09 handshake done sender: [5:7574435782080670719:2121] self: [5:7574435782080670729:0] peer: [3:7574435776018417928:0] socket: 408 qp: -1
2025-11-19T13:59:18.112284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435782080670734:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:18.112297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:18.126050Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:18.126141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435784226291844:2048] [node 1] ICS01 socket: 377 reason# EndOfStream
2025-11-19T13:59:18.126150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435779931324461:2048] [node 1] ICP30 unregister session Session# [2:7574435784226291844:2048] VirtualId# [2:7574435779931324544:0]
2025-11-19T13:59:18.126156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435784226291844:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.126190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:18.126209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:18.126626Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:18.126687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435780313385247:2048] [node 1] ICS07 socket disconnect 393 reason# EndOfStream
2025-11-19T13:59:18.126692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435780313385247:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.126721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:18.126740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435780313385247:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:18.132567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435776018417859:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12861) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:18.132582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435780313385247:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:18.132913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435776018417859:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12861) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:18.132924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435780313385247:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:18.132929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435776018417859:2048] [node 1] ICP30 unregister session Session# [3:7574435780313385247:2048] VirtualId# [3:7574435780313385240:0]
2025-11-19T13:59:18.132949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:18.132955Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435776018417859:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:18.132957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:18.135182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:18.135266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435780750517033:2048] [node 1] ICS01 socket: 376 reason# EndOfStream
2025-11-19T13:59:18.135273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435776455549636:2048] [node 1] ICP30 unregister session Session# [6:7574435780750517033:2048] VirtualId# [6:7574435780750517022:0]
2025-11-19T13:59:18.135281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435780750517033:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.135328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:18.135351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:18.140077Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:18.140138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435782080670731:2048] [node 1] ICS01 socket: 402 reason# EndOfStream
2025-11-19T13:59:18.140142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435777785703336:2048] [node 1] ICP30 unregister session Session# [5:7574435782080670731:2048] VirtualId# [5:7574435782080670726:0]
2025-11-19T13:59:18.140146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435782080670731:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.140176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:18.140196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:18.139860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:18.139947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435780632601230:2048] [node 1] ICS01 socket: 388 reason# EndOfStream
2025-11-19T13:59:18.139956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435776337633840:2048] [node 1] ICP30 unregister session Session# [4:7574435780632601230:2048] VirtualId# [4:7574435780632601222:0]
2025-11-19T13:59:18.139962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435780632601230:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.140013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:18.140037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:18.142705Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:18.142776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435780313385256:2049] [node 2] ICS01 socket: 389 reason# EndOfStream
2025-11-19T13:59:18.142782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435776018417860:2049] [node 2] ICP30 unregister session Session# [3:7574435780313385256:2049] VirtualId# [3:7574435780313385249:0]
2025-11-19T13:59:18.142788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435780313385256:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.142815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:18.142833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:18.152048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:18.152149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435780750517031:2049] [node 2] ICS01 socket: 374 reason# EndOfStream
2025-11-19T13:59:18.152159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435776455549637:2049] [node 2] ICP30 unregister session Session# [6:7574435780750517031:2049] VirtualId# [6:7574435776455549718:0]
2025-11-19T13:59:18.152167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435780750517031:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.152241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:18.152271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:18.152502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:18.152561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435780632601236:2049] [node 2] ICS01 socket: 397 reason# EndOfStream
2025-11-19T13:59:18.152566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435776337633841:2049] [node 2] ICP30 unregister session Session# [4:7574435780632601236:2049] VirtualId# [4:7574435780632601234:0]
2025-11-19T13:59:18.152570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435780632601236:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.152594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:18.152615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:18.152296Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:18.152379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435782080670721:2049] [node 2] ICS01 socket: 351 reason# EndOfStream
2025-11-19T13:59:18.152386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435777785703337:2049] [node 2] ICP30 unregister session Session# [5:7574435782080670721:2049] VirtualId# [5:7574435782080670711:0]
2025-11-19T13:59:18.152393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435782080670721:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.152442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:18.152466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:18.166760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:18.166861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435782080670734:2050] [node 3] ICS01 socket: 408 reason# EndOfStream
2025-11-19T13:59:18.166870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435777785703338:2050] [node 3] ICP30 unregister session Session# [5:7574435782080670734:2050] VirtualId# [5:7574435782080670729:0]
2025-11-19T13:59:18.166876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435782080670734:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.166936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:18.166960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:18.167951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:18.168038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435780750517027:2050] [node 3] ICS01 socket: 381 reason# EndOfStream
2025-11-19T13:59:18.168046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435776455549638:2050] [node 3] ICP30 unregister session Session# [6:7574435780750517027:2050] VirtualId# [6:7574435780750517021:0]
2025-11-19T13:59:18.168052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435780750517027:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.168119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:18.168149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:18.183432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T13:59:18.183531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435782080670723:2079] [node 4] ICS01 socket: 390 reason# EndOfStream
2025-11-19T13:59:18.183542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435777785703400:2079] [node 4] ICP30 unregister session Session# [5:7574435782080670723:2079] VirtualId# [5:7574435782080670715:0]
2025-11-19T13:59:18.183549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435782080670723:2079] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.183586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:18.183608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:18.183876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:18.183960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435776455549722:2067] [node 4] ICS01 socket: 375 reason# EndOfStream
2025-11-19T13:59:18.183967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435776455549660:2067] [node 4] ICP30 unregister session Session# [6:7574435776455549722:2067] VirtualId# [6:7574435776455549719:0]
2025-11-19T13:59:18.183973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435776455549722:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.184027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:18.184048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:18.203614Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:59:18.203719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435780750517035:2069] [node 5] ICS01 socket: 359 reason# EndOfStream
2025-11-19T13:59:18.203729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435776455549661:2069] [node 5] ICP30 unregister session Session# [6:7574435780750517035:2069] VirtualId# [6:7574435780750517023:0]
2025-11-19T13:59:18.203736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435780750517035:2069] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.203825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:59:18.203847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:18.307984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435783994401811:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:18.307998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435783994401812:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:18.395920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435781069129439:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:18.395937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435781069129440:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:18.447921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435783698763542:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:18.447937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435783698763543:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:18.468251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435781709302572:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:18.468265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435781709302573:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:18.468269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435781709302574:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:18.500064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435782577974972:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:18.500078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435782577974973:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:18.500083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435782577974974:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:18.535873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435781566788307:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:18.535887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435781566788308:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:18.535892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435781566788309:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:18.576984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435781566788367:2103] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.579953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435783698763602:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13070
2025-11-19T13:59:18.579967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435781069129501:2105] [node 5] ICP01 ready to work
2025-11-19T13:59:18.580062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435781069129502:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.582436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782577975033:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6489
2025-11-19T13:59:18.577540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435783994401875:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:18.577642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435783994401877:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:18.580743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435783994401886:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27658
2025-11-19T13:59:18.584136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435783698763605:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:18.584230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435783698763606:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-19T13:59:18.581093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435781069129503:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26651
2025-11-19T13:59:18.586733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435781566788374:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14903
2025-11-19T13:59:18.587804Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435781566788369:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.587808Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435781566788369:2105] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.587000Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435782577975033:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.587213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435782577975035:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:18.587633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435782577975035:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.588690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782577975037:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6489
2025-11-19T13:59:18.589453Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435782577975033:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:18.589606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435782577975035:2109] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.589620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:18.589626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435782577975035:2109] [node 6] ICP22 created new session: [5:7574435782577975038:2109]
2025-11-19T13:59:18.589633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435782577975038:2109] [node 6] ICS09 handshake done sender: [5:7574435782577975033:2108] self: [5:7574435782577975036:0] peer: [6:7574435781566788368:0] socket: 362 qp: -1
2025-11-19T13:59:18.589640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435782577975038:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:18.589662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:18.589258Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435781566788369:2105] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:18.592851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435783698763615:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:18.595576Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435783994401886:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.592046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435781069129512:2112] [node 4] ICP01 ready to work
2025-11-19T13:59:18.595783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435781566788373:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14903
2025-11-19T13:59:18.595918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435781069129503:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.599998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435781709302633:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
2025-11-19T13:59:18.601562Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435781709302633:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.607854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435783994401888:2114] [node 6] ICP01 ready to work
2025-11-19T13:59:18.608507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435781709302641:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.612326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435783698763621:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13070
2025-11-19T13:59:18.615797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435781566788367:2103] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.615835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:18.615848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435781566788367:2103] [node 5] ICP22 created new session: [6:7574435781566788376:2103]
2025-11-19T13:59:18.615860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435781566788376:2103] [node 5] ICS09 handshake done sender: [6:7574435781566788369:2105] self: [6:7574435781566788368:0] peer: [5:7574435782577975036:0] socket: 356 qp: -1
2025-11-19T13:59:18.615871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435781566788376:2103] [node 5] ICS10 traffic start
2025-11-19T13:59:18.615904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:18.615992Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435783994401880:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.616016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435783994401880:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.616375Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435783698763602:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.616497Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435781709302641:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.620050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435783698763542:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(35120636ns) {Configure:104(35120118ns) {ProcessConfigured:135(34909920ns) {StartInitialHandshake:157(34909435ns) {DropHandshakes:469(148ns) {DropIncomingHandshake:441(37ns) DropOutgoingHandshake:456(32ns)} GenerateSessionVirtualId:988(95ns) PrepareNewSessionHandshake:476(892ns) {SwitchToState:178(617ns)}}}}}
2025-11-19T13:59:18.620095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435783698763542:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.619336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435783994401892:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27658
2025-11-19T13:59:18.619412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435783994401893:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27658
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.623829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435781709302643:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.623338Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435781069129512:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(31221414ns) {Configure:104(31220187ns) {ProcessConfigured:135(30870161ns) {StartInitialHandshake:157(30869628ns) {DropHandshakes:469(950ns) {DropIncomingHandshake:441(262ns) DropOutgoingHandshake:456(170ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(1903ns) {SwitchToState:178(1192ns)}}}}}
2025-11-19T13:59:18.623382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435781069129439:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.624192Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435783994401892:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.624156Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435781069129440:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(28186770ns) {Configure:104(28185863ns) {ProcessConfigured:135(27828410ns) {StartInitialHandshake:157(27827868ns) {DropHandshakes:469(571ns) {DropIncomingHandshake:441(82ns) DropOutgoingHandshake:456(108ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(657ns) {SwitchToState:178(587ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.623803Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435781566788374:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.624200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435781566788309:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.624230Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435781566788373:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.624481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435781566788308:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.625313Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435783994401883:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.625325Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435783994401883:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.626220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435781709302572:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.626253Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435781709302572:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(24642764ns) {IncomingHandshake:261(24642164ns) {DropIncomingHandshake:441(186ns) PrepareNewSessionHandshake:476(832ns) {SwitchToState:178(334ns)} GenerateSessionVirtualId:988(275ns) IssueIncomingHandshakeReply:193(24637450ns) {SwitchToState:178(240ns)}}}
2025-11-19T13:59:18.627881Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435783698763608:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.627900Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435783698763608:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.628856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435783994401872:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.628874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435783994401872:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.627813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435781069129519:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26651
2025-11-19T13:59:18.627847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435781069129518:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26651
2025-11-19T13:59:18.628912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435781069129520:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26651
2025-11-19T13:59:18.631999Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435783698763621:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.635925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435781069129501:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(54693707ns) {Configure:104(54691959ns) {ProcessConfigured:135(54292489ns) {StartInitialHandshake:157(54291709ns) {DropHandshakes:469(740ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(105ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:476(1728ns) {SwitchToState:178(1516ns)}}}}}
2025-11-19T13:59:18.635982Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435781069129519:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.635998Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435781069129518:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.636498Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435783698763610:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.636515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435783698763610:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.636136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435781709302645:2113] [node 5] ICP01 ready to work
2025-11-19T13:59:18.636348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435781709302574:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.636393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435781709302574:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19830342ns) {Configure:104(19829552ns) {ProcessConfigured:135(19488316ns) {IncomingHandshake:261(19477986ns) {DropIncomingHandshake:441(136ns) PrepareNewSessionHandshake:476(511ns) {SwitchToState:178(208ns)} GenerateSessionVirtualId:988(348ns) IssueIncomingHandshakeReply:193(19473334ns) {SwitchToState:178(649ns)}}}}}
2025-11-19T13:59:18.637039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435781709302648:2114] [node 6] ICP01 ready to work
2025-11-19T13:59:18.639861Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435781069129506:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.639887Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435781069129506:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.641232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435781566788367:2103] [node 5] ICP35 event processing took too much time PendingConnection:278(51213620ns) {HandleHandshakeStatus:329(51213097ns) {DropIncomingHandshake:441(919ns) DropOutgoingHandshake:456(842ns) SwitchToState:178(297ns) ProcessPendingSessionEvents:495(15011459ns) {ForwardSessionEventToSession:612(15004226ns) ForwardSessionEventToSession:612(2468ns) ForwardSessionEventToSession:612(1047ns)}}}
2025-11-19T13:59:18.643974Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435783994401883:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:18.644313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435783994401812:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.644332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:18.644340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435783994401812:2049] [node 3] ICP22 created new session: [1:7574435783994401896:2049]
2025-11-19T13:59:18.644347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435783994401896:2049] [node 3] ICS09 handshake done sender: [1:7574435783994401883:2111] self: [1:7574435783994401882:0] peer: [3:7574435783698763623:0] socket: 358 qp: -1
2025-11-19T13:59:18.644366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435783994401896:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:18.644385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:18.644431Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435782577974972:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24529097ns) {Configure:104(24528110ns) {ProcessConfigured:135(24180487ns) {StartInitialHandshake:157(24179712ns) {DropHandshakes:469(784ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(140ns)} GenerateSessionVirtualId:988(290ns) PrepareNewSessionHandshake:476(772ns) {SwitchToState:178(44ns)}}}}}
2025-11-19T13:59:18.646097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435782577975050:2114] [node 4] ICP01 ready to work
2025-11-19T13:59:18.646271Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435783994401893:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.650821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435781709302650:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
2025-11-19T13:59:18.650654Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435783698763608:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:18.650952Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435781069129520:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.652715Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435781709302643:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.651662Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435783994401872:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:18.659260Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435783994401880:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:18.660376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435781566788384:2113] [node 4] ICP01 ready to work
2025-11-19T13:59:18.660919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435781709302648:2114] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23827500ns) {Configure:104(23826527ns) {ProcessConfigured:135(23541404ns) {StartInitialHandshake:157(23540961ns) {DropHandshakes:469(642ns) {DropIncomingHandshake:441(101ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(90ns) PrepareNewSessionHandshake:476(956ns) {SwitchToState:178(803ns)}}}}}
2025-11-19T13:59:18.660948Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435781709302641:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.663401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435781566788388:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14903
2025-11-19T13:59:18.663428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435781566788389:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14903
2025-11-19T13:59:18.661922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435783698763628:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13070
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.663986Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435781069129506:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:18.664145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782577975052:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6489
2025-11-19T13:59:18.664166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782577975053:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6489
2025-11-19T13:59:18.665228Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435782577975050:2114] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19075478ns) {Configure:104(19074254ns) {ProcessConfigured:135(18747931ns) {StartInitialHandshake:157(18747417ns) {DropHandshakes:469(751ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(94ns) PrepareNewSessionHandshake:476(510ns) {SwitchToState:178(222ns)}}}}}
2025-11-19T13:59:18.665838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435781069129502:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.665853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:18.665869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435781069129502:2107] [node 6] ICP22 created new session: [2:7574435781069129527:2107]
2025-11-19T13:59:18.665874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435781069129527:2107] [node 6] ICS09 handshake done sender: [2:7574435781069129506:2110] self: [2:7574435781069129505:0] peer: [6:7574435781566788382:0] socket: 363 qp: -1
2025-11-19T13:59:18.665879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435781069129527:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:18.665886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:18.665989Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435781069129515:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.665992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435781069129515:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.664787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435783994401811:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.664813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:18.664820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435783994401811:2048] [node 2] ICP22 created new session: [1:7574435783994401903:2048]
2025-11-19T13:59:18.664828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435783994401903:2048] [node 2] ICS09 handshake done sender: [1:7574435783994401872:2107] self: [1:7574435783994401871:0] peer: [2:7574435781069129516:0] socket: 354 qp: -1
2025-11-19T13:59:18.664844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435783994401903:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:18.664853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:18.664969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435783994401877:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.664977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:18.664980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435783994401877:2109] [node 4] ICP22 created new session: [1:7574435783994401905:2109]
2025-11-19T13:59:18.664985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435783994401905:2109] [node 4] ICS09 handshake done sender: [1:7574435783994401880:2110] self: [1:7574435783994401879:0] peer: [4:7574435781709302640:0] socket: 357 qp: -1
2025-11-19T13:59:18.664988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435783994401905:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:18.664993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.665339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435781709302574:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.665356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:18.665362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435781709302574:2050] [node 3] ICP22 created new session: [4:7574435781709302652:2050]
2025-11-19T13:59:18.665367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435781709302652:2050] [node 3] ICS09 handshake done sender: [4:7574435781709302641:2111] self: [4:7574435781709302642:0] peer: [3:7574435783698763607:0] socket: 371 qp: -1
2025-11-19T13:59:18.665373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435781709302652:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:18.665389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:18.665506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435781709302573:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.667874Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435783698763610:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:18.668566Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435783698763602:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:18.669368Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435781566788373:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:18.667974Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435781069129515:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:18.669491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435781069129512:2112] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.669508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:18.669513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435781069129512:2112] [node 4] ICP22 created new session: [2:7574435781069129531:2112]
2025-11-19T13:59:18.669520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435781069129531:2112] [node 4] ICS09 handshake done sender: [2:7574435781069129515:2113] self: [2:7574435781069129513:0] peer: [4:7574435781709302654:0] socket: 382 qp: -1
2025-11-19T13:59:18.669526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435781069129531:2112] [node 4] ICS10 traffic start
2025-11-19T13:59:18.669533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:18.675925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435783698763605:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.675964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:18.675980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435783698763605:2108] [node 4] ICP22 created new session: [3:7574435783698763630:2108]
2025-11-19T13:59:18.675988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435783698763630:2108] [node 4] ICS09 handshake done sender: [3:7574435783698763608:2110] self: [3:7574435783698763607:0] peer: [4:7574435781709302642:0] socket: 367 qp: -1
2025-11-19T13:59:18.675998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435783698763630:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:18.676009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.683949Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435781709302572:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2668) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435781709302633:2108] held: no
2025-11-19T13:59:18.684067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435781709302656:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
2025-11-19T13:59:18.684307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435781709302657:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
2025-11-19T13:59:18.684368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435781709302658:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
2025-11-19T13:59:18.685019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435783994401902:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27658
2025-11-19T13:59:18.689894Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435782577975052:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.689960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435782577974972:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.691891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435781709302645:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(55681010ns) {Configure:104(55679914ns) {ProcessConfigured:135(55324646ns) {StartInitialHandshake:157(55324023ns) {DropHandshakes:469(838ns) {DropIncomingHandshake:441(114ns) DropOutgoingHandshake:456(185ns)} GenerateSessionVirtualId:988(323ns) PrepareNewSessionHandshake:476(1237ns) {SwitchToState:178(850ns)}}}}}
2025-11-19T13:59:18.692030Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435781709302643:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:18.691910Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435783994401902:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.691961Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435783994401885:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.691966Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435783994401885:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.694200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435783698763542:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.694229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:18.694244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435783698763542:2048] [node 1] ICP22 created new session: [3:7574435783698763632:2048]
2025-11-19T13:59:18.694252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435783698763632:2048] [node 1] ICS09 handshake done sender: [3:7574435783698763602:2107] self: [3:7574435783698763623:0] peer: [1:7574435783994401882:0] socket: 359 qp: -1
2025-11-19T13:59:18.694260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435783698763632:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:18.694280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:18.694461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435783698763606:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.694470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:18.694475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435783698763606:2109] [node 6] ICP22 created new session: [3:7574435783698763634:2109]
2025-11-19T13:59:18.694479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435783698763634:2109] [node 6] ICS09 handshake done sender: [3:7574435783698763610:2111] self: [3:7574435783698763609:0] peer: [6:7574435781566788381:0] socket: 368 qp: -1
2025-11-19T13:59:18.694485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435783698763634:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:18.694490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:18.694640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435783698763629:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13070
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.701183Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435782577975053:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.701291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435782577974974:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.701656Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435783698763619:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.701670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435783698763619:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.702646Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435783698763619:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:18.702763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435783698763615:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.702783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:18.702790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435783698763615:2112] [node 5] ICP22 created new session: [3:7574435783698763637:2112]
2025-11-19T13:59:18.702795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435783698763637:2112] [node 5] ICS09 handshake done sender: [3:7574435783698763619:2114] self: [3:7574435783698763618:0] peer: [5:7574435782577975056:0] socket: 399 qp: -1
2025-11-19T13:59:18.702800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435783698763637:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:18.702809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:18.700640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435781566788308:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.700665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:18.700689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435781566788308:2049] [node 2] ICP22 created new session: [6:7574435781566788392:2049]
2025-11-19T13:59:18.700697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435781566788392:2049] [node 2] ICS09 handshake done sender: [6:7574435781566788373:2108] self: [6:7574435781566788382:0] peer: [2:7574435781069129505:0] socket: 369 qp: -1
2025-11-19T13:59:18.700706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435781566788392:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:18.700715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:18.701045Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435781566788374:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:18.701235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435781566788309:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.701244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:18.701253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435781566788309:2050] [node 3] ICP22 created new session: [6:7574435781566788394:2050]
2025-11-19T13:59:18.701258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435781566788394:2050] [node 3] ICS09 handshake done sender: [6:7574435781566788374:2109] self: [6:7574435781566788381:0] peer: [3:7574435783698763609:0] socket: 370 qp: -1
2025-11-19T13:59:18.701264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435781566788394:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:18.701270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:18.701445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435781566788390:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14903
2025-11-19T13:59:18.701489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435781566788391:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14903
2025-11-19T13:59:18.703831Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435781709302633:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:18.708576Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435782577974974:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3635) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435782577975053:2116] held: no
2025-11-19T13:59:18.709510Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435781069129512:2112] [node 4] ICP35 event processing took too much time PendingConnection:278(39981317ns) {HandleHandshakeStatus:329(39980547ns) {DropIncomingHandshake:441(4895ns) DropOutgoingHandshake:456(119ns) SwitchToState:178(213ns) ProcessPendingSessionEvents:495(39892598ns) {ForwardSessionEventToSession:612(39878870ns) ForwardSessionEventToSession:612(2618ns) ForwardSessionEventToSession:612(700ns) ForwardSessionEventToSession:612(988ns) ForwardSessionEventToSession:612(571ns) ForwardSessionEventToSession:612(899ns) ForwardSessionEventToSession:612(949ns) ForwardSessionEventToSession:612(580ns) ForwardSessionEventToSession:612(509ns) ForwardSessionEventToSession:612(747ns)}}}
2025-11-19T13:59:18.709704Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435781069129439:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2668) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435781069129503:2109] held: no
2025-11-19T13:59:18.710429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435783698763629:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.710478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435783698763543:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.712289Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435781566788391:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.712323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435781566788307:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.720087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435781069129534:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26651
2025-11-19T13:59:18.720376Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435783698763605:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(68499579ns) {HandleHandshakeStatus:329(68498486ns) {DropIncomingHandshake:441(748ns) DropOutgoingHandshake:456(721ns) SwitchToState:178(344ns) ProcessPendingSessionEvents:495(32385590ns) {ForwardSessionEventToSession:612(32373591ns) ForwardSessionEventToSession:612(2321ns) ForwardSessionEventToSession:612(1135ns) ForwardSessionEventToSession:612(835ns) ForwardSessionEventToSession:612(854ns) ForwardSessionEventToSession:612(1346ns) ForwardSessionEventToSession:612(574ns)}}}
2025-11-19T13:59:18.720813Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435781069129503:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:18.720893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435781069129439:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.720911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:18.720917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435781069129439:2048] [node 1] ICP22 created new session: [2:7574435781069129535:2048]
2025-11-19T13:59:18.720924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435781069129535:2048] [node 1] ICS09 handshake done sender: [2:7574435781069129503:2109] self: [2:7574435781069129516:0] peer: [1:7574435783994401871:0] socket: 361 qp: -1
2025-11-19T13:59:18.720931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435781069129535:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:18.720938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:18.731874Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435783994401885:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.732004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782577975058:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6489
2025-11-19T13:59:18.732436Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435781709302657:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.732505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435781709302573:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.732523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:18.732529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435781709302573:2049] [node 2] ICP22 created new session: [4:7574435781709302660:2049]
2025-11-19T13:59:18.732537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435781709302660:2049] [node 2] ICS09 handshake done sender: [4:7574435781709302643:2112] self: [4:7574435781709302654:0] peer: [2:7574435781069129513:0] socket: 383 qp: -1
2025-11-19T13:59:18.732543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435781709302660:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:18.732552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:18.732677Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435781069129521:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.732693Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435781069129521:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.735818Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435783994401890:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.735835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435783994401890:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.736059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435783994401875:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.736079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:18.736087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435783994401875:2108] [node 5] ICP22 created new session: [1:7574435783994401910:2108]
2025-11-19T13:59:18.736094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435783994401910:2108] [node 5] ICS09 handshake done sender: [1:7574435783994401885:2112] self: [1:7574435783994401878:0] peer: [5:7574435782577975055:0] socket: 389 qp: -1
2025-11-19T13:59:18.736100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435783994401910:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:18.736109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.737594Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435781566788390:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.737731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435781566788384:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.737752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435781566788397:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14903
2025-11-19T13:59:18.737036Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435782577975053:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.739927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435781709302572:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.739948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:18.739957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435781709302572:2048] [node 1] ICP22 created new session: [4:7574435781709302662:2048]
2025-11-19T13:59:18.739964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435781709302662:2048] [node 1] ICS09 handshake done sender: [4:7574435781709302633:2108] self: [4:7574435781709302640:0] peer: [1:7574435783994401879:0] socket: 365 qp: -1
2025-11-19T13:59:18.739971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435781709302662:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:18.739979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:18.740277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435781709302651:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.740281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435781709302651:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.741260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435781709302664:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
2025-11-19T13:59:18.741494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435782577974974:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.741512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:18.741518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435782577974974:2050] [node 3] ICP22 created new session: [5:7574435782577975061:2050]
2025-11-19T13:59:18.741524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435782577975061:2050] [node 3] ICS09 handshake done sender: [5:7574435782577975053:2116] self: [5:7574435782577975056:0] peer: [3:7574435783698763618:0] socket: 404 qp: -1
2025-11-19T13:59:18.741529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435782577975061:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:18.741536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:18.741418Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435783994401890:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:18.746366Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435781709302651:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:18.747503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782577975059:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6489
2025-11-19T13:59:18.747604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782577975060:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6489
2025-11-19T13:59:18.748965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435783994401888:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.748989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:18.748997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435783994401888:2114] [node 6] ICP22 created new session: [1:7574435783994401913:2114]
2025-11-19T13:59:18.749004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435783994401913:2114] [node 6] ICS09 handshake done sender: [1:7574435783994401890:2115] self: [1:7574435783994401889:0] peer: [6:7574435781566788396:0] socket: 410 qp: -1
2025-11-19T13:59:18.749010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435783994401913:2114] [node 6] ICS10 traffic start
2025-11-19T13:59:18.749019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:18.749188Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435781709302664:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.749202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435781709302648:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.749213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:18.749220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435781709302648:2114] [node 6] ICP22 created new session: [4:7574435781709302668:2114]
2025-11-19T13:59:18.749224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435781709302668:2114] [node 6] ICS09 handshake done sender: [4:7574435781709302651:2116] self: [4:7574435781709302649:0] peer: [6:7574435781566788398:0] socket: 405 qp: -1
2025-11-19T13:59:18.749228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435781709302668:2114] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.749523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435781566788399:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14903
2025-11-19T13:59:18.749233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:18.748447Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435782577975059:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.748837Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435782577975052:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:18.748910Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435782577974972:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2668) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435782577975052:2115] held: no
2025-11-19T13:59:18.748931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435782577974972:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.748942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:18.748949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435782577974972:2048] [node 1] ICP22 created new session: [5:7574435782577975063:2048]
2025-11-19T13:59:18.748955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435782577975063:2048] [node 1] ICS09 handshake done sender: [5:7574435782577975052:2115] self: [5:7574435782577975055:0] peer: [1:7574435783994401878:0] socket: 390 qp: -1
2025-11-19T13:59:18.748961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435782577975063:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:18.748969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:18.749077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435782577974973:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.749731Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435781069129521:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:18.749886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435781069129440:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.749909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:18.749916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435781069129440:2049] [node 3] ICP22 created new session: [2:7574435781069129538:2049]
2025-11-19T13:59:18.749923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435781069129538:2049] [node 3] ICS09 handshake done sender: [2:7574435781069129521:2117] self: [2:7574435781069129514:0] peer: [3:7574435783698763639:0] socket: 407 qp: -1
2025-11-19T13:59:18.749929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435781069129538:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:18.749957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:18.750285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435781566788307:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2668) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435781566788391:2118] held: no
2025-11-19T13:59:18.750313Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435781566788391:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:18.749755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435781709302573:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(17219713ns) {HandleHandshakeStatus:329(17218989ns) {DropIncomingHandshake:441(453ns) DropOutgoingHandshake:456(2277ns) SwitchToState:178(129ns) ProcessPendingSessionEvents:495(17131339ns) {ForwardSessionEventToSession:612(17120438ns) ForwardSessionEventToSession:612(1694ns) ForwardSessionEventToSession:612(745ns) ForwardSessionEventToSession:612(632ns) ForwardSessionEventToSession:612(554ns) ForwardSessionEventToSession:612(929ns) ForwardSessionEventToSession:612(614ns) ForwardSessionEventToSession:612(551ns) ForwardSessionEventToSession:612(558ns) ForwardSessionEventToSession:612(882ns)}}}
2025-11-19T13:59:18.749809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435781709302573:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:18.751931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435781566788307:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.751946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:18.751953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435781566788307:2048] [node 1] ICP22 created new session: [6:7574435781566788400:2048]
2025-11-19T13:59:18.751959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435781566788400:2048] [node 1] ICS09 handshake done sender: [6:7574435781566788391:2118] self: [6:7574435781566788396:0] peer: [1:7574435783994401889:0] socket: 385 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.751972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435781566788400:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:18.751991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435781566788384:2113] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14766) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435781566788390:2117] held: no
2025-11-19T13:59:18.752014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:18.752033Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435781566788390:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:18.752239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435783698763640:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13070
2025-11-19T13:59:18.752498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435781566788384:2113] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.752511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:18.752516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435781566788384:2113] [node 4] ICP22 created new session: [6:7574435781566788402:2113]
2025-11-19T13:59:18.752521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435781566788402:2113] [node 4] ICS09 handshake done sender: [6:7574435781566788390:2117] self: [6:7574435781566788398:0] peer: [4:7574435781709302649:0] socket: 409 qp: -1
2025-11-19T13:59:18.752526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435781566788402:2113] [node 4] ICS10 traffic start
2025-11-19T13:59:18.752532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.753413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782577975066:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6489
2025-11-19T13:59:18.753712Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435783698763629:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:18.753666Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435782577975066:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.753709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435782577975050:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:18.753816Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435781709302659:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.753826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435781709302659:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.753233Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435781069129522:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:18.753250Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435781069129522:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:18.754998Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435781709302659:2120] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:18.755152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435781709302645:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.755168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:18.755176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435781709302645:2113] [node 5] ICP22 created new session: [4:7574435781709302672:2113]
2025-11-19T13:59:18.755182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435781709302672:2113] [node 5] ICS09 handshake done sender: [4:7574435781709302659:2120] self: [4:7574435781709302646:0] peer: [5:7574435782577975067:0] socket: 416 qp: -1
2025-11-19T13:59:18.755187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435781709302672:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:18.755194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:18.760050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435782577975050:2114] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14766) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435782577975066:2121] held: no
2025-11-19T13:59:18.755472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435783698763543:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.755497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:18.755502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435783698763543:2049] [node 2] ICP22 created new session: [3:7574435783698763641:2049]
2025-11-19T13:59:18.755508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435783698763641:2049] [node 2] ICS09 handshake done sender: [3:7574435783698763629:2118] self: [3:7574435783698763639:0] peer: [2:7574435781069129514:0] socket: 408 qp: -1
2025-11-19T13:59:18.755514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435783698763641:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:18.755519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:18.763928Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435781069129522:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:18.768339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782577975068:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6489
2025-11-19T13:59:18.768428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435782577975069:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6489
2025-11-19T13:59:18.773746Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435782577975066:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:18.773867Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435782577975059:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:18.774029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435781069129501:2105] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:18.774062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:18.774071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435781069129501:2105] [node 5] ICP22 created new session: [2:7574435781069129541:2105]
2025-11-19T13:59:18.774078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435781069129541:2105] [node 5] ICS09 handshake done sender: [2:7574435781069129522:2118] self: [2:7574435781069129504:0] peer: [5:7574435782577975065:0] socket: 412 qp: -1
2025-11-19T13:59:18.774087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435781069129541:2105] [node 5] ICS10 traffic start
2025-11-19T13:59:18.774097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:18.774604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435782577974973:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.774618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:18.774624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435782577974973:2049] [node 2] ICP22 created new session: [5:7574435782577975070:2049]
2025-11-19T13:59:18.774630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435782577975070:2049] [node 2] ICS09 handshake done sender: [5:7574435782577975059:2119] self: [5:7574435782577975065:0] peer: [2:7574435781069129504:0] socket: 366 qp: -1
2025-11-19T13:59:18.774636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435782577975070:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:18.774642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:18.775841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435782577975050:2114] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:18.775854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:18.775861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435782577975050:2114] [node 4] ICP22 created new session: [5:7574435782577975072:2114]
2025-11-19T13:59:18.775867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435782577975072:2114] [node 4] ICS09 handshake done sender: [5:7574435782577975066:2121] self: [5:7574435782577975067:0] peer: [4:7574435781709302646:0] socket: 419 qp: -1
2025-11-19T13:59:18.775872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435782577975072:2114] [node 4] ICS10 traffic start
2025-11-19T13:59:18.775878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:18.792804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:18.792873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435783698763632:2048] [node 1] ICS01 socket: 359 reason# EndOfStream
2025-11-19T13:59:18.792879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435783698763542:2048] [node 1] ICP30 unregister session Session# [3:7574435783698763632:2048] VirtualId# [3:7574435783698763623:0]
2025-11-19T13:59:18.792885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435783698763632:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.792913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:18.792932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:18.791952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:18.792047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435781069129535:2048] [node 1] ICS01 socket: 361 reason# EndOfStream
2025-11-19T13:59:18.792057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435781069129439:2048] [node 1] ICP30 unregister session Session# [2:7574435781069129535:2048] VirtualId# [2:7574435781069129516:0]
2025-11-19T13:59:18.792064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435781069129535:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.792101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:18.792122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:18.795930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:18.796015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435781709302662:2048] [node 1] ICS01 socket: 365 reason# EndOfStream
2025-11-19T13:59:18.796022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435781709302572:2048] [node 1] ICP30 unregister session Session# [4:7574435781709302662:2048] VirtualId# [4:7574435781709302640:0]
2025-11-19T13:59:18.796030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435781709302662:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.796092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:18.796118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:18.796164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:18.796232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435781566788400:2048] [node 1] ICS01 socket: 385 reason# EndOfStream
2025-11-19T13:59:18.796237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435781566788307:2048] [node 1] ICP30 unregister session Session# [6:7574435781566788400:2048] VirtualId# [6:7574435781566788396:0]
2025-11-19T13:59:18.796242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435781566788400:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.796282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:18.796305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:18.798679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:18.798760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435782577975063:2048] [node 1] ICS01 socket: 390 reason# EndOfStream
2025-11-19T13:59:18.798767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435782577974972:2048] [node 1] ICP30 unregister session Session# [5:7574435782577975063:2048] VirtualId# [5:7574435782577975055:0]
2025-11-19T13:59:18.798773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435782577975063:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.798822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:18.798844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:18.801717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:18.801807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435783698763641:2049] [node 2] ICS01 socket: 408 reason# EndOfStream
2025-11-19T13:59:18.801815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435783698763543:2049] [node 2] ICP30 unregister session Session# [3:7574435783698763641:2049] VirtualId# [3:7574435783698763639:0]
2025-11-19T13:59:18.801823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435783698763641:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.801860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:18.801881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:18.812275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:18.812367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435782577975070:2049] [node 2] ICS01 socket: 366 reason# EndOfStream
2025-11-19T13:59:18.812375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435782577974973:2049] [node 2] ICP30 unregister session Session# [5:7574435782577975070:2049] VirtualId# [5:7574435782577975065:0]
2025-11-19T13:59:18.812382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435782577975070:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.812442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:18.812477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:18.812031Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:18.812125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435781566788392:2049] [node 2] ICS01 socket: 369 reason# EndOfStream
2025-11-19T13:59:18.812133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435781566788308:2049] [node 2] ICP30 unregister session Session# [6:7574435781566788392:2049] VirtualId# [6:7574435781566788382:0]
2025-11-19T13:59:18.812141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435781566788392:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.812211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:18.812250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:18.812292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:18.812359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435781709302660:2049] [node 2] ICS07 socket disconnect 383 reason# EndOfStream
2025-11-19T13:59:18.812363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435781709302660:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.812392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:18.812412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435781709302660:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:18.813013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435781709302573:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7016) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:18.813034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435781709302660:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:18.813040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435781709302573:2049] [node 2] ICP30 unregister session Session# [4:7574435781709302660:2049] VirtualId# [4:7574435781709302654:0]
2025-11-19T13:59:18.813058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:18.813064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435781709302573:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7016) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:18.813067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7016) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:18.823837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435781709302573:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:18.829526Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:18.829632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435782577975061:2050] [node 3] ICS01 socket: 404 reason# EndOfStream
2025-11-19T13:59:18.829642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435782577974974:2050] [node 3] ICP30 unregister session Session# [5:7574435782577975061:2050] VirtualId# [5:7574435782577975056:0]
2025-11-19T13:59:18.829650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435782577975061:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.829725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:18.829752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:18.830202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:18.830283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435781566788394:2050] [node 3] ICS01 socket: 370 reason# EndOfStream
2025-11-19T13:59:18.830290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435781566788309:2050] [node 3] ICP30 unregister session Session# [6:7574435781566788394:2050] VirtualId# [6:7574435781566788381:0]
2025-11-19T13:59:18.830296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435781566788394:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.830347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:18.830371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:18.846744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:18.846836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435781566788402:2113] [node 4] ICS01 socket: 409 reason# EndOfStream
2025-11-19T13:59:18.846845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435781566788384:2113] [node 4] ICP30 unregister session Session# [6:7574435781566788402:2113] VirtualId# [6:7574435781566788398:0]
2025-11-19T13:59:18.846852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435781566788402:2113] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:18.846911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:18.846935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:19.112249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435784879058642:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:19.112263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435784879058643:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:19.184719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435788120263253:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:19.184734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435788120263254:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:19.249007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435786654971792:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:19.249019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435786654971793:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:19.359462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435785780569575:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:19.359478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435785780569576:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:19.359483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435785780569577:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:19.424066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435786665412637:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:19.424079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435786665412638:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:19.424084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435786665412639:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:19.465594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435785707782895:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:19.465608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435785707782896:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:19.465612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435785707782897:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:19.548005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435784879058705:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:19.548072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435784879058706:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:19.548113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435784879058707:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:19.548119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435786665412698:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.549753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435784879058708:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9795
2025-11-19T13:59:19.549249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435785707782950:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57869
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.560155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435785707782957:2104] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.563068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435785780569636:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28198
2025-11-19T13:59:19.563648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435786665412702:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32787
2025-11-19T13:59:19.564018Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435785707782950:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.564210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435788120263321:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:19.564309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435788120263322:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:19.564908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435786654971853:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.575340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435786654971857:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13150
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.575698Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435784879058707:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27447306ns) {Configure:104(27446426ns) {ProcessConfigured:135(22726381ns) {StartInitialHandshake:157(22725527ns) {DropHandshakes:469(434ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(1393ns) {SwitchToState:178(878ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.576956Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435784879058708:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.576538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435788120263325:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6725
2025-11-19T13:59:19.577182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435788120263327:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.579307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435786665412704:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32787
2025-11-19T13:59:19.575846Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435786654971857:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.576279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435786654971859:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:19.579649Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435786665412702:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.580032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435786665412637:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.580090Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435786665412704:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.580317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435786665412638:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.582471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435784879058722:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9795
2025-11-19T13:59:19.583235Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435784879058722:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.583145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435785707782963:2106] [node 4] ICP01 ready to work
2025-11-19T13:59:19.589946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435786654971792:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18299235ns) {Configure:104(18297677ns) {ProcessConfigured:135(17975021ns) {StartInitialHandshake:157(17974384ns) {DropHandshakes:469(616ns) {DropIncomingHandshake:441(123ns) DropOutgoingHandshake:456(127ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(1223ns) {SwitchToState:178(693ns)}}}}}
2025-11-19T13:59:19.590161Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435785780569636:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.591852Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435788120263329:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.591874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435788120263329:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:19.592359Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435788120263325:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.592389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435788120263253:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.591942Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435784879058714:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.591958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435784879058714:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.599856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435785707782957:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(35399188ns) {Configure:104(35397964ns) {ProcessConfigured:135(35058849ns) {StartInitialHandshake:157(35058204ns) {DropHandshakes:469(659ns) {DropIncomingHandshake:441(117ns) DropOutgoingHandshake:456(155ns)} GenerateSessionVirtualId:988(357ns) PrepareNewSessionHandshake:476(2133ns) {SwitchToState:178(261ns)}}}}}
2025-11-19T13:59:19.599917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435785707782957:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.601471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435785707782963:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18258556ns) {Configure:104(18257721ns) {ProcessConfigured:135(17920291ns) {StartInitialHandshake:157(17919733ns) {DropHandshakes:469(787ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(182ns)} GenerateSessionVirtualId:988(290ns) PrepareNewSessionHandshake:476(864ns) {SwitchToState:178(685ns)}}}}}
2025-11-19T13:59:19.602704Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435784879058717:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.602721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435784879058717:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:19.600959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435785780569639:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:19.601916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435785780569640:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28198
2025-11-19T13:59:19.604083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435788120263322:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(39373845ns) {Configure:104(34362479ns) {ProcessConfigured:135(34033129ns) {StartInitialHandshake:157(34032494ns) {DropHandshakes:469(554ns) {DropIncomingHandshake:441(124ns) DropOutgoingHandshake:456(212ns)} GenerateSessionVirtualId:988(445ns) PrepareNewSessionHandshake:476(743ns) {SwitchToState:178(562ns)}}}}}
2025-11-19T13:59:19.600047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435786654971859:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23716129ns) {Configure:104(23714921ns) {ProcessConfigured:135(23393709ns) {StartInitialHandshake:157(23393443ns) {DropHandshakes:469(873ns) {DropIncomingHandshake:441(263ns) DropOutgoingHandshake:456(183ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:476(840ns) {SwitchToState:178(547ns)}}}}}
2025-11-19T13:59:19.603898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435786654971793:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(31832612ns) {Configure:104(31831578ns) {ProcessConfigured:135(31528672ns) {StartInitialHandshake:157(31528148ns) {DropHandshakes:469(921ns) {DropIncomingHandshake:441(165ns) DropOutgoingHandshake:456(284ns)} GenerateSessionVirtualId:988(298ns) PrepareNewSessionHandshake:476(1137ns) {SwitchToState:178(955ns)}}}}}
2025-11-19T13:59:19.603947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435786654971793:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.604044Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435786665412700:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.604059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435786665412700:2108] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.609167Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435784879058714:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:19.609270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435784879058705:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.609283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:19.609293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435784879058705:2107] [node 5] ICP22 created new session: [1:7574435784879058726:2107]
2025-11-19T13:59:19.609300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435784879058726:2107] [node 5] ICS09 handshake done sender: [1:7574435784879058714:2112] self: [1:7574435784879058712:0] peer: [5:7574435786665412705:0] socket: 368 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.609306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435784879058726:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:19.609317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:19.611234Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435784879058719:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.609651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435786654971871:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13150
2025-11-19T13:59:19.611238Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435784879058719:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:19.611103Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435786654971871:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.611136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435786654971792:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.611740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435786654971873:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13150
2025-11-19T13:59:19.611851Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435788120263329:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:19.611979Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435788120263314:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.611983Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435788120263314:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:19.611886Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435784879058710:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.611897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435784879058710:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:19.611899Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435784879058719:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:19.612043Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435786654971871:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:19.612063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435786654971868:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:19.612128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435788120263327:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.612142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:19.612149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435788120263327:2112] [node 5] ICP22 created new session: [2:7574435788120263335:2112]
2025-11-19T13:59:19.612155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435788120263335:2112] [node 5] ICS09 handshake done sender: [2:7574435788120263329:2113] self: [2:7574435788120263328:0] peer: [5:7574435786665412706:0] socket: 374 qp: -1
2025-11-19T13:59:19.612162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435788120263335:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:19.612183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:19.611977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435785780569575:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.612025Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435785780569575:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21791359ns) {Configure:104(21790646ns) {ProcessConfigured:135(21357418ns) {IncomingHandshake:261(21348319ns) {DropIncomingHandshake:441(188ns) PrepareNewSessionHandshake:476(355ns) {SwitchToState:178(88ns)} GenerateSessionVirtualId:988(364ns) IssueIncomingHandshakeReply:193(21344238ns) {SwitchToState:178(239ns)}}}}}
2025-11-19T13:59:19.612295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435786654971792:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.612308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:19.612314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435786654971792:2048] [node 1] ICP22 created new session: [3:7574435786654971874:2048]
2025-11-19T13:59:19.612319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435786654971874:2048] [node 1] ICS09 handshake done sender: [3:7574435786654971871:2115] self: [3:7574435786654971872:0] peer: [1:7574435784879058718:0] socket: 388 qp: -1
2025-11-19T13:59:19.612325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435786654971874:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:19.612357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:19.615794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435784879058643:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.615816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:19.615826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435784879058643:2049] [node 3] ICP22 created new session: [1:7574435784879058730:2049]
2025-11-19T13:59:19.615832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435784879058730:2049] [node 3] ICS09 handshake done sender: [1:7574435784879058719:2114] self: [1:7574435784879058718:0] peer: [3:7574435786654971872:0] socket: 387 qp: -1
2025-11-19T13:59:19.615840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435784879058730:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:19.615864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.616090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435785780569642:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28198
2025-11-19T13:59:19.616109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435785780569643:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28198
2025-11-19T13:59:19.617701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435785780569645:2113] [node 5] ICP01 ready to work
2025-11-19T13:59:19.617283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435786665412708:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.620763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435785707782971:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57869
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.621783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435788120263338:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6725
2025-11-19T13:59:19.621808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435788120263339:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6725
2025-11-19T13:59:19.621903Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435784879058717:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:19.622073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435784879058642:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.622095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:19.622102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435784879058642:2048] [node 2] ICP22 created new session: [1:7574435784879058734:2048]
2025-11-19T13:59:19.622109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435784879058734:2048] [node 2] ICS09 handshake done sender: [1:7574435784879058717:2113] self: [1:7574435784879058716:0] peer: [2:7574435788120263332:0] socket: 370 qp: -1
2025-11-19T13:59:19.622114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435784879058734:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:19.622122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:19.622569Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435788120263325:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:19.622688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435788120263253:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.622703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:19.622709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435788120263253:2048] [node 1] ICP22 created new session: [2:7574435788120263340:2048]
2025-11-19T13:59:19.622714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435788120263340:2048] [node 1] ICS09 handshake done sender: [2:7574435788120263325:2111] self: [2:7574435788120263332:0] peer: [1:7574435784879058716:0] socket: 373 qp: -1
2025-11-19T13:59:19.622720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435788120263340:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:19.622739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.625310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435785707782972:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57869
2025-11-19T13:59:19.618700Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435785780569642:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.618759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435785780569653:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28198
2025-11-19T13:59:19.621092Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435785780569653:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.627098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435784879058721:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.627107Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435784879058721:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:19.627726Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435784879058721:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:19.628694Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435786665412700:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:19.628488Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435788120263333:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.628497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435788120263333:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:19.626926Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435785707782971:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.626978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435785707782895:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.627959Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435785707782972:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.628344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435785707782896:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.627959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435785780569639:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26934178ns) {Configure:104(26932808ns) {ProcessConfigured:135(26599499ns) {StartInitialHandshake:157(26597905ns) {DropHandshakes:469(1071ns) {DropIncomingHandshake:441(117ns) DropOutgoingHandshake:456(136ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:476(634ns) {SwitchToState:178(198ns)}}}}}
2025-11-19T13:59:19.630191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435784879058707:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.630211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:19.630217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435784879058707:2109] [node 6] ICP22 created new session: [1:7574435784879058737:2109]
2025-11-19T13:59:19.630222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435784879058737:2109] [node 6] ICS09 handshake done sender: [1:7574435784879058721:2115] self: [1:7574435784879058713:0] peer: [6:7574435785707782973:0] socket: 399 qp: -1
2025-11-19T13:59:19.630228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435784879058737:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:19.630235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:19.631957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435786654971792:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(19633920ns) {HandleHandshakeStatus:329(19633148ns) {DropIncomingHandshake:441(165ns) DropOutgoingHandshake:456(1617ns) SwitchToState:178(111ns) ProcessPendingSessionEvents:495(42256ns) {ForwardSessionEventToSession:612(31891ns) ForwardSessionEventToSession:612(784ns) ForwardSessionEventToSession:612(869ns) ForwardSessionEventToSession:612(587ns) ForwardSessionEventToSession:612(603ns) ForwardSessionEventToSession:612(1191ns) ForwardSessionEventToSession:612(763ns) ForwardSessionEventToSession:612(531ns) ForwardSessionEventToSession:612(683ns) ForwardSessionEventToSession:612(1368ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.633596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435784879058739:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9795
2025-11-19T13:59:19.633739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435785707782895:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17190) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435785707782971:2114] held: no
2025-11-19T13:59:19.633987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435786665412698:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.634006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:19.634015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435786665412698:2107] [node 6] ICP22 created new session: [5:7574435786665412716:2107]
2025-11-19T13:59:19.634032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435786665412716:2107] [node 6] ICS09 handshake done sender: [5:7574435786665412700:2108] self: [5:7574435786665412699:0] peer: [6:7574435785707782967:0] socket: 360 qp: -1
2025-11-19T13:59:19.634040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435786665412716:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:19.634071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.635172Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435788120263314:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:19.635293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435788120263254:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.635305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:19.635311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435788120263254:2049] [node 3] ICP22 created new session: [2:7574435788120263344:2049]
2025-11-19T13:59:19.635317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435788120263344:2049] [node 3] ICS09 handshake done sender: [2:7574435788120263314:2107] self: [2:7574435788120263313:0] peer: [3:7574435786654971870:0] socket: 361 qp: -1
2025-11-19T13:59:19.635323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435788120263344:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:19.635329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.634853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435786665412718:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32787
2025-11-19T13:59:19.634901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435786665412719:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32787
2025-11-19T13:59:19.634925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435786665412720:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32787
2025-11-19T13:59:19.636016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435786665412721:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32787
2025-11-19T13:59:19.636056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435786665412722:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32787
2025-11-19T13:59:19.638557Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435786665412702:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:19.634556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435786654971877:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13150
2025-11-19T13:59:19.634641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435786654971878:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13150
2025-11-19T13:59:19.634997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435786654971868:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(22888156ns) {Configure:104(22887558ns) {ProcessConfigured:135(22562294ns) {StartInitialHandshake:157(22561690ns) {DropHandshakes:469(413ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(256ns) PrepareNewSessionHandshake:476(543ns) {SwitchToState:178(118ns)}}}}}
2025-11-19T13:59:19.638838Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435786654971877:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.638713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435786665412637:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.638727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:19.638733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435786665412637:2048] [node 1] ICP22 created new session: [5:7574435786665412723:2048]
2025-11-19T13:59:19.638739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435786665412723:2048] [node 1] ICS09 handshake done sender: [5:7574435786665412702:2109] self: [5:7574435786665412705:0] peer: [1:7574435784879058712:0] socket: 371 qp: -1
2025-11-19T13:59:19.638744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435786665412723:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:19.638765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:19.638945Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435786665412704:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:19.639171Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435786665412719:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.639194Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435786665412722:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.639215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435786665412639:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.639225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435786665412708:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.639530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435785707782975:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57869
2025-11-19T13:59:19.639630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435785707782976:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57869
2025-11-19T13:59:19.639407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435786665412638:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.639419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:19.639423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435786665412638:2049] [node 2] ICP22 created new session: [5:7574435786665412727:2049]
2025-11-19T13:59:19.639427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435786665412727:2049] [node 2] ICS09 handshake done sender: [5:7574435786665412704:2111] self: [5:7574435786665412706:0] peer: [2:7574435788120263328:0] socket: 375 qp: -1
2025-11-19T13:59:19.639432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435786665412727:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:19.639440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.640531Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435786654971865:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.640543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435786654971865:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:19.641141Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435786654971857:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:19.641818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435786654971793:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.641835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:19.641841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435786654971793:2049] [node 2] ICP22 created new session: [3:7574435786654971882:2049]
2025-11-19T13:59:19.641846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435786654971882:2049] [node 2] ICS09 handshake done sender: [3:7574435786654971857:2108] self: [3:7574435786654971870:0] peer: [2:7574435788120263313:0] socket: 367 qp: -1
2025-11-19T13:59:19.641851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435786654971882:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:19.641858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.642550Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435786654971865:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:19.642671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435786654971859:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.642690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:19.642695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435786654971859:2109] [node 5] ICP22 created new session: [3:7574435786654971885:2109]
2025-11-19T13:59:19.642700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435786654971885:2109] [node 5] ICS09 handshake done sender: [3:7574435786654971865:2112] self: [3:7574435786654971860:0] peer: [5:7574435786665412725:0] socket: 381 qp: -1
2025-11-19T13:59:19.642705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435786654971885:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:19.642711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:19.640322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435785707782978:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57869
2025-11-19T13:59:19.641400Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435785707782971:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:19.641482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435785707782895:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.641492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:19.641499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435785707782895:2048] [node 1] ICP22 created new session: [6:7574435785707782979:2048]
2025-11-19T13:59:19.641505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435785707782979:2048] [node 1] ICS09 handshake done sender: [6:7574435785707782971:2114] self: [6:7574435785707782973:0] peer: [1:7574435784879058713:0] socket: 400 qp: -1
2025-11-19T13:59:19.641511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435785707782979:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:19.641518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:19.642404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435785707782981:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57869
2025-11-19T13:59:19.643889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435788120263327:2112] [node 5] ICP35 event processing took too much time PendingConnection:278(31726012ns) {HandleHandshakeStatus:329(31724798ns) {DropIncomingHandshake:441(96ns) DropOutgoingHandshake:456(303ns) SwitchToState:178(72ns) ProcessPendingSessionEvents:495(14127612ns) {ForwardSessionEventToSession:612(14114732ns) ForwardSessionEventToSession:612(1770ns) ForwardSessionEventToSession:612(1069ns) ForwardSessionEventToSession:612(535ns) ForwardSessionEventToSession:612(924ns) ForwardSessionEventToSession:612(923ns) ForwardSessionEventToSession:612(508ns) ForwardSessionEventToSession:612(553ns) ForwardSessionEventToSession:612(697ns) ForwardSessionEventToSession:612(821ns)}}}
2025-11-19T13:59:19.643839Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435785780569650:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.643855Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435785780569650:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:19.644586Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435786665412718:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.643970Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435788120263333:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:19.644684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435785707782977:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57869
2025-11-19T13:59:19.645233Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435785707782976:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.645251Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435785707782972:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:19.645372Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435785780569643:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.647313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435786665412639:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24115) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435786665412719:2116] held: no
2025-11-19T13:59:19.647439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435786665412698:2107] [node 6] ICP18 handshake (actor [5:7574435786665412718:2115]) is too old
2025-11-19T13:59:19.647887Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435784879058710:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:19.648011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435784879058706:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.648022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:19.648028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435784879058706:2108] [node 4] ICP22 created new session: [1:7574435784879058741:2108]
2025-11-19T13:59:19.648033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435784879058741:2108] [node 4] ICS09 handshake done sender: [1:7574435784879058710:2111] self: [1:7574435784879058709:0] peer: [4:7574435785780569637:0] socket: 365 qp: -1
2025-11-19T13:59:19.648039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435784879058741:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:19.648046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:19.654082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435784879058707:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(23853196ns) {HandleHandshakeStatus:329(23852642ns) {DropIncomingHandshake:441(3649ns) DropOutgoingHandshake:456(120ns) SwitchToState:178(149ns) ProcessPendingSessionEvents:495(23765511ns) {ForwardSessionEventToSession:612(23752027ns) ForwardSessionEventToSession:612(2333ns) ForwardSessionEventToSession:612(573ns) ForwardSessionEventToSession:612(1323ns) ForwardSessionEventToSession:612(655ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(693ns) ForwardSessionEventToSession:612(876ns) ForwardSessionEventToSession:612(820ns) ForwardSessionEventToSession:612(784ns)}}}
2025-11-19T13:59:19.654327Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435785780569577:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(36655932ns) {Configure:104(36655115ns) {ProcessConfigured:135(36320719ns) {StartInitialHandshake:157(36319114ns) {DropHandshakes:469(519ns) {DropIncomingHandshake:441(122ns) DropOutgoingHandshake:456(85ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:476(996ns) {SwitchToState:178(730ns)}}}}}
2025-11-19T13:59:19.654989Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435788120263331:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.655002Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435788120263331:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:19.656253Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435788120263331:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:19.656387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435788120263321:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.656403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:19.656409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435788120263321:2109] [node 4] ICP22 created new session: [2:7574435788120263348:2109]
2025-11-19T13:59:19.656416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435788120263348:2109] [node 4] ICS09 handshake done sender: [2:7574435788120263331:2114] self: [2:7574435788120263323:0] peer: [4:7574435785780569658:0] socket: 391 qp: -1
2025-11-19T13:59:19.656422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435788120263348:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:19.656443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:19.657277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435788120263338:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.657303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435788120263321:2109] [node 4] ICP18 handshake (actor [2:7574435788120263338:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.660460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435786654971887:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13150
2025-11-19T13:59:19.664060Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435785780569640:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.664184Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435785780569650:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:19.664240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435785780569577:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.664485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435786654971887:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.664748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435785780569645:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.664769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:19.664775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435785780569645:2113] [node 5] ICP22 created new session: [4:7574435785780569663:2113]
2025-11-19T13:59:19.664781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435785780569663:2113] [node 5] ICS09 handshake done sender: [4:7574435785780569650:2115] self: [4:7574435785780569649:0] peer: [5:7574435786665412726:0] socket: 396 qp: -1
2025-11-19T13:59:19.664787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435785780569663:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:19.664797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.668286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435786665412729:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32787
2025-11-19T13:59:19.668391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435786665412730:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32787
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.670425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435785780569665:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28198
2025-11-19T13:59:19.670624Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435786665412722:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:19.670842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435786665412708:2112] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.670859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:19.670866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435786665412708:2112] [node 4] ICP22 created new session: [5:7574435786665412731:2112]
2025-11-19T13:59:19.670872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435786665412731:2112] [node 4] ICS09 handshake done sender: [5:7574435786665412722:2119] self: [5:7574435786665412726:0] peer: [4:7574435785780569649:0] socket: 412 qp: -1
2025-11-19T13:59:19.670878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435786665412731:2112] [node 4] ICS10 traffic start
2025-11-19T13:59:19.670887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.672070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435785707782896:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.672092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:19.672097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435785707782896:2049] [node 2] ICP22 created new session: [6:7574435785707782984:2049]
2025-11-19T13:59:19.672102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435785707782984:2049] [node 2] ICS09 handshake done sender: [6:7574435785707782972:2115] self: [6:7574435785707782974:0] peer: [2:7574435788120263324:0] socket: 364 qp: -1
2025-11-19T13:59:19.672102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435785707782963:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.672110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435785707782984:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:19.672145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:19.672540Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435786654971861:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.672558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435786654971861:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:19.675443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435785780569576:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.675486Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435785780569576:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(27298657ns) {IncomingHandshake:261(27297999ns) {DropIncomingHandshake:441(151ns) PrepareNewSessionHandshake:476(558ns) {SwitchToState:178(270ns)} GenerateSessionVirtualId:988(213ns) IssueIncomingHandshakeReply:193(27291032ns) {SwitchToState:178(867ns)}}}
2025-11-19T13:59:19.675519Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435785780569576:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17690) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435785780569643:2112] held: no
2025-11-19T13:59:19.675540Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435785780569636:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:19.675789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435785780569666:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28198
2025-11-19T13:59:19.672826Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435786665412719:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:19.675991Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435785707782950:2102] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:19.679378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435786665412639:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.679399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:19.679408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435786665412639:2050] [node 3] ICP22 created new session: [5:7574435786665412733:2050]
2025-11-19T13:59:19.679415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435786665412733:2050] [node 3] ICS09 handshake done sender: [5:7574435786665412719:2116] self: [5:7574435786665412725:0] peer: [3:7574435786654971860:0] socket: 409 qp: -1
2025-11-19T13:59:19.679422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435786665412733:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:19.679430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:19.679834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435788120263322:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.679858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:19.679868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435788120263322:2110] [node 6] ICP22 created new session: [2:7574435788120263350:2110]
2025-11-19T13:59:19.679875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435788120263350:2110] [node 6] ICS09 handshake done sender: [2:7574435788120263333:2115] self: [2:7574435788120263324:0] peer: [6:7574435785707782974:0] socket: 363 qp: -1
2025-11-19T13:59:19.679881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435788120263350:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:19.679895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:19.679992Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435788120263322:2110] [node 6] ICP35 event processing took too much time PendingConnection:278(35259920ns) {HandleHandshakeStatus:329(35259308ns) {DropIncomingHandshake:441(307ns) DropOutgoingHandshake:456(445ns) SwitchToState:178(66ns) ProcessPendingSessionEvents:495(30512ns) {ForwardSessionEventToSession:612(21514ns) ForwardSessionEventToSession:612(688ns) ForwardSessionEventToSession:612(919ns) ForwardSessionEventToSession:612(530ns) ForwardSessionEventToSession:612(591ns) ForwardSessionEventToSession:612(917ns) ForwardSessionEventToSession:612(641ns) ForwardSessionEventToSession:612(731ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(1019ns)}}}
2025-11-19T13:59:19.681592Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435785707782957:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4992) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435785707782950:2102] held: no
2025-11-19T13:59:19.681619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435785707782957:2104] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.681628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:19.681633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435785707782957:2104] [node 5] ICP22 created new session: [6:7574435785707782986:2104]
2025-11-19T13:59:19.681638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435785707782986:2104] [node 5] ICS09 handshake done sender: [6:7574435785707782950:2102] self: [6:7574435785707782967:0] peer: [5:7574435786665412699:0] socket: 362 qp: -1
2025-11-19T13:59:19.681644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435785707782986:2104] [node 5] ICS10 traffic start
2025-11-19T13:59:19.681651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:19.681816Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435785707782981:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.691875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435785780569575:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.691903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:19.691914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435785780569575:2048] [node 1] ICP22 created new session: [4:7574435785780569667:2048]
2025-11-19T13:59:19.691923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435785780569667:2048] [node 1] ICS09 handshake done sender: [4:7574435785780569636:2108] self: [4:7574435785780569637:0] peer: [1:7574435784879058709:0] socket: 366 qp: -1
2025-11-19T13:59:19.691934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435785780569667:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:19.691942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:19.696005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435784879058706:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(47950135ns) {HandleHandshakeStatus:329(47949335ns) {DropIncomingHandshake:441(26ns) DropOutgoingHandshake:456(185ns) SwitchToState:178(129ns) ProcessPendingSessionEvents:495(47875676ns) {ForwardSessionEventToSession:612(47860954ns) ForwardSessionEventToSession:612(2552ns) ForwardSessionEventToSession:612(1062ns) ForwardSessionEventToSession:612(776ns) ForwardSessionEventToSession:612(638ns) ForwardSessionEventToSession:612(1465ns) ForwardSessionEventToSession:612(683ns) ForwardSessionEventToSession:612(723ns) ForwardSessionEventToSession:612(474ns) ForwardSessionEventToSession:612(1019ns)}}}
2025-11-19T13:59:19.696471Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435785707782897:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(52424006ns) {Configure:104(52421949ns) {ProcessConfigured:135(52094293ns) {StartInitialHandshake:157(52093498ns) {DropHandshakes:469(461ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(1620ns) {SwitchToState:178(1089ns)}}}}}
2025-11-19T13:59:19.696518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435785707782897:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:19.697402Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435785780569645:2113] [node 5] ICP35 event processing took too much time PendingConnection:278(32628043ns) {HandleHandshakeStatus:329(32627340ns) {DropIncomingHandshake:441(3559ns) DropOutgoingHandshake:456(261ns) SwitchToState:178(34ns) ProcessPendingSessionEvents:495(24737427ns) {ForwardSessionEventToSession:612(24725108ns) ForwardSessionEventToSession:612(1664ns) ForwardSessionEventToSession:612(1283ns) ForwardSessionEventToSession:612(521ns) ForwardSessionEventToSession:612(512ns) ForwardSessionEventToSession:612(631ns) ForwardSessionEventToSession:612(638ns) ForwardSessionEventToSession:612(544ns) ForwardSessionEventToSession:612(763ns) ForwardSessionEventToSession:612(1547ns)}}}
2025-11-19T13:59:19.697377Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435786654971879:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.697390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435786654971879:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:19.697530Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435786654971861:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:19.697672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435786654971853:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.697686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:19.697692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435786654971853:2107] [node 4] ICP22 created new session: [3:7574435786654971890:2107]
2025-11-19T13:59:19.697697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435786654971890:2107] [node 4] ICS09 handshake done sender: [3:7574435786654971861:2110] self: [3:7574435786654971854:0] peer: [4:7574435785780569662:0] socket: 376 qp: -1
2025-11-19T13:59:19.697702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435786654971890:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:19.697712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.702825Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435785780569643:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:19.703192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435785780569669:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28198
2025-11-19T13:59:19.704190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435785780569576:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.704210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:19.704221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435785780569576:2049] [node 2] ICP22 created new session: [4:7574435785780569670:2049]
2025-11-19T13:59:19.704230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435785780569670:2049] [node 2] ICS09 handshake done sender: [4:7574435785780569643:2112] self: [4:7574435785780569658:0] peer: [2:7574435788120263323:0] socket: 392 qp: -1
2025-11-19T13:59:19.704240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435785780569670:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:19.704264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:19.704776Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435785780569640:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:19.707794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435785780569577:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.707815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:19.707824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435785780569577:2050] [node 3] ICP22 created new session: [4:7574435785780569672:2050]
2025-11-19T13:59:19.707832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435785780569672:2050] [node 3] ICS09 handshake done sender: [4:7574435785780569640:2110] self: [4:7574435785780569662:0] peer: [3:7574435786654971854:0] socket: 377 qp: -1
2025-11-19T13:59:19.707840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435785780569672:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:19.707849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:19.708033Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435785780569655:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.708049Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435785780569655:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.714166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435786654971892:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13150
2025-11-19T13:59:19.714627Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435786654971853:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(16904835ns) {HandleHandshakeStatus:329(16904144ns) {DropIncomingHandshake:441(3138ns) DropOutgoingHandshake:456(320ns) SwitchToState:178(152ns) ProcessPendingSessionEvents:495(16832538ns) {ForwardSessionEventToSession:612(16816583ns) ForwardSessionEventToSession:612(2001ns) ForwardSessionEventToSession:612(1013ns) ForwardSessionEventToSession:612(1115ns) ForwardSessionEventToSession:612(884ns) ForwardSessionEventToSession:612(2757ns) ForwardSessionEventToSession:612(1082ns) ForwardSessionEventToSession:612(695ns) ForwardSessionEventToSession:612(741ns) ForwardSessionEventToSession:612(1120ns)}}}
2025-11-19T13:59:19.715934Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435786654971879:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:19.716039Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435786654971892:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:19.716145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435786654971868:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.716161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:19.716168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435786654971868:2113] [node 6] ICP22 created new session: [3:7574435786654971895:2113]
2025-11-19T13:59:19.716175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435786654971895:2113] [node 6] ICS09 handshake done sender: [3:7574435786654971879:2119] self: [3:7574435786654971876:0] peer: [6:7574435785707782989:0] socket: 418 qp: -1
2025-11-19T13:59:19.716181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435786654971895:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:19.716190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:19.718621Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435785780569655:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:19.718769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435785780569639:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:19.718789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:19.718795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435785780569639:2109] [node 6] ICP22 created new session: [4:7574435785780569675:2109]
2025-11-19T13:59:19.718801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435785780569675:2109] [node 6] ICS09 handshake done sender: [4:7574435785780569655:2117] self: [4:7574435785780569641:0] peer: [6:7574435785707782983:0] socket: 405 qp: -1
2025-11-19T13:59:19.718808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435785780569675:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:19.718817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:19.719478Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435785707782897:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24115) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435785707782981:2120] held: no
2025-11-19T13:59:19.719581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435785707782963:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9838) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435785707782976:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:19.720274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435785707782991:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57869
2025-11-19T13:59:19.720318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435785707782992:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57869
2025-11-19T13:59:19.729353Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435785780569577:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(21537114ns) {HandleHandshakeStatus:329(21536264ns) {DropIncomingHandshake:441(275ns) DropOutgoingHandshake:456(2958ns) SwitchToState:178(631ns) ProcessPendingSessionEvents:495(11909222ns) {ForwardSessionEventToSession:612(11893520ns) ForwardSessionEventToSession:612(2686ns) ForwardSessionEventToSession:612(1191ns) ForwardSessionEventToSession:612(581ns) ForwardSessionEventToSession:612(910ns) ForwardSessionEventToSession:612(1466ns) ForwardSessionEventToSession:612(881ns) ForwardSessionEventToSession:612(990ns) ForwardSessionEventToSession:612(697ns) ForwardSessionEventToSession:612(784ns)}}}
2025-11-19T13:59:19.728118Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435785707782981:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:19.728251Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435785707782976:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:19.730123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435785707782963:2106] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.730139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:19.730147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435785707782963:2106] [node 4] ICP22 created new session: [6:7574435785707782993:2106]
2025-11-19T13:59:19.730155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435785707782993:2106] [node 4] ICS09 handshake done sender: [6:7574435785707782976:2117] self: [6:7574435785707782983:0] peer: [4:7574435785780569641:0] socket: 413 qp: -1
2025-11-19T13:59:19.730163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435785707782993:2106] [node 4] ICS10 traffic start
2025-11-19T13:59:19.730172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:19.730547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435785707782897:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:19.730558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:19.730564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435785707782897:2050] [node 3] ICP22 created new session: [6:7574435785707782995:2050]
2025-11-19T13:59:19.730569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435785707782995:2050] [node 3] ICS09 handshake done sender: [6:7574435785707782981:2120] self: [6:7574435785707782989:0] peer: [3:7574435786654971876:0] socket: 419 qp: -1
2025-11-19T13:59:19.730575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435785707782995:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:19.730582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:19.742861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:19.742934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435786654971874:2048] [node 1] ICS01 socket: 388 reason# EndOfStream
2025-11-19T13:59:19.742940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435786654971792:2048] [node 1] ICP30 unregister session Session# [3:7574435786654971874:2048] VirtualId# [3:7574435786654971872:0]
2025-11-19T13:59:19.742945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435786654971874:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:19.742973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:19.742992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:19.742217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:19.742317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435788120263340:2048] [node 1] ICS01 socket: 373 reason# EndOfStream
2025-11-19T13:59:19.742327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435788120263253:2048] [node 1] ICP30 unregister session Session# [2:7574435788120263340:2048] VirtualId# [2:7574435788120263332:0]
2025-11-19T13:59:19.742334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435788120263340:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:19.742375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:19.742411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:19.759929Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:19.760032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435785707782979:2048] [node 1] ICS07 socket disconnect 400 reason# EndOfStream
2025-11-19T13:59:19.760039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435785707782979:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:19.760104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:19.760128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435785707782979:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:19.759875Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:19.760473Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:19.760537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435785780569667:2048] [node 1] ICS07 socket disconnect 366 reason# EndOfStream
2025-11-19T13:59:19.760542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435785780569667:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:19.760572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:19.760591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435785780569667:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:19.760854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435786665412723:2048] [node 1] ICS01 socket: 371 reason# EndOfStream
2025-11-19T13:59:19.760868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435786665412637:2048] [node 1] ICP30 unregister session Session# [5:7574435786665412723:2048] VirtualId# [5:7574435786665412705:0]
2025-11-19T13:59:19.760874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435786665412723:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:19.760901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:19.760924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:19.761185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435785780569575:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17190) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:19.761195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435785780569667:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:19.761200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435785780569575:2048] [node 1] ICP30 unregister session Session# [4:7574435785780569667:2048] VirtualId# [4:7574435785780569637:0]
2025-11-19T13:59:19.761211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:19.761216Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435785780569575:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17190) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:19.761219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17190) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:19.761483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435785707782895:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17190) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:19.761492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435785707782979:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:19.761496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435785707782895:2048] [node 1] ICP30 unregister session Session# [6:7574435785707782979:2048] VirtualId# [6:7574435785707782973:0]
2025-11-19T13:59:19.761504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:19.761508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435785707782895:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17190) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:19.761510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17190) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:19.775856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435785707782895:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:19.775912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435785780569575:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:19.776153Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:19.776249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435786654971882:2049] [node 2] ICS01 socket: 367 reason# EndOfStream
2025-11-19T13:59:19.776258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435786654971793:2049] [node 2] ICP30 unregister session Session# [3:7574435786654971882:2049] VirtualId# [3:7574435786654971870:0]
2025-11-19T13:59:19.776266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435786654971882:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:19.776328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:19.776358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:19.786809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:19.786902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435785780569670:2049] [node 2] ICS01 socket: 392 reason# EndOfStream
2025-11-19T13:59:19.786913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435785780569576:2049] [node 2] ICP30 unregister session Session# [4:7574435785780569670:2049] VirtualId# [4:7574435785780569658:0]
2025-11-19T13:59:19.786920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435785780569670:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:19.786972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:19.786996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:19.787029Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:19.787094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435785707782984:2049] [node 2] ICS01 socket: 364 reason# EndOfStream
2025-11-19T13:59:19.787100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435785707782896:2049] [node 2] ICP30 unregister session Session# [6:7574435785707782984:2049] VirtualId# [6:7574435785707782974:0]
2025-11-19T13:59:19.787104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435785707782984:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:19.787127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:19.787146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:19.787171Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:19.787228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435786665412727:2049] [node 2] ICS01 socket: 375 reason# EndOfStream
2025-11-19T13:59:19.787232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435786665412638:2049] [node 2] ICP30 unregister session Session# [5:7574435786665412727:2049] VirtualId# [5:7574435786665412706:0]
2025-11-19T13:59:19.787236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435786665412727:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:19.787269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:19.787289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:19.809242Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T13:59:19.809347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435785780569672:2050] [node 3] ICS07 socket disconnect 377 reason# EndOfStream
2025-11-19T13:59:19.809354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435785780569672:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:19.809418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:19.809441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435785780569672:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:19.809848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:19.809939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435786665412733:2050] [node 3] ICS01 socket: 409 reason# EndOfStream
2025-11-19T13:59:19.809947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435786665412639:2050] [node 3] ICP30 unregister session Session# [5:7574435786665412733:2050] VirtualId# [5:7574435786665412725:0]
2025-11-19T13:59:19.809953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435786665412733:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:19.809995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:19.810020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:19.810056Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:19.810132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435785707782995:2050] [node 3] ICS01 socket: 419 reason# EndOfStream
2025-11-19T13:59:19.810137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435785707782897:2050] [node 3] ICP30 unregister session Session# [6:7574435785707782995:2050] VirtualId# [6:7574435785707782989:0]
2025-11-19T13:59:19.810143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435785707782995:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:19.810186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:19.810208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:19.810752Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435785780569577:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24115) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:19.810767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435785780569672:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:19.810772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435785780569577:2050] [node 3] ICP30 unregister session Session# [4:7574435785780569672:2050] VirtualId# [4:7574435785780569662:0]
2025-11-19T13:59:19.810788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:19.810793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435785780569577:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24115) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:19.810796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24115) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:19.840801Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:19.840897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435785707782993:2106] [node 4] ICS01 socket: 413 reason# EndOfStream
2025-11-19T13:59:19.840906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435785707782963:2106] [node 4] ICP30 unregister session Session# [6:7574435785707782993:2106] VirtualId# [6:7574435785707782983:0]
2025-11-19T13:59:19.840913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435785707782993:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:19.840979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:19.841006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:20.076012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435790877438930:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:20.076032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435790877438931:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:20.103887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435790684814662:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:20.103901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435790684814663:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:20.104767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435792182905115:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:20.104779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435792182905116:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:20.171110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435789650557489:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:20.171129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435789650557490:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:20.171134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435789650557491:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:20.211217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435790694233964:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:20.211230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435790694233965:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:20.211235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435790694233966:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:20.279861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435789708868578:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:20.279883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435789708868579:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:20.279888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435789708868580:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:20.294128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435790877438994:2104] [node 5] ICP01 ready to work
2025-11-19T13:59:20.294213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435790877438995:2106] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.295914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435790877439004:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:20.295237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435790694233977:2060] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51315
2025-11-19T13:59:20.296126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435789650557523:2077] [node 6] ICP01 ready to work
2025-11-19T13:59:20.296232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435789650557527:2079] [node 5] ICP01 ready to work
2025-11-19T13:59:20.296434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435790684814705:2083] [node 4] ICP01 ready to work
2025-11-19T13:59:20.296517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435790684814706:2087] [node 5] ICP01 ready to work
2025-11-19T13:59:20.295786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435792182905164:2094] [node 4] ICP01 ready to work
2025-11-19T13:59:20.295900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435792182905170:2095] [node 6] ICP01 ready to work
2025-11-19T13:59:20.295988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435792182905172:2098] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.311342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435792182905187:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62836
2025-11-19T13:59:20.311368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435792182905188:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62836
2025-11-19T13:59:20.312592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435790684814723:2092] [node 6] ICP01 ready to work
2025-11-19T13:59:20.313023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435790684814721:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22889
2025-11-19T13:59:20.313036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435790684814722:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22889
2025-11-19T13:59:20.332505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435789650557490:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(35384907ns) {Configure:104(35384051ns) {ProcessConfigured:135(35219188ns) {StartInitialHandshake:157(35218227ns) {DropHandshakes:469(163ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(23ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(1379ns) {SwitchToState:178(714ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.333278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435789650557567:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51011
2025-11-19T13:59:20.333299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435789650557568:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51011
2025-11-19T13:59:20.336556Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435789650557568:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.338276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435790694234037:2070] [node 4] ICP01 ready to work
2025-11-19T13:59:20.338363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435790694234038:2073] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.338894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435789650557491:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.343911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435789708868578:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(44077689ns) {Configure:104(44076559ns) {ProcessConfigured:135(43713768ns) {StartInitialHandshake:157(43712705ns) {DropHandshakes:469(321ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(26ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:476(972ns) {SwitchToState:178(712ns)}}}}}
2025-11-19T13:59:20.344061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435789708868643:2062] [node 5] ICP01 ready to work
2025-11-19T13:59:20.344155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435789708868644:2070] [node 4] ICP01 ready to work
2025-11-19T13:59:20.345260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435790877439009:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33865
2025-11-19T13:59:20.344262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435792182905196:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62836
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.351457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435790694234039:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51315
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.356893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789708868645:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49517
2025-11-19T13:59:20.356910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789708868646:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49517
2025-11-19T13:59:20.357001Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790684814718:2098] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.357016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435790684814718:2098] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.357512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435790877439010:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33865
2025-11-19T13:59:20.359346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435790877439011:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33865
2025-11-19T13:59:20.359424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435790877439012:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33865
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.363092Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435790684814718:2098] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:20.363437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435790684814705:2083] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.363457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:20.363465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435790684814705:2083] [node 4] ICP22 created new session: [3:7574435790684814744:2083]
2025-11-19T13:59:20.363472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435790684814744:2083] [node 4] ICS09 handshake done sender: [3:7574435790684814718:2098] self: [3:7574435790684814717:0] peer: [4:7574435789650557569:0] socket: 383 qp: -1
2025-11-19T13:59:20.363479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435790684814744:2083] [node 4] ICS10 traffic start
2025-11-19T13:59:20.363489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.365282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435789650557570:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51011
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.365199Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792182905196:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.365417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435792182905197:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62836
2025-11-19T13:59:20.362806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435790877439013:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33865
2025-11-19T13:59:20.365666Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435789708868646:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.366758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435789708868644:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.372110Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435789650557567:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.372145Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435789650557529:2084] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.372148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435789650557529:2084] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.372738Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435789650557568:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.373145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435790694234044:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51315
2025-11-19T13:59:20.373183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435790694234045:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51315
2025-11-19T13:59:20.373430Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792182905188:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.373837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435789650557491:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.373859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:20.373865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435789650557491:2050] [node 3] ICP22 created new session: [4:7574435789650557572:2050]
2025-11-19T13:59:20.373871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435789650557572:2050] [node 3] ICS09 handshake done sender: [4:7574435789650557568:2116] self: [4:7574435789650557569:0] peer: [3:7574435790684814717:0] socket: 386 qp: -1
2025-11-19T13:59:20.373878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435789650557572:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:20.373886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:20.374067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435789650557490:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.374421Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792182905175:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.374429Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435792182905175:2103] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.375144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435792182905200:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62836
2025-11-19T13:59:20.376092Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790684814722:2101] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.376535Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435790694234039:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.376562Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435790694234045:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.376585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435790694234037:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.376603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435790694233966:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.376739Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790684814720:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.376745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435790684814720:2099] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.377283Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435790684814720:2099] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:20.377356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435790684814706:2087] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.377373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:20.377388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435790684814706:2087] [node 5] ICP22 created new session: [3:7574435790684814749:2087]
2025-11-19T13:59:20.377393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435790684814749:2087] [node 5] ICS09 handshake done sender: [3:7574435790684814720:2099] self: [3:7574435790684814719:0] peer: [5:7574435790694234047:0] socket: 401 qp: -1
2025-11-19T13:59:20.377400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435790684814749:2087] [node 5] ICS10 traffic start
2025-11-19T13:59:20.377407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:20.378170Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435792182905175:2103] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:20.378284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435792182905164:2094] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.378297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:20.378303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435792182905164:2094] [node 4] ICP22 created new session: [2:7574435792182905202:2094]
2025-11-19T13:59:20.378308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435792182905202:2094] [node 4] ICS09 handshake done sender: [2:7574435792182905175:2103] self: [2:7574435792182905174:0] peer: [4:7574435789650557574:0] socket: 376 qp: -1
2025-11-19T13:59:20.378313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435792182905202:2094] [node 4] ICS10 traffic start
2025-11-19T13:59:20.378319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.378971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435790684814752:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22889
2025-11-19T13:59:20.379686Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435789708868645:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.379866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435790694233966:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26969) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435790694234045:2118] held: no
2025-11-19T13:59:20.379937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435790694234048:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51315
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.380130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435789708868579:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.380157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789708868659:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49517
2025-11-19T13:59:20.380167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789708868660:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49517
2025-11-19T13:59:20.380403Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435790694234044:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.380475Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435790694234045:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:20.380834Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435789708868660:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.380879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435789708868580:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.380971Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790684814721:2100] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.381004Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790684814726:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.381007Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435790684814726:2102] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.381579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435790684814662:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.380574Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435790877439011:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.380702Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435790877439012:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.380804Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435790877439009:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.382112Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435790694233977:2060] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.382122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435790694233966:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.382132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:20.382138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435790694233966:2050] [node 3] ICP22 created new session: [5:7574435790694234050:2050]
2025-11-19T13:59:20.382142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435790694234050:2050] [node 3] ICS09 handshake done sender: [5:7574435790694234045:2118] self: [5:7574435790694234047:0] peer: [3:7574435790684814719:0] socket: 405 qp: -1
2025-11-19T13:59:20.382148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435790694234050:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:20.382153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:20.382286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435790694233964:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.382459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789708868663:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49517
2025-11-19T13:59:20.382544Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435790877439013:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.382570Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435790877438993:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.382572Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435790877438993:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.383045Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435790877438997:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.383048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435790877438997:2109] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.384944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435790694234054:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51315
2025-11-19T13:59:20.385131Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435790684814726:2102] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:20.385334Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435789708868660:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:20.385397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435789708868580:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.385413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:20.385418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435789708868580:2050] [node 3] ICP22 created new session: [6:7574435789708868664:2050]
2025-11-19T13:59:20.385423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435789708868664:2050] [node 3] ICS09 handshake done sender: [6:7574435789708868660:2118] self: [6:7574435789708868662:0] peer: [3:7574435790684814725:0] socket: 414 qp: -1
2025-11-19T13:59:20.385428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435789708868664:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:20.385434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.387197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435790684814723:2092] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.387219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:20.387223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435790684814723:2092] [node 6] ICP22 created new session: [3:7574435790684814756:2092]
2025-11-19T13:59:20.387228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435790684814756:2092] [node 6] ICS09 handshake done sender: [3:7574435790684814726:2102] self: [3:7574435790684814725:0] peer: [6:7574435789708868662:0] socket: 380 qp: -1
2025-11-19T13:59:20.387233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435790684814756:2092] [node 6] ICS10 traffic start
2025-11-19T13:59:20.387240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:20.387392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435790684814755:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22889
2025-11-19T13:59:20.387558Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435790877438993:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:20.387731Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435790684814721:2100] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:20.387824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435790684814662:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.387837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:20.387842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435790684814662:2048] [node 1] ICP22 created new session: [3:7574435790684814758:2048]
2025-11-19T13:59:20.387846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435790684814758:2048] [node 1] ICS09 handshake done sender: [3:7574435790684814721:2100] self: [3:7574435790684814754:0] peer: [1:7574435790877438992:0] socket: 379 qp: -1
2025-11-19T13:59:20.387851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435790684814758:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:20.387864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:20.388557Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435789650557529:2084] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:20.388571Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435790877438997:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:20.388871Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435790694233977:2060] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:20.389124Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792182905197:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.389152Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792182905177:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.389155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435792182905177:2104] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.389812Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792182905187:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.389911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435792182905115:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.390104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435790694233964:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.390119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:20.390124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435790694233964:2048] [node 1] ICP22 created new session: [5:7574435790694234055:2048]
2025-11-19T13:59:20.390129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435790694234055:2048] [node 1] ICS09 handshake done sender: [5:7574435790694233977:2060] self: [5:7574435790694234053:0] peer: [1:7574435790877438996:0] socket: 369 qp: -1
2025-11-19T13:59:20.390134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435790694234055:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:20.390139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:20.391190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435790877438931:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.391206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:20.391210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435790877438931:2049] [node 3] ICP22 created new session: [1:7574435790877439020:2049]
2025-11-19T13:59:20.391215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435790877439020:2049] [node 3] ICS09 handshake done sender: [1:7574435790877438993:2108] self: [1:7574435790877438992:0] peer: [3:7574435790684814754:0] socket: 367 qp: -1
2025-11-19T13:59:20.391220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435790877439020:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:20.391225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:20.391371Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435790877439010:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.391391Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435790877438991:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.391393Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435790877438991:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.391915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435790877438994:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.391926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:20.391930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435790877438994:2104] [node 5] ICP22 created new session: [1:7574435790877439022:2104]
2025-11-19T13:59:20.391935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435790877439022:2104] [node 5] ICS09 handshake done sender: [1:7574435790877438997:2109] self: [1:7574435790877438996:0] peer: [5:7574435790694234053:0] socket: 368 qp: -1
2025-11-19T13:59:20.391939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435790877439022:2104] [node 5] ICS10 traffic start
2025-11-19T13:59:20.391945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.392437Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435789650557532:2086] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.392453Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435789650557532:2086] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.392930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435789650557523:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.392938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:20.392943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435789650557523:2077] [node 6] ICP22 created new session: [4:7574435789650557579:2077]
2025-11-19T13:59:20.392947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435789650557579:2077] [node 6] ICS09 handshake done sender: [4:7574435789650557529:2084] self: [4:7574435789650557528:0] peer: [6:7574435789708868658:0] socket: 387 qp: -1
2025-11-19T13:59:20.392951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435789650557579:2077] [node 6] ICS10 traffic start
2025-11-19T13:59:20.392958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:20.394141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435789650557576:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51011
2025-11-19T13:59:20.394810Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435792182905177:2104] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:20.395135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435792182905170:2095] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.395150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:20.395155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435792182905170:2095] [node 6] ICP22 created new session: [2:7574435792182905208:2095]
2025-11-19T13:59:20.395159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435792182905208:2095] [node 6] ICS09 handshake done sender: [2:7574435792182905177:2104] self: [2:7574435792182905176:0] peer: [6:7574435789708868661:0] socket: 377 qp: -1
2025-11-19T13:59:20.395163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435792182905208:2095] [node 6] ICS10 traffic start
2025-11-19T13:59:20.395168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:20.396208Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790684814752:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.396238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435790684814663:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.397649Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435789708868646:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.397829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435790694234057:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51315
2025-11-19T13:59:20.397867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435790694234059:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51315
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.397999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435789708868579:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29941) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435789708868645:2111] held: no
2025-11-19T13:59:20.398058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435789708868644:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.398067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:20.398071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435789708868644:2070] [node 4] ICP22 created new session: [6:7574435789708868669:2070]
2025-11-19T13:59:20.398075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435789708868669:2070] [node 4] ICS09 handshake done sender: [6:7574435789708868646:2112] self: [6:7574435789708868658:0] peer: [4:7574435789650557528:0] socket: 395 qp: -1
2025-11-19T13:59:20.398080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435789708868669:2070] [node 4] ICS10 traffic start
2025-11-19T13:59:20.398085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:20.398209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789708868667:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49517
2025-11-19T13:59:20.401334Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435790877438991:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:20.401442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435790877438930:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.401456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:20.401460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435790877438930:2048] [node 2] ICP22 created new session: [1:7574435790877439028:2048]
2025-11-19T13:59:20.401465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435790877439028:2048] [node 2] ICS09 handshake done sender: [1:7574435790877438991:2107] self: [1:7574435790877438990:0] peer: [2:7574435792182905207:0] socket: 366 qp: -1
2025-11-19T13:59:20.401470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435790877439028:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:20.401475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.408889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435789650557577:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51011
2025-11-19T13:59:20.408940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435789650557578:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51011
2025-11-19T13:59:20.409568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435789650557523:2077] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:20.409671Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435789650557577:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.409704Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435789650557567:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:20.409936Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435789650557532:2086] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:20.410081Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435790694234039:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:20.410153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435790694234037:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.410165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:20.410169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435790694234037:2070] [node 4] ICP22 created new session: [5:7574435790694234060:2070]
2025-11-19T13:59:20.410174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435790694234060:2070] [node 4] ICS09 handshake done sender: [5:7574435790694234039:2114] self: [5:7574435790694234046:0] peer: [4:7574435789650557531:0] socket: 393 qp: -1
2025-11-19T13:59:20.410180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435790694234060:2070] [node 4] ICS10 traffic start
2025-11-19T13:59:20.410188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:20.410789Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792182905186:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.410799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435792182905186:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.416186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435789650557490:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.416208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:20.416215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435789650557490:2049] [node 2] ICP22 created new session: [4:7574435789650557583:2049]
2025-11-19T13:59:20.416221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435789650557583:2049] [node 2] ICS09 handshake done sender: [4:7574435789650557567:2115] self: [4:7574435789650557574:0] peer: [2:7574435792182905174:0] socket: 385 qp: -1
2025-11-19T13:59:20.416231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435789650557583:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:20.416239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:20.416400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435789650557582:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51011
2025-11-19T13:59:20.416429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435789650557527:2079] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.416435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:20.416438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435789650557527:2079] [node 5] ICP22 created new session: [4:7574435789650557585:2079]
2025-11-19T13:59:20.416442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435789650557585:2079] [node 5] ICS09 handshake done sender: [4:7574435789650557532:2086] self: [4:7574435789650557531:0] peer: [5:7574435790694234046:0] socket: 388 qp: -1
2025-11-19T13:59:20.416446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435789650557585:2079] [node 5] ICS10 traffic start
2025-11-19T13:59:20.416450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:20.416499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435789650557527:2079] [node 5] ICP18 handshake (actor [4:7574435789650557577:2119]) is too old
2025-11-19T13:59:20.417321Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435790694234059:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.417429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435790694233965:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.417743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789708868668:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49517
2025-11-19T13:59:20.417813Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435789708868645:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.417946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435789708868579:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.417957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:20.417964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435789708868579:2049] [node 2] ICP22 created new session: [6:7574435789708868672:2049]
2025-11-19T13:59:20.417969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435789708868672:2049] [node 2] ICS09 handshake done sender: [6:7574435789708868645:2111] self: [6:7574435789708868661:0] peer: [2:7574435792182905176:0] socket: 394 qp: -1
2025-11-19T13:59:20.417973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435789708868672:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:20.417979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:20.418096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789708868671:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49517
2025-11-19T13:59:20.418306Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435789708868668:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.418331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435789708868643:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.417787Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792182905200:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.417826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435792182905115:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18820) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435792182905187:2112] held: no
2025-11-19T13:59:20.417934Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792182905190:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.417938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435792182905190:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.418672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435792182905164:2094] [node 4] ICP18 handshake (actor [2:7574435792182905200:2119]) is too old
2025-11-19T13:59:20.418699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435792182905211:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62836
2025-11-19T13:59:20.418537Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435790694234043:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.418544Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435790694234043:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.419203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435790684814761:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22889
2025-11-19T13:59:20.419214Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435789650557582:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.419233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435789650557489:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.419300Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435789708868671:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.419334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789708868675:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49517
2025-11-19T13:59:20.419345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435789708868578:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.419449Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435790694234043:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:20.419347Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435790877438999:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.419357Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435790877438999:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.420475Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435792182905186:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.420759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435789650557588:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51011
2025-11-19T13:59:20.420837Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435790877439006:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.420844Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435790877439006:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.421444Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435790877438999:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:20.421635Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435789650557582:2121] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:20.421767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435789650557489:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.421777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:20.421782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435789650557489:2048] [node 1] ICP22 created new session: [4:7574435789650557589:2048]
2025-11-19T13:59:20.421786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435789650557589:2048] [node 1] ICS09 handshake done sender: [4:7574435789650557582:2121] self: [4:7574435789650557587:0] peer: [1:7574435790877438998:0] socket: 390 qp: -1
2025-11-19T13:59:20.421791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435789650557589:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:20.421796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:20.420189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435790694234038:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.420207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:20.420212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435790694234038:2073] [node 6] ICP22 created new session: [5:7574435790694234064:2073]
2025-11-19T13:59:20.420216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435790694234064:2073] [node 6] ICS09 handshake done sender: [5:7574435790694234043:2116] self: [5:7574435790694234042:0] peer: [6:7574435789708868674:0] socket: 403 qp: -1
2025-11-19T13:59:20.420221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435790694234064:2073] [node 6] ICS10 traffic start
2025-11-19T13:59:20.420227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:20.424360Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435790684814752:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.425050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435790694234066:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51315
2025-11-19T13:59:20.425118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435790684814663:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.425132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:20.425138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435790684814663:2049] [node 2] ICP22 created new session: [3:7574435790684814762:2049]
2025-11-19T13:59:20.425143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435790684814762:2049] [node 2] ICS09 handshake done sender: [3:7574435790684814752:2117] self: [3:7574435790684814760:0] peer: [2:7574435792182905185:0] socket: 412 qp: -1
2025-11-19T13:59:20.425148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435790684814762:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:20.425154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:20.425949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435792182905116:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.425965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:20.425970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435792182905116:2049] [node 3] ICP22 created new session: [2:7574435792182905214:2049]
2025-11-19T13:59:20.425975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435792182905214:2049] [node 3] ICS09 handshake done sender: [2:7574435792182905186:2111] self: [2:7574435792182905185:0] peer: [3:7574435790684814760:0] socket: 411 qp: -1
2025-11-19T13:59:20.425979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435792182905214:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:20.425985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:20.426112Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435792182905187:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:20.426398Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435792182905190:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:20.426557Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435790694234059:2122] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:20.426619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435790694233965:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.426630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:20.426634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435790694233965:2049] [node 2] ICP22 created new session: [5:7574435790694234067:2049]
2025-11-19T13:59:20.426638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435790694234067:2049] [node 2] ICS09 handshake done sender: [5:7574435790694234059:2122] self: [5:7574435790694234062:0] peer: [2:7574435792182905189:0] socket: 398 qp: -1
2025-11-19T13:59:20.426642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435790694234067:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:20.426648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:20.427298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435789708868643:2062] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29640) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435789708868668:2121] held: no
2025-11-19T13:59:20.427321Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435789708868668:2121] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:20.427430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435792182905115:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.427441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:20.427444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435792182905115:2048] [node 1] ICP22 created new session: [2:7574435792182905216:2048]
2025-11-19T13:59:20.427448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435792182905216:2048] [node 1] ICS09 handshake done sender: [2:7574435792182905187:2112] self: [2:7574435792182905207:0] peer: [1:7574435790877438990:0] socket: 374 qp: -1
2025-11-19T13:59:20.427452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435792182905216:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:20.427457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:20.427517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435792182905172:2098] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.427524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:20.427527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435792182905172:2098] [node 5] ICP22 created new session: [2:7574435792182905218:2098]
2025-11-19T13:59:20.427530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435792182905218:2098] [node 5] ICS09 handshake done sender: [2:7574435792182905190:2114] self: [2:7574435792182905189:0] peer: [5:7574435790694234062:0] socket: 391 qp: -1
2025-11-19T13:59:20.427533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435792182905218:2098] [node 5] ICS10 traffic start
2025-11-19T13:59:20.427537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:20.427826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435789708868643:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.427838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:20.427842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435789708868643:2062] [node 5] ICP22 created new session: [6:7574435789708868677:2062]
2025-11-19T13:59:20.427846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435789708868677:2062] [node 5] ICS09 handshake done sender: [6:7574435789708868668:2121] self: [6:7574435789708868674:0] peer: [5:7574435790694234042:0] socket: 428 qp: -1
2025-11-19T13:59:20.427850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435789708868677:2062] [node 5] ICS10 traffic start
2025-11-19T13:59:20.427855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:20.432017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435790877438995:2106] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.432034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:20.432040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435790877438995:2106] [node 4] ICP22 created new session: [1:7574435790877439032:2106]
2025-11-19T13:59:20.432044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435790877439032:2106] [node 4] ICS09 handshake done sender: [1:7574435790877438999:2110] self: [1:7574435790877438998:0] peer: [4:7574435789650557587:0] socket: 429 qp: -1
2025-11-19T13:59:20.432048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435790877439032:2106] [node 4] ICS10 traffic start
2025-11-19T13:59:20.432054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:20.432823Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435790877439006:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.435860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789708868679:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49517
2025-11-19T13:59:20.436420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435790877439004:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.436436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:20.436441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435790877439004:2111] [node 6] ICP22 created new session: [1:7574435790877439034:2111]
2025-11-19T13:59:20.436446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435790877439034:2111] [node 6] ICS09 handshake done sender: [1:7574435790877439006:2112] self: [1:7574435790877439005:0] peer: [6:7574435789708868676:0] socket: 430 qp: -1
2025-11-19T13:59:20.436450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435790877439034:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:20.436457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:20.440694Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435789708868578:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18820) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435789708868671:2122] held: no
2025-11-19T13:59:20.440720Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435789708868671:2122] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:20.440797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435789708868578:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.440808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:20.440813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435789708868578:2048] [node 1] ICP22 created new session: [6:7574435789708868680:2048]
2025-11-19T13:59:20.440818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435789708868680:2048] [node 1] ICS09 handshake done sender: [6:7574435789708868671:2122] self: [6:7574435789708868676:0] peer: [1:7574435790877439005:0] socket: 407 qp: -1
2025-11-19T13:59:20.440823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435789708868680:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:20.440829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:20.443713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:20.443848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435792182905216:2048] [node 1] ICS01 socket: 374 reason# EndOfStream
2025-11-19T13:59:20.443856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435792182905115:2048] [node 1] ICP30 unregister session Session# [2:7574435792182905216:2048] VirtualId# [2:7574435792182905207:0]
2025-11-19T13:59:20.443861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435792182905216:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.443889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:20.443906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:20.444171Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:20.444227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435790684814758:2048] [node 1] ICS01 socket: 379 reason# EndOfStream
2025-11-19T13:59:20.444232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435790684814662:2048] [node 1] ICP30 unregister session Session# [3:7574435790684814758:2048] VirtualId# [3:7574435790684814754:0]
2025-11-19T13:59:20.444236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435790684814758:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.444256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:20.444271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:20.451958Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:20.452031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435789650557589:2048] [node 1] ICS01 socket: 390 reason# EndOfStream
2025-11-19T13:59:20.451769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:20.452036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435789650557489:2048] [node 1] ICP30 unregister session Session# [4:7574435789650557589:2048] VirtualId# [4:7574435789650557587:0]
2025-11-19T13:59:20.452041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435789650557589:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.452075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:20.451845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435790694234055:2048] [node 1] ICS01 socket: 369 reason# EndOfStream
2025-11-19T13:59:20.452094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:20.451851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435790694233964:2048] [node 1] ICP30 unregister session Session# [5:7574435790694234055:2048] VirtualId# [5:7574435790694234053:0]
2025-11-19T13:59:20.451858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435790694234055:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.451901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:20.451923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:20.451949Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:20.452003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435789708868680:2048] [node 1] ICS01 socket: 407 reason# EndOfStream
2025-11-19T13:59:20.452008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435789708868578:2048] [node 1] ICP30 unregister session Session# [6:7574435789708868680:2048] VirtualId# [6:7574435789708868676:0]
2025-11-19T13:59:20.452011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435789708868680:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.452036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:20.452053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:20.460406Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:20.460489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435790684814762:2049] [node 2] ICS01 socket: 412 reason# EndOfStream
2025-11-19T13:59:20.460497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435790684814663:2049] [node 2] ICP30 unregister session Session# [3:7574435790684814762:2049] VirtualId# [3:7574435790684814760:0]
2025-11-19T13:59:20.460502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435790684814762:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.460538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:20.460557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:20.478655Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:20.478747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435790694234067:2049] [node 2] ICS01 socket: 398 reason# EndOfStream
2025-11-19T13:59:20.478757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435790694233965:2049] [node 2] ICP30 unregister session Session# [5:7574435790694234067:2049] VirtualId# [5:7574435790694234062:0]
2025-11-19T13:59:20.478764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435790694234067:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.478808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:20.478828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:20.480095Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:20.480171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435789650557583:2049] [node 2] ICS01 socket: 385 reason# EndOfStream
2025-11-19T13:59:20.480178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435789650557490:2049] [node 2] ICP30 unregister session Session# [4:7574435789650557583:2049] VirtualId# [4:7574435789650557574:0]
2025-11-19T13:59:20.480184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435789650557583:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.480222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:20.480243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:20.483980Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:20.484063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435789708868672:2049] [node 2] ICS01 socket: 394 reason# EndOfStream
2025-11-19T13:59:20.484070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435789708868579:2049] [node 2] ICP30 unregister session Session# [6:7574435789708868672:2049] VirtualId# [6:7574435789708868661:0]
2025-11-19T13:59:20.484074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435789708868672:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.484114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:20.484134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:20.492374Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:20.492454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435790694234050:2050] [node 3] ICS01 socket: 405 reason# EndOfStream
2025-11-19T13:59:20.492463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435790694233966:2050] [node 3] ICP30 unregister session Session# [5:7574435790694234050:2050] VirtualId# [5:7574435790694234047:0]
2025-11-19T13:59:20.492468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435790694234050:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.492508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:20.492525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:20.495306Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:20.495375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435789708868664:2050] [node 3] ICS01 socket: 414 reason# EndOfStream
2025-11-19T13:59:20.495382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435789708868580:2050] [node 3] ICP30 unregister session Session# [6:7574435789708868664:2050] VirtualId# [6:7574435789708868662:0]
2025-11-19T13:59:20.495387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435789708868664:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.495430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:20.495451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:20.499944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T13:59:20.500031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435790694234060:2070] [node 4] ICS01 socket: 393 reason# EndOfStream
2025-11-19T13:59:20.500039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435790694234037:2070] [node 4] ICP30 unregister session Session# [5:7574435790694234060:2070] VirtualId# [5:7574435790694234046:0]
2025-11-19T13:59:20.500045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435790694234060:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.500080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:20.500098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:20.501217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:20.501283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435789708868669:2070] [node 4] ICS01 socket: 395 reason# EndOfStream
2025-11-19T13:59:20.501290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435789708868644:2070] [node 4] ICP30 unregister session Session# [6:7574435789708868669:2070] VirtualId# [6:7574435789708868658:0]
2025-11-19T13:59:20.501295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435789708868669:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.501334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:20.501355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:20.520930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:59:20.521036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435789708868677:2062] [node 5] ICS01 socket: 428 reason# EndOfStream
2025-11-19T13:59:20.521045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435789708868643:2062] [node 5] ICP30 unregister session Session# [6:7574435789708868677:2062] VirtualId# [6:7574435789708868674:0]
2025-11-19T13:59:20.521052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435789708868677:2062] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:20.521096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:59:20.521118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:20.623453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435789909843474:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:20.623466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435789909843475:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:20.672088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435792464450344:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:20.672100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435792464450345:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:20.718098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435790714863084:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:20.718112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435790714863085:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:20.764377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435790215958679:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:20.764391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435790215958680:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:20.764397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435790215958681:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:20.802825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435791038563420:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:20.802837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435791038563421:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:20.802841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435791038563422:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:20.873078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435789223658443:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:20.873095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435789223658444:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:20.873100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435789223658445:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:20.889606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435790215958714:2079] [node 6] ICP01 ready to work
2025-11-19T13:59:20.889336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435791038563453:2079] [node 4] ICP01 ready to work
2025-11-19T13:59:20.889950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435791038563459:2081] [node 6] ICP01 ready to work
2025-11-19T13:59:20.890447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435789909843534:2106] [node 5] ICP01 ready to work
2025-11-19T13:59:20.890107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435789223658464:2064] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.891299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435789909843539:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.891724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435790714863141:2103] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.892862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435790215958718:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35596
2025-11-19T13:59:20.893024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435790215958722:2085] [node 5] ICP01 ready to work
2025-11-19T13:59:20.893434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435790215958723:2087] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35596
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.895001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435792464450403:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39696
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.892096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435789223658481:2075] [node 4] ICP01 ready to work
2025-11-19T13:59:20.895696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789223658484:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13584
2025-11-19T13:59:20.895712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789223658485:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13584
2025-11-19T13:59:20.896176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435789909843542:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18754
2025-11-19T13:59:20.897486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435790714863145:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8803
2025-11-19T13:59:20.897644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435790714863150:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:20.904628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435792464450412:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:20.904723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435792464450413:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.910107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435791038563470:2089] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8559
2025-11-19T13:59:20.910129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435791038563471:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8559
2025-11-19T13:59:20.912352Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435789223658485:2079] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.914320Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792464450403:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.915735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435789223658464:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.916251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435789909843549:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:20.917677Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435791038563461:2084] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.917689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435791038563461:2084] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.919025Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435791038563461:2084] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:20.919149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435791038563471:2090] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.919167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435791038563459:2081] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.919194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:20.919201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435791038563459:2081] [node 6] ICP22 created new session: [5:7574435791038563489:2081]
2025-11-19T13:59:20.919206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435791038563489:2081] [node 6] ICS09 handshake done sender: [5:7574435791038563461:2084] self: [5:7574435791038563460:0] peer: [6:7574435789223658509:0] socket: 374 qp: -1
2025-11-19T13:59:20.919212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435791038563489:2081] [node 6] ICS10 traffic start
2025-11-19T13:59:20.919220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:20.919350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435791038563459:2081] [node 6] ICP18 handshake (actor [5:7574435791038563471:2090]) is too old
2025-11-19T13:59:20.918938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789223658510:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13584
2025-11-19T13:59:20.919229Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435789223658481:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(27074821ns) {Configure:104(27074063ns) {ProcessConfigured:135(26797045ns) {StartInitialHandshake:157(26796580ns) {DropHandshakes:469(396ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(187ns) PrepareNewSessionHandshake:476(589ns) {SwitchToState:178(387ns)}}}}}
2025-11-19T13:59:20.919625Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435789223658485:2079] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:20.919672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435789223658464:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.919681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:20.919685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435789223658464:2064] [node 5] ICP22 created new session: [6:7574435789223658512:2064]
2025-11-19T13:59:20.919690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435789223658512:2064] [node 5] ICS09 handshake done sender: [6:7574435789223658485:2079] self: [6:7574435789223658509:0] peer: [5:7574435791038563460:0] socket: 391 qp: -1
2025-11-19T13:59:20.919695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435789223658512:2064] [node 5] ICS10 traffic start
2025-11-19T13:59:20.919709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:20.919533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435790714863150:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21841130ns) {Configure:104(21840607ns) {ProcessConfigured:135(21531706ns) {StartInitialHandshake:157(21531318ns) {DropHandshakes:469(561ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(134ns) PrepareNewSessionHandshake:476(741ns) {SwitchToState:178(107ns)}}}}}
2025-11-19T13:59:20.921283Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435790215958718:2083] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.923132Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435789909843542:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.928956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435790215958751:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35596
2025-11-19T13:59:20.929003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435790215958752:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35596
2025-11-19T13:59:20.931608Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435789909843474:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(31246385ns) {Configure:104(31245647ns) {ProcessConfigured:135(30936237ns) {StartInitialHandshake:157(30935696ns) {DropHandshakes:469(421ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:476(1430ns) {SwitchToState:178(855ns)}}}}}
2025-11-19T13:59:20.931991Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435790215958751:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.932023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435790215958679:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.932239Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435791038563470:2089] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.932342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435791038563504:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8559
2025-11-19T13:59:20.933649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435792464450411:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:20.932278Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435789909843541:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.932291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435789909843541:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.936021Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435790215958752:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.936052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435790215958680:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.935965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435791038563505:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8559
2025-11-19T13:59:20.936067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435791038563420:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.941087Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792464450415:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.941100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435792464450415:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.941711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435792464450424:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39696
2025-11-19T13:59:20.941937Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435792464450415:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.942850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435790215958764:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35596
2025-11-19T13:59:20.943038Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435790215958752:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:20.943129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435790215958680:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.943144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:20.943149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435790215958680:2049] [node 2] ICP22 created new session: [4:7574435790215958765:2049]
2025-11-19T13:59:20.943155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435790215958765:2049] [node 2] ICS09 handshake done sender: [4:7574435790215958752:2113] self: [4:7574435790215958763:0] peer: [2:7574435792464450414:0] socket: 401 qp: -1
2025-11-19T13:59:20.943161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435790215958765:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:20.943167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:20.943962Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435791038563505:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.944020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435791038563421:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.942070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435792464450412:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.942088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:20.942095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435792464450412:2110] [node 4] ICP22 created new session: [2:7574435792464450426:2110]
2025-11-19T13:59:20.942100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435792464450426:2110] [node 4] ICS09 handshake done sender: [2:7574435792464450415:2112] self: [2:7574435792464450414:0] peer: [4:7574435790215958763:0] socket: 394 qp: -1
2025-11-19T13:59:20.942107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435792464450426:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:20.942118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.943788Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435789223658484:2078] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.943826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435789223658481:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.947392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789223658529:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13584
2025-11-19T13:59:20.944814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435790714863159:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:20.945724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435790714863155:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8803
2025-11-19T13:59:20.945758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435790714863156:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8803
2025-11-19T13:59:20.945791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435790714863157:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8803
2025-11-19T13:59:20.945818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435790714863158:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8803
2025-11-19T13:59:20.947794Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790714863145:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.947967Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792464450417:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.947978Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435792464450417:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.952822Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435789909843536:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.952846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435789909843536:2107] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.953979Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435792464450411:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20170425ns) {Configure:104(20169456ns) {ProcessConfigured:135(19827356ns) {StartInitialHandshake:157(19826412ns) {DropHandshakes:469(884ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(137ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:476(1420ns) {SwitchToState:178(793ns)}}}}}
2025-11-19T13:59:20.954145Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792464450424:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.956620Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435790215958716:2082] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.956645Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435790215958716:2082] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.965091Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790714863155:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.965186Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790714863156:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.965229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435790714863085:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.965239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435790714863084:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.963593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435789909843555:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18754
2025-11-19T13:59:20.963941Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435789909843541:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:20.964125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435789909843539:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.964138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:20.964146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435789909843539:2108] [node 4] ICP22 created new session: [1:7574435789909843557:2108]
2025-11-19T13:59:20.964152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435789909843557:2108] [node 4] ICS09 handshake done sender: [1:7574435789909843541:2109] self: [1:7574435789909843540:0] peer: [4:7574435790215958762:0] socket: 393 qp: -1
2025-11-19T13:59:20.964158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435789909843557:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:20.964168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:20.965908Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435789909843545:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.965913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435789909843545:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.967913Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792464450405:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.967930Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435792464450405:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.968948Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790714863158:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.969545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435791038563508:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8559
2025-11-19T13:59:20.969708Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435792464450405:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:20.969813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435792464450345:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.969828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:20.969834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435792464450345:2049] [node 3] ICP22 created new session: [2:7574435792464450432:2049]
2025-11-19T13:59:20.969840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435792464450432:2049] [node 3] ICS09 handshake done sender: [2:7574435792464450405:2107] self: [2:7574435792464450404:0] peer: [3:7574435790714863169:0] socket: 389 qp: -1
2025-11-19T13:59:20.969846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435792464450432:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:20.969862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:20.971968Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435789909843545:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:20.971844Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435789223658529:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.971885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435789223658445:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.972380Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435789909843555:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.972095Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435791038563508:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.972123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435791038563422:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.973106Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435791038563504:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.973127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435791038563453:2079] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.975774Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790714863154:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.975789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435790714863154:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.977168Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435789909843536:2107] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:20.979934Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435792464450417:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.980095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789223658531:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13584
2025-11-19T13:59:20.980147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789223658532:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13584
2025-11-19T13:59:20.980178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789223658533:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13584
2025-11-19T13:59:20.980520Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435789223658532:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.980549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435789223658444:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:20.977645Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435790215958725:2088] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.977659Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435790215958725:2088] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.980657Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792464450429:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.980668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435792464450429:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.981296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435789223658535:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13584
2025-11-19T13:59:20.981410Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435792464450429:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:20.981580Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435789223658532:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:20.981660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435789223658444:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.981677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:20.981682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435789223658444:2049] [node 2] ICP22 created new session: [6:7574435789223658536:2049]
2025-11-19T13:59:20.981688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435789223658536:2049] [node 2] ICS09 handshake done sender: [6:7574435789223658532:2120] self: [6:7574435789223658534:0] peer: [2:7574435792464450423:0] socket: 430 qp: -1
2025-11-19T13:59:20.981695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435789223658536:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:20.981702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:20.983728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435792464450411:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.983771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:20.983780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435792464450411:2109] [node 6] ICP22 created new session: [2:7574435792464450437:2109]
2025-11-19T13:59:20.983786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435792464450437:2109] [node 6] ICS09 handshake done sender: [2:7574435792464450429:2116] self: [2:7574435792464450423:0] peer: [6:7574435789223658534:0] socket: 422 qp: -1
2025-11-19T13:59:20.983798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435792464450437:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:20.983808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:20.984069Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435790215958716:2082] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.984255Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435790215958725:2088] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.986325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435790215958723:2087] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.985057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790714863161:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.986350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435790215958714:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.985071Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435790714863161:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:20.986360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:20.985560Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790714863157:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:20.986366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435790215958714:2079] [node 6] ICP22 created new session: [4:7574435790215958772:2079]
2025-11-19T13:59:20.986372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435790215958772:2079] [node 6] ICS09 handshake done sender: [4:7574435790215958716:2082] self: [4:7574435790215958715:0] peer: [6:7574435789223658528:0] socket: 372 qp: -1
2025-11-19T13:59:20.986379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435790215958772:2079] [node 6] ICS10 traffic start
2025-11-19T13:59:20.986387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:20.987592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435790215958770:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35596
2025-11-19T13:59:20.984424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435791038563511:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8559
2025-11-19T13:59:20.985847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435791038563512:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8559
2025-11-19T13:59:20.986105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435791038563513:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8559
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:20.988134Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435791038563505:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:20.988265Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435791038563470:2089] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:20.988328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435791038563421:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:20.988343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:20.988349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435791038563421:2049] [node 2] ICP22 created new session: [5:7574435791038563515:2049]
2025-11-19T13:59:20.988354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435791038563515:2049] [node 2] ICS09 handshake done sender: [5:7574435791038563505:2118] self: [5:7574435791038563507:0] peer: [2:7574435792464450416:0] socket: 384 qp: -1
2025-11-19T13:59:20.988359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435791038563515:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:20.988365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:20.987986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435789909843475:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.988008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:20.988016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435789909843475:2049] [node 3] ICP22 created new session: [1:7574435789909843563:2049]
2025-11-19T13:59:20.988022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435789909843563:2049] [node 3] ICS09 handshake done sender: [1:7574435789909843545:2111] self: [1:7574435789909843543:0] peer: [3:7574435790714863170:0] socket: 397 qp: -1
2025-11-19T13:59:20.988030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435789909843563:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:20.988039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:20.988179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435789909843534:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.988184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:20.988188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435789909843534:2106] [node 5] ICP22 created new session: [1:7574435789909843565:2106]
2025-11-19T13:59:20.988192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435789909843565:2106] [node 5] ICS09 handshake done sender: [1:7574435789909843536:2107] self: [1:7574435789909843535:0] peer: [5:7574435791038563506:0] socket: 377 qp: -1
2025-11-19T13:59:20.988197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435789909843565:2106] [node 5] ICS10 traffic start
2025-11-19T13:59:20.988203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:20.988751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435790215958771:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35596
2025-11-19T13:59:20.988809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435790215958722:2085] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.988821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:20.988826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435790215958722:2085] [node 5] ICP22 created new session: [4:7574435790215958774:2085]
2025-11-19T13:59:20.988830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435790215958774:2085] [node 5] ICS09 handshake done sender: [4:7574435790215958725:2088] self: [4:7574435790215958724:0] peer: [5:7574435791038563510:0] socket: 387 qp: -1
2025-11-19T13:59:20.988835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435790215958774:2085] [node 5] ICS10 traffic start
2025-11-19T13:59:20.988841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:20.988619Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435790714863154:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:20.991923Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435790714863161:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:20.995878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435792464450413:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:20.995898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:20.995905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435792464450413:2111] [node 5] ICP22 created new session: [2:7574435792464450439:2111]
2025-11-19T13:59:20.995910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435792464450439:2111] [node 5] ICS09 handshake done sender: [2:7574435792464450417:2113] self: [2:7574435792464450416:0] peer: [5:7574435791038563507:0] socket: 395 qp: -1
2025-11-19T13:59:20.995915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435792464450439:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:20.995924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.008001Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435789223658531:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.008164Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435789223658484:2078] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:21.008287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435794204810864:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18754
2025-11-19T13:59:21.008615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435790215958681:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.008791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435790215958722:2085] [node 5] ICP35 event processing took too much time PendingConnection:278(19950919ns) {HandleHandshakeStatus:329(19950376ns) {DropIncomingHandshake:441(3212ns) DropOutgoingHandshake:456(156ns) SwitchToState:178(75ns) ProcessPendingSessionEvents:495(19894684ns) {ForwardSessionEventToSession:612(19881750ns) ForwardSessionEventToSession:612(1396ns) ForwardSessionEventToSession:612(771ns) ForwardSessionEventToSession:612(614ns) ForwardSessionEventToSession:612(598ns) ForwardSessionEventToSession:612(1092ns) ForwardSessionEventToSession:612(844ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(497ns) ForwardSessionEventToSession:612(492ns)}}}
2025-11-19T13:59:21.008841Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435790215958771:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.008989Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435790215958751:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:21.009556Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435794204810864:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.009584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435789909843539:2108] [node 4] ICP18 handshake (actor [1:7574435794204810864:2117]) is too old
2025-11-19T13:59:21.009791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435790215958714:2079] [node 6] ICP18 handshake (actor [4:7574435790215958771:2121]) is too old
2025-11-19T13:59:21.009971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435790215958679:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.009985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:21.009991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435790215958679:2048] [node 1] ICP22 created new session: [4:7574435794510926073:2048]
2025-11-19T13:59:21.009997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435794510926073:2048] [node 1] ICS09 handshake done sender: [4:7574435790215958751:2112] self: [4:7574435790215958762:0] peer: [1:7574435789909843540:0] socket: 400 qp: -1
2025-11-19T13:59:21.010004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435794510926073:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:21.010012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:21.010908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435789223658481:2075] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.010924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:21.010929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435789223658481:2075] [node 4] ICP22 created new session: [6:7574435793518625835:2075]
2025-11-19T13:59:21.010934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435793518625835:2075] [node 4] ICS09 handshake done sender: [6:7574435789223658484:2078] self: [6:7574435789223658528:0] peer: [4:7574435790215958715:0] socket: 390 qp: -1
2025-11-19T13:59:21.010947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435793518625835:2075] [node 4] ICS10 traffic start
2025-11-19T13:59:21.010954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.012189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435790714863150:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.012223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:21.012229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435790714863150:2108] [node 6] ICP22 created new session: [3:7574435795009830473:2108]
2025-11-19T13:59:21.012235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435795009830473:2108] [node 6] ICS09 handshake done sender: [3:7574435790714863154:2112] self: [3:7574435790714863151:0] peer: [6:7574435789223658530:0] socket: 411 qp: -1
2025-11-19T13:59:21.012242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435795009830473:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:21.012249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:21.012388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435790714863159:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.012396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:21.012402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435790714863159:2111] [node 5] ICP22 created new session: [3:7574435795009830475:2111]
2025-11-19T13:59:21.012406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435795009830475:2111] [node 5] ICS09 handshake done sender: [3:7574435790714863161:2117] self: [3:7574435790714863160:0] peer: [5:7574435791038563509:0] socket: 412 qp: -1
2025-11-19T13:59:21.012409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435795009830475:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:21.012413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:21.012105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435791038563514:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8559
2025-11-19T13:59:21.012323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435792464450441:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39696
2025-11-19T13:59:21.012512Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435790714863144:2105] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.012519Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435790714863144:2105] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.013460Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435791038563504:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:21.014220Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435792464450441:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.014277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435792464450344:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.014426Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435789909843554:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.014435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435789909843554:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.015279Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435789909843554:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:21.015362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435789909843474:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.015379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:21.015384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435789909843474:2048] [node 2] ICP22 created new session: [1:7574435794204810866:2048]
2025-11-19T13:59:21.015389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435794204810866:2048] [node 2] ICS09 handshake done sender: [1:7574435789909843554:2115] self: [1:7574435789909843546:0] peer: [2:7574435796759417738:0] socket: 441 qp: -1
2025-11-19T13:59:21.015394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435794204810866:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:21.015401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.015732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435796759417739:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39696
2025-11-19T13:59:21.015922Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435792464450441:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:21.016784Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435792464450344:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16969) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435792464450441:2117] held: no
2025-11-19T13:59:21.016805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435792464450344:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.016813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:21.016820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435792464450344:2048] [node 1] ICP22 created new session: [2:7574435796759417740:2048]
2025-11-19T13:59:21.016825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435796759417740:2048] [node 1] ICS09 handshake done sender: [2:7574435792464450441:2117] self: [2:7574435796759417738:0] peer: [1:7574435789909843546:0] socket: 444 qp: -1
2025-11-19T13:59:21.016831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435796759417740:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:21.016838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.017191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435790714863175:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8803
2025-11-19T13:59:21.017467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435791038563420:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.017484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:21.017490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435791038563420:2048] [node 1] ICP22 created new session: [5:7574435795333530813:2048]
2025-11-19T13:59:21.017495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435795333530813:2048] [node 1] ICS09 handshake done sender: [5:7574435791038563470:2089] self: [5:7574435791038563506:0] peer: [1:7574435789909843535:0] socket: 383 qp: -1
2025-11-19T13:59:21.017501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435795333530813:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:21.017507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.018438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435790714863176:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8803
2025-11-19T13:59:21.018791Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435790714863156:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.018910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435794204810868:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18754
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.019002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435796759417742:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39696
2025-11-19T13:59:21.019099Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435790714863155:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.019195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435794510926075:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35596
2025-11-19T13:59:21.020192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435789223658443:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.019362Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435791038563421:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(31005813ns) {HandleHandshakeStatus:329(31005383ns) {DropIncomingHandshake:441(187ns) DropOutgoingHandshake:456(1677ns) SwitchToState:178(77ns) ProcessPendingSessionEvents:495(30918002ns) {ForwardSessionEventToSession:612(30908489ns) ForwardSessionEventToSession:612(1650ns) ForwardSessionEventToSession:612(602ns) ForwardSessionEventToSession:612(821ns) ForwardSessionEventToSession:612(630ns) ForwardSessionEventToSession:612(608ns) ForwardSessionEventToSession:612(535ns) ForwardSessionEventToSession:612(571ns) ForwardSessionEventToSession:612(859ns) ForwardSessionEventToSession:612(804ns)}}}
2025-11-19T13:59:21.019383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435791038563453:2079] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.019391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:21.019396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435791038563453:2079] [node 4] ICP22 created new session: [5:7574435795333530815:2079]
2025-11-19T13:59:21.019400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435795333530815:2079] [node 4] ICS09 handshake done sender: [5:7574435791038563504:2117] self: [5:7574435791038563510:0] peer: [4:7574435790215958724:0] socket: 396 qp: -1
2025-11-19T13:59:21.019404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435795333530815:2079] [node 4] ICS10 traffic start
2025-11-19T13:59:21.019411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:21.019466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435791038563453:2079] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.021129Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435789223658445:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435789223658529:2118] held: no
2025-11-19T13:59:21.021186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435793518625837:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13584
2025-11-19T13:59:21.021493Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435791038563508:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:21.021570Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435789909843552:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.021579Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435789909843552:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.022478Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435789223658529:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.022738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435789223658445:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.022749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:21.022754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435789223658445:2050] [node 3] ICP22 created new session: [6:7574435793518625840:2050]
2025-11-19T13:59:21.022759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435793518625840:2050] [node 3] ICS09 handshake done sender: [6:7574435789223658529:2118] self: [6:7574435789223658530:0] peer: [3:7574435790714863151:0] socket: 413 qp: -1
2025-11-19T13:59:21.022764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435793518625840:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:21.022773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:21.022861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435793518625839:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13584
2025-11-19T13:59:21.022981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435791038563422:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435791038563508:2119] held: no
2025-11-19T13:59:21.023000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435791038563422:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.023006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:21.023010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435791038563422:2050] [node 3] ICP22 created new session: [5:7574435795333530817:2050]
2025-11-19T13:59:21.023014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435795333530817:2050] [node 3] ICS09 handshake done sender: [5:7574435791038563508:2119] self: [5:7574435791038563509:0] peer: [3:7574435790714863160:0] socket: 415 qp: -1
2025-11-19T13:59:21.023018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435795333530817:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:21.023023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:21.023393Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435790714863144:2105] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:21.023505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435790714863084:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.023516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:21.023521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435790714863084:2048] [node 1] ICP22 created new session: [3:7574435795009830480:2048]
2025-11-19T13:59:21.023526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435795009830480:2048] [node 1] ICS09 handshake done sender: [3:7574435790714863156:2114] self: [3:7574435790714863170:0] peer: [1:7574435789909843543:0] socket: 408 qp: -1
2025-11-19T13:59:21.023531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435795009830480:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:21.023537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:21.023694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435790714863141:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.023703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:21.023706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435790714863141:2103] [node 4] ICP22 created new session: [3:7574435795009830482:2103]
2025-11-19T13:59:21.023710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435795009830482:2103] [node 4] ICS09 handshake done sender: [3:7574435790714863144:2105] self: [3:7574435790714863143:0] peer: [4:7574435794510926072:0] socket: 385 qp: -1
2025-11-19T13:59:21.023713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435795009830482:2103] [node 4] ICS10 traffic start
2025-11-19T13:59:21.023717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:21.024244Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435790215958723:2087] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:21.026219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435790215958681:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.026237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:21.026244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435790215958681:2050] [node 3] ICP22 created new session: [4:7574435794510926076:2050]
2025-11-19T13:59:21.026249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435794510926076:2050] [node 3] ICS09 handshake done sender: [4:7574435790215958723:2087] self: [4:7574435794510926072:0] peer: [3:7574435790714863143:0] socket: 386 qp: -1
2025-11-19T13:59:21.026255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435794510926076:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:21.026261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:21.026327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435790215958681:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:21.026924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435790714863085:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.026937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:21.026941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435790714863085:2049] [node 2] ICP22 created new session: [3:7574435795009830484:2049]
2025-11-19T13:59:21.026946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435795009830484:2049] [node 2] ICS09 handshake done sender: [3:7574435790714863155:2113] self: [3:7574435790714863169:0] peer: [2:7574435792464450404:0] socket: 392 qp: -1
2025-11-19T13:59:21.026950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435795009830484:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:21.026956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:21.027799Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435796759417742:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.027911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435792464450345:2049] [node 3] ICP18 handshake (actor [2:7574435796759417742:2119]) is too old
2025-11-19T13:59:21.032062Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435789909843552:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:21.032299Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435789223658531:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:21.032486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435789223658443:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.032504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:21.032511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435789223658443:2048] [node 1] ICP22 created new session: [6:7574435793518625842:2048]
2025-11-19T13:59:21.032518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435793518625842:2048] [node 1] ICS09 handshake done sender: [6:7574435789223658531:2119] self: [6:7574435793518625838:0] peer: [1:7574435789909843551:0] socket: 419 qp: -1
2025-11-19T13:59:21.032524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435793518625842:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:21.032532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:21.034049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435789909843549:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.034071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:21.034078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435789909843549:2113] [node 6] ICP22 created new session: [1:7574435794204810870:2113]
2025-11-19T13:59:21.034084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435794204810870:2113] [node 6] ICS09 handshake done sender: [1:7574435789909843552:2114] self: [1:7574435789909843551:0] peer: [6:7574435793518625838:0] socket: 417 qp: -1
2025-11-19T13:59:21.034090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435794204810870:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:21.034097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:21.110309Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:21.110397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435796759417740:2048] [node 1] ICS01 socket: 444 reason# EndOfStream
2025-11-19T13:59:21.110406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435792464450344:2048] [node 1] ICP30 unregister session Session# [2:7574435796759417740:2048] VirtualId# [2:7574435796759417738:0]
2025-11-19T13:59:21.110413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435796759417740:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.110449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:21.110468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:21.110854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:21.110927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435795009830480:2048] [node 1] ICS01 socket: 408 reason# EndOfStream
2025-11-19T13:59:21.110933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435790714863084:2048] [node 1] ICP30 unregister session Session# [3:7574435795009830480:2048] VirtualId# [3:7574435790714863170:0]
2025-11-19T13:59:21.110938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435795009830480:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.110963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:21.110979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:21.121210Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:21.121299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435795333530813:2048] [node 1] ICS01 socket: 383 reason# EndOfStream
2025-11-19T13:59:21.121308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435791038563420:2048] [node 1] ICP30 unregister session Session# [5:7574435795333530813:2048] VirtualId# [5:7574435791038563506:0]
2025-11-19T13:59:21.121315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435795333530813:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.121369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:21.121393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:21.121426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:21.121481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435794510926073:2048] [node 1] ICS01 socket: 400 reason# EndOfStream
2025-11-19T13:59:21.121485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435790215958679:2048] [node 1] ICP30 unregister session Session# [4:7574435794510926073:2048] VirtualId# [4:7574435790215958762:0]
2025-11-19T13:59:21.121489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435794510926073:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.121513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:21.121533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:21.121562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:21.121621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435793518625842:2048] [node 1] ICS01 socket: 419 reason# EndOfStream
2025-11-19T13:59:21.121625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435789223658443:2048] [node 1] ICP30 unregister session Session# [6:7574435793518625842:2048] VirtualId# [6:7574435793518625838:0]
2025-11-19T13:59:21.121629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435793518625842:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.121651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:21.121671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:21.128696Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:21.128787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435795009830484:2049] [node 2] ICS01 socket: 392 reason# EndOfStream
2025-11-19T13:59:21.128794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435790714863085:2049] [node 2] ICP30 unregister session Session# [3:7574435795009830484:2049] VirtualId# [3:7574435790714863169:0]
2025-11-19T13:59:21.128801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435795009830484:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.128851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:21.128872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:21.139916Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:21.140007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435790215958765:2049] [node 2] ICS07 socket disconnect 401 reason# EndOfStream
2025-11-19T13:59:21.140014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435790215958765:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.140083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:21.140107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435790215958765:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:21.140316Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:21.140382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435791038563515:2049] [node 2] ICS07 socket disconnect 384 reason# EndOfStream
2025-11-19T13:59:21.140387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435791038563515:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.140415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:21.140434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435791038563515:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:21.140677Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:21.140754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435789223658536:2049] [node 2] ICS01 socket: 430 reason# EndOfStream
2025-11-19T13:59:21.140759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435789223658444:2049] [node 2] ICP30 unregister session Session# [6:7574435789223658536:2049] VirtualId# [6:7574435789223658534:0]
2025-11-19T13:59:21.140764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435789223658536:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.140791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:21.140813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:21.141164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435791038563421:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4251) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:21.141175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435791038563515:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:21.141179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435791038563421:2049] [node 2] ICP30 unregister session Session# [5:7574435791038563515:2049] VirtualId# [5:7574435791038563507:0]
2025-11-19T13:59:21.141190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:21.141195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435791038563421:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4251) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:21.141198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4251) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:21.141440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435790215958680:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4251) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:21.141450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435790215958765:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:21.141454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435790215958680:2049] [node 2] ICP30 unregister session Session# [4:7574435790215958765:2049] VirtualId# [4:7574435790215958763:0]
2025-11-19T13:59:21.141462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:21.141466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435790215958680:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4251) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:21.141468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4251) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:21.151838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435790215958680:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:21.151935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435791038563421:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:21.156780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:21.156868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435793518625840:2050] [node 3] ICS01 socket: 413 reason# EndOfStream
2025-11-19T13:59:21.156876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435789223658445:2050] [node 3] ICP30 unregister session Session# [6:7574435793518625840:2050] VirtualId# [6:7574435789223658530:0]
2025-11-19T13:59:21.156882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435793518625840:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.156939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:21.156962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:21.156998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:21.157065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435795333530817:2050] [node 3] ICS01 socket: 415 reason# EndOfStream
2025-11-19T13:59:21.157069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435791038563422:2050] [node 3] ICP30 unregister session Session# [5:7574435795333530817:2050] VirtualId# [5:7574435791038563509:0]
2025-11-19T13:59:21.157073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435795333530817:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.157096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:21.157114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:21.172489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:21.172584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435793518625835:2075] [node 4] ICS01 socket: 390 reason# EndOfStream
2025-11-19T13:59:21.172593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435789223658481:2075] [node 4] ICP30 unregister session Session# [6:7574435793518625835:2075] VirtualId# [6:7574435789223658528:0]
2025-11-19T13:59:21.172600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435793518625835:2075] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.172661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:21.172685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:21.296081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435796663113454:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:21.296092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435796663113455:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:21.327573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435794397668916:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:21.327606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435794397668917:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:21.359090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435797132836166:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:21.359117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435797132836167:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:21.359121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435797132836168:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:21.343896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435795819734207:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:21.343906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435795819734208:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:21.392098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435795262259245:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:21.392113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435795262259246:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:21.392117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435795262259247:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:21.417313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435793240588478:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:21.417324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435793240588479:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:21.417328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435793240588480:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:21.452479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435793240588495:2059] [node 5] ICP01 ready to work
2025-11-19T13:59:21.452580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435793240588499:2061] [node 4] ICP01 ready to work
2025-11-19T13:59:21.452819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435795262259267:2065] [node 6] ICP01 ready to work
2025-11-19T13:59:21.456591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435794397668978:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:21.472469Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435795262259247:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(20212545ns) {Configure:104(20211257ns) {ProcessConfigured:135(19778186ns) {StartInitialHandshake:157(19777226ns) {DropHandshakes:469(767ns) {DropIncomingHandshake:441(108ns) DropOutgoingHandshake:456(79ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(1726ns) {SwitchToState:178(1005ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.473230Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435796663113455:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(20579982ns) {Configure:104(20578855ns) {ProcessConfigured:135(20347487ns) {StartInitialHandshake:157(20347285ns) {DropHandshakes:469(180ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(46ns)} GenerateSessionVirtualId:988(454ns) PrepareNewSessionHandshake:476(1546ns) {SwitchToState:178(688ns)}}}}}
2025-11-19T13:59:21.473418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435796663113524:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:21.476164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435796663113528:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59198
2025-11-19T13:59:21.476228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435795819734267:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55859
2025-11-19T13:59:21.481097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435793240588478:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(27992782ns) {Configure:104(27991784ns) {ProcessConfigured:135(26614926ns) {StartInitialHandshake:157(26613773ns) {DropHandshakes:469(829ns) {DropIncomingHandshake:441(219ns) DropOutgoingHandshake:456(139ns)} GenerateSessionVirtualId:988(261ns) PrepareNewSessionHandshake:476(1336ns) {SwitchToState:178(776ns)}}}}}
2025-11-19T13:59:21.482098Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435793240588495:2059] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(29472404ns) {Configure:104(29471997ns) {ProcessConfigured:135(27581023ns) {StartInitialHandshake:157(27580389ns) {DropHandshakes:469(283ns) {DropIncomingHandshake:441(46ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(175ns) PrepareNewSessionHandshake:476(707ns) {SwitchToState:178(552ns)}}}}}
2025-11-19T13:59:21.482262Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435794397668978:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25607172ns) {Configure:104(25606411ns) {ProcessConfigured:135(25198400ns) {StartInitialHandshake:157(25198118ns) {DropHandshakes:469(756ns) {DropIncomingHandshake:441(133ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(1167ns) {SwitchToState:178(374ns)}}}}}
2025-11-19T13:59:21.482956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435794397668987:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:21.483804Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435793240588499:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(30741028ns) {Configure:104(25950723ns) {ProcessConfigured:135(25544881ns) {StartInitialHandshake:157(25544306ns) {DropHandshakes:469(592ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:476(760ns) {SwitchToState:178(242ns)}}}}}
2025-11-19T13:59:21.484188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435795262259272:2062] [node 4] ICP01 ready to work
2025-11-19T13:59:21.484814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435796663113522:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:21.485629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435796663113519:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.491509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435794397668993:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48635
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.491612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435794397668994:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48635
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.502992Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435796663113528:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.500174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435795262259318:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36885
2025-11-19T13:59:21.503820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435794397668981:2105] [node 5] ICP01 ready to work
2025-11-19T13:59:21.503881Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435794397668993:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.508229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435795819734269:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:21.508409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435795819734274:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:21.509332Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435795262259267:2065] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(56448086ns) {Configure:104(56447063ns) {ProcessConfigured:135(56221905ns) {StartInitialHandshake:157(56221443ns) {DropHandshakes:469(209ns) {DropIncomingHandshake:441(39ns) DropOutgoingHandshake:456(33ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:476(959ns) {SwitchToState:178(344ns)}}}}}
2025-11-19T13:59:21.512165Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435795262259272:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(27875578ns) {Configure:104(27875054ns) {ProcessConfigured:135(27563678ns) {StartInitialHandshake:157(27563120ns) {DropHandshakes:469(409ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(52ns)} GenerateSessionVirtualId:988(274ns) PrepareNewSessionHandshake:476(669ns) {SwitchToState:178(396ns)}}}}}
2025-11-19T13:59:21.519407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435794397668916:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.520355Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435796663113517:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.520368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435796663113517:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.520887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435796663113535:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59198
2025-11-19T13:59:21.520899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435796663113536:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59198
2025-11-19T13:59:21.523244Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435796663113517:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:21.523362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435796663113454:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.523379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:21.523386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435796663113454:2048] [node 2] ICP22 created new session: [1:7574435796663113538:2048]
2025-11-19T13:59:21.523394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435796663113538:2048] [node 2] ICS09 handshake done sender: [1:7574435796663113517:2108] self: [1:7574435796663113514:0] peer: [2:7574435794397668996:0] socket: 378 qp: -1
2025-11-19T13:59:21.523401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435796663113538:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:21.523411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:21.523434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435795819734272:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:21.522496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435797132836231:2082] [node 5] ICP01 ready to work
2025-11-19T13:59:21.522550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435797132836227:2084] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.529369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435797132836167:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(69291691ns) {Configure:104(69290613ns) {ProcessConfigured:135(68919009ns) {StartInitialHandshake:157(68918699ns) {DropHandshakes:469(528ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:476(826ns) {SwitchToState:178(266ns)}}}}}
2025-11-19T13:59:21.527999Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435796663113536:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.529614Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435795262259318:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.529660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435795262259245:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.528617Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435795819734267:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.528913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435795819734273:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55859
2025-11-19T13:59:21.528998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435795819734277:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55859
2025-11-19T13:59:21.529009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435795819734278:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55859
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.533300Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435795819734278:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.530573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435794397668997:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48635
2025-11-19T13:59:21.530612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435794397668998:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48635
2025-11-19T13:59:21.530662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435794397668999:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48635
2025-11-19T13:59:21.530673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435794397669000:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48635
2025-11-19T13:59:21.530752Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435794397668981:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(22941996ns) {Configure:104(22940350ns) {ProcessConfigured:135(22585869ns) {StartInitialHandshake:157(22585403ns) {DropHandshakes:469(1211ns) {DropIncomingHandshake:441(138ns) DropOutgoingHandshake:456(140ns)} GenerateSessionVirtualId:988(258ns) PrepareNewSessionHandshake:476(531ns) {SwitchToState:178(120ns)}}}}}
2025-11-19T13:59:21.530966Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435794397668993:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:21.531061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435794397668916:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.531077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:21.531083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435794397668916:2048] [node 1] ICP22 created new session: [2:7574435794397669002:2048]
2025-11-19T13:59:21.531088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435794397669002:2048] [node 1] ICS09 handshake done sender: [2:7574435794397668993:2114] self: [2:7574435794397668996:0] peer: [1:7574435796663113514:0] socket: 379 qp: -1
2025-11-19T13:59:21.531095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435794397669002:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:21.531102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:21.535279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435796663113454:2048] [node 2] ICP18 handshake (actor [1:7574435796663113536:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.540354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435793240588556:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15972
2025-11-19T13:59:21.540373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435793240588557:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15972
2025-11-19T13:59:21.540387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435793240588558:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15972
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.540950Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435794397668997:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.540587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435797132836243:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4702
2025-11-19T13:59:21.540636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435797132836244:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4702
2025-11-19T13:59:21.540663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435797132836245:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4702
2025-11-19T13:59:21.544309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435797132836168:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21692394ns) {Configure:104(21691608ns) {ProcessConfigured:135(21315345ns) {StartInitialHandshake:157(21314255ns) {DropHandshakes:469(1210ns) {DropIncomingHandshake:441(131ns) DropOutgoingHandshake:456(493ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:476(1256ns) {SwitchToState:178(995ns)}}}}}
2025-11-19T13:59:21.547895Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435796663113535:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.548961Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435794397668982:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.548972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435794397668982:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.549530Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435794397668986:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.549533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435794397668986:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.548005Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435797132836243:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.548040Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435797132836244:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.548064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435797132836167:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.548072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435797132836166:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.550275Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435796663113531:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.550282Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435796663113531:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.550706Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435794397668982:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:21.550941Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435794397668986:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:21.548372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435795819734269:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(24851671ns) {Configure:104(24850512ns) {ProcessConfigured:135(24515440ns) {StartInitialHandshake:157(24514719ns) {DropHandshakes:469(412ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:476(778ns) {SwitchToState:178(280ns)}}}}}
2025-11-19T13:59:21.548398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435795819734208:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.548420Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435795819734273:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.550640Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435795819734277:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.551002Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435796663113526:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.551013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435796663113526:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.552067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435794397668978:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.552086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:21.552095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435794397668978:2108] [node 4] ICP22 created new session: [2:7574435794397669008:2108]
2025-11-19T13:59:21.552101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435794397669008:2108] [node 4] ICS09 handshake done sender: [2:7574435794397668982:2110] self: [2:7574435794397668979:0] peer: [4:7574435797132836247:0] socket: 391 qp: -1
2025-11-19T13:59:21.552108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435794397669008:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:21.552131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.553357Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435796663113526:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:21.553516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435796663113524:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.553534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:21.553541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435796663113524:2110] [node 5] ICP22 created new session: [1:7574435796663113544:2110]
2025-11-19T13:59:21.553547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435796663113544:2110] [node 5] ICS09 handshake done sender: [1:7574435796663113526:2112] self: [1:7574435796663113525:0] peer: [5:7574435795262259324:0] socket: 386 qp: -1
2025-11-19T13:59:21.553551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435796663113544:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:21.553568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.554070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435796663113543:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59198
2025-11-19T13:59:21.552604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435797132836249:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4702
2025-11-19T13:59:21.552672Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435797132836243:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:21.552841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435797132836167:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.552852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:21.552858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435797132836167:2049] [node 2] ICP22 created new session: [4:7574435797132836250:2049]
2025-11-19T13:59:21.552863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435797132836250:2049] [node 2] ICS09 handshake done sender: [4:7574435797132836243:2114] self: [4:7574435797132836247:0] peer: [2:7574435794397668979:0] socket: 401 qp: -1
2025-11-19T13:59:21.552868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435797132836250:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:21.552883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.558453Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435793240588558:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.558540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435793240588479:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.556285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435795262259245:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16103) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435795262259318:2112] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.560607Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435796663113543:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.560538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435793240588561:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15972
2025-11-19T13:59:21.560630Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435793240588557:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.560653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435793240588499:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.561099Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435793240588561:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.561132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435793240588495:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.560807Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435797132836237:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.560820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435797132836237:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.563807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435795262259326:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36885
2025-11-19T13:59:21.563878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435795262259327:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36885
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.568769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435795262259328:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36885
2025-11-19T13:59:21.563914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435794397668917:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.563938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:21.563946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435794397668917:2049] [node 3] ICP22 created new session: [2:7574435794397669011:2049]
2025-11-19T13:59:21.563953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435794397669011:2049] [node 3] ICS09 handshake done sender: [2:7574435794397668986:2112] self: [2:7574435794397668985:0] peer: [3:7574435795819734289:0] socket: 393 qp: -1
2025-11-19T13:59:21.563960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435794397669011:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:21.563969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.571660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435797132836253:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4702
2025-11-19T13:59:21.572100Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435796663113531:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:21.572487Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435795262259326:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.572522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435795262259272:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.572609Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435795262259322:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.572618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435795262259322:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.572671Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435794397668989:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.572684Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435794397668989:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.572796Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435793240588556:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.572823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435793240588480:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.573050Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435795262259318:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:21.573124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435796663113522:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.573144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:21.573150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435796663113522:2109] [node 4] ICP22 created new session: [1:7574435796663113549:2109]
2025-11-19T13:59:21.573156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435796663113549:2109] [node 4] ICS09 handshake done sender: [1:7574435796663113531:2114] self: [1:7574435796663113530:0] peer: [4:7574435797132836248:0] socket: 402 qp: -1
2025-11-19T13:59:21.573161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435796663113549:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:21.573179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:21.575780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435793240588559:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15972
2025-11-19T13:59:21.576721Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435797132836244:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.577102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435793240588565:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15972
2025-11-19T13:59:21.577599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435795262259245:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.577620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:21.577627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435795262259245:2048] [node 1] ICP22 created new session: [5:7574435795262259330:2048]
2025-11-19T13:59:21.577634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435795262259330:2048] [node 1] ICS09 handshake done sender: [5:7574435795262259318:2112] self: [5:7574435795262259324:0] peer: [1:7574435796663113525:0] socket: 387 qp: -1
2025-11-19T13:59:21.577641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435795262259330:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:21.577649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:21.577817Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435795262259328:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.577957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435795262259246:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.571269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435795819734291:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55859
2025-11-19T13:59:21.572675Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435795819734207:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(64120280ns) {Configure:104(64118708ns) {ProcessConfigured:135(63737440ns) {StartInitialHandshake:157(63736634ns) {DropHandshakes:469(599ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:476(840ns) {SwitchToState:178(536ns)}}}}}
2025-11-19T13:59:21.572784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435795819734207:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.576467Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435795819734278:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:21.577088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435795819734208:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.577100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:21.577105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435795819734208:2049] [node 2] ICP22 created new session: [3:7574435795819734295:2049]
2025-11-19T13:59:21.577120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435795819734295:2049] [node 2] ICS09 handshake done sender: [3:7574435795819734278:2113] self: [3:7574435795819734289:0] peer: [2:7574435794397668985:0] socket: 403 qp: -1
2025-11-19T13:59:21.577125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435795819734295:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:21.577132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:21.579509Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435797132836237:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:21.579945Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435793240588557:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:21.580046Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435797132836245:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.580059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435797132836166:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.580071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:21.580078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435797132836166:2048] [node 1] ICP22 created new session: [4:7574435797132836255:2048]
2025-11-19T13:59:21.580083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435797132836255:2048] [node 1] ICS09 handshake done sender: [4:7574435797132836244:2115] self: [4:7574435797132836248:0] peer: [1:7574435796663113530:0] socket: 417 qp: -1
2025-11-19T13:59:21.580089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435797132836255:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:21.580097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:21.579393Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435794397668994:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.579438Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435794397668999:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.579557Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435794397669001:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.579561Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435794397669001:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.580230Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435794397668989:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:21.580362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435794397668987:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.580373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:21.580378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435794397668987:2109] [node 6] ICP22 created new session: [2:7574435794397669016:2109]
2025-11-19T13:59:21.580383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435794397669016:2109] [node 6] ICS09 handshake done sender: [2:7574435794397668989:2113] self: [2:7574435794397668988:0] peer: [6:7574435793240588560:0] socket: 413 qp: -1
2025-11-19T13:59:21.580389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435794397669016:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:21.580396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:21.580499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435794397668917:2049] [node 3] ICP18 handshake (actor [2:7574435794397668999:2118]) is too old
2025-11-19T13:59:21.578050Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435795262259322:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:21.582971Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435796663113523:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.582983Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435796663113523:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.583052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435795819734284:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.583062Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435795819734284:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.582888Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435794397669001:2120] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:21.584031Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435797132836235:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.584047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435797132836235:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.584232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435797132836227:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.584242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:21.584247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435797132836227:2084] [node 6] ICP22 created new session: [4:7574435797132836257:2084]
2025-11-19T13:59:21.584252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435797132836257:2084] [node 6] ICS09 handshake done sender: [4:7574435797132836237:2112] self: [4:7574435797132836236:0] peer: [6:7574435793240588562:0] socket: 409 qp: -1
2025-11-19T13:59:21.584257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435797132836257:2084] [node 6] ICS10 traffic start
2025-11-19T13:59:21.584263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.585618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435797132836168:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.587918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435795262259267:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.587934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:21.587944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435795262259267:2065] [node 6] ICP22 created new session: [5:7574435795262259335:2065]
2025-11-19T13:59:21.587949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435795262259335:2065] [node 6] ICS09 handshake done sender: [5:7574435795262259322:2114] self: [5:7574435795262259268:0] peer: [6:7574435793240588563:0] socket: 381 qp: -1
2025-11-19T13:59:21.587954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435795262259335:2065] [node 6] ICS10 traffic start
2025-11-19T13:59:21.587973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:21.588123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435795262259334:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36885
2025-11-19T13:59:21.588350Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435795819734280:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.588359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435795819734280:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.588943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435795819734297:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55859
2025-11-19T13:59:21.589266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435794397668981:2105] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.589285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:21.589290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435794397668981:2105] [node 5] ICP22 created new session: [2:7574435794397669018:2105]
2025-11-19T13:59:21.589295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435794397669018:2105] [node 5] ICS09 handshake done sender: [2:7574435794397669001:2120] self: [2:7574435794397668995:0] peer: [5:7574435795262259333:0] socket: 422 qp: -1
2025-11-19T13:59:21.589300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435794397669018:2105] [node 5] ICS10 traffic start
2025-11-19T13:59:21.589306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:21.588324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435793240588566:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15972
2025-11-19T13:59:21.589685Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435795262259328:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:21.589763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435795262259246:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.589783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:21.589790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435795262259246:2049] [node 2] ICP22 created new session: [5:7574435795262259338:2049]
2025-11-19T13:59:21.589794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435795262259338:2049] [node 2] ICS09 handshake done sender: [5:7574435795262259328:2118] self: [5:7574435795262259333:0] peer: [2:7574435794397668995:0] socket: 427 qp: -1
2025-11-19T13:59:21.589799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435795262259338:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:21.589804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:21.589655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435793240588499:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.589671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:21.589679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435793240588499:2061] [node 4] ICP22 created new session: [6:7574435793240588568:2061]
2025-11-19T13:59:21.589685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435793240588568:2061] [node 4] ICS09 handshake done sender: [6:7574435793240588557:2116] self: [6:7574435793240588562:0] peer: [4:7574435797132836236:0] socket: 415 qp: -1
2025-11-19T13:59:21.589691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435793240588568:2061] [node 4] ICS10 traffic start
2025-11-19T13:59:21.589699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:21.590851Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435796663113523:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:21.591034Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435795819734280:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.592374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435795819734274:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.592387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:21.592393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435795819734274:2110] [node 4] ICP22 created new session: [3:7574435795819734300:2110]
2025-11-19T13:59:21.592398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435795819734300:2110] [node 4] ICS09 handshake done sender: [3:7574435795819734280:2114] self: [3:7574435795819734279:0] peer: [4:7574435797132836260:0] socket: 404 qp: -1
2025-11-19T13:59:21.592403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435795819734300:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:21.592409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:21.592525Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435795819734297:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.592623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435795819734274:2110] [node 4] ICP18 handshake (actor [3:7574435795819734297:2120]) is too old
2025-11-19T13:59:21.592644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435795819734299:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55859
2025-11-19T13:59:21.592912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435796663113522:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(19761394ns) {HandleHandshakeStatus:329(19760501ns) {DropIncomingHandshake:441(3925ns) DropOutgoingHandshake:456(227ns) SwitchToState:178(42ns) ProcessPendingSessionEvents:495(9780136ns) {ForwardSessionEventToSession:612(9769336ns) ForwardSessionEventToSession:612(1342ns) ForwardSessionEventToSession:612(997ns) ForwardSessionEventToSession:612(614ns) ForwardSessionEventToSession:612(885ns) ForwardSessionEventToSession:612(916ns) ForwardSessionEventToSession:612(590ns) ForwardSessionEventToSession:612(798ns) ForwardSessionEventToSession:612(426ns) ForwardSessionEventToSession:612(784ns)}}}
2025-11-19T13:59:21.592928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435796663113455:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.592935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:21.592940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435796663113455:2049] [node 3] ICP22 created new session: [1:7574435796663113552:2049]
2025-11-19T13:59:21.592944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435796663113552:2049] [node 3] ICS09 handshake done sender: [1:7574435796663113523:2111] self: [1:7574435796663113515:0] peer: [3:7574435795819734294:0] socket: 383 qp: -1
2025-11-19T13:59:21.592948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435796663113552:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:21.592952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:21.596304Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435793240588559:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.602984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435793240588567:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15972
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.604910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435795262259340:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36885
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.605473Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435796663113533:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.605485Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435796663113533:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.606472Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435796663113533:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:21.606568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435796663113519:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.606586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:21.606591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435796663113519:2107] [node 6] ICP22 created new session: [1:7574435796663113555:2107]
2025-11-19T13:59:21.606596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435796663113555:2107] [node 6] ICS09 handshake done sender: [1:7574435796663113533:2115] self: [1:7574435796663113532:0] peer: [6:7574435793240588574:0] socket: 382 qp: -1
2025-11-19T13:59:21.606601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435796663113555:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:21.606610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.603907Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435793240588561:2119] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:21.604296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435793240588479:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435793240588558:2117] held: no
2025-11-19T13:59:21.604321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435793240588495:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.604333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:21.604340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435793240588495:2059] [node 5] ICP22 created new session: [6:7574435793240588572:2059]
2025-11-19T13:59:21.604345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435793240588572:2059] [node 5] ICS09 handshake done sender: [6:7574435793240588561:2119] self: [6:7574435793240588563:0] peer: [5:7574435795262259268:0] socket: 412 qp: -1
2025-11-19T13:59:21.604351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435793240588572:2059] [node 5] ICS10 traffic start
2025-11-19T13:59:21.604359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:21.605310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435793240588478:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.605336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435793240588570:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15972
2025-11-19T13:59:21.607943Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435793240588478:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16103) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435793240588559:2118] held: no
2025-11-19T13:59:21.607995Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435793240588558:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:21.608056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435793240588575:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15972
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.609779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435797132836261:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4702
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.612441Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435795819734284:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:21.612151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435797132836262:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4702
2025-11-19T13:59:21.612581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435793240588479:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.612600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:21.612606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435793240588479:2049] [node 2] ICP22 created new session: [6:7574435793240588576:2049]
2025-11-19T13:59:21.612612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435793240588576:2049] [node 2] ICS09 handshake done sender: [6:7574435793240588558:2117] self: [6:7574435793240588560:0] peer: [2:7574435794397668988:0] socket: 416 qp: -1
2025-11-19T13:59:21.612636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435793240588576:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:21.612656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:21.620730Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435797132836262:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.620754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435797132836168:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435797132836245:2116] held: no
2025-11-19T13:59:21.620804Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435797132836245:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.621468Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435795819734273:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:21.621603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435795262259341:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36885
2025-11-19T13:59:21.621621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435795262259342:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36885
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.622328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435796663113557:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59198
2025-11-19T13:59:21.621841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435795819734272:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.621858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:21.621863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435795819734272:2109] [node 6] ICP22 created new session: [3:7574435795819734304:2109]
2025-11-19T13:59:21.621870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435795819734304:2109] [node 6] ICS09 handshake done sender: [3:7574435795819734284:2116] self: [3:7574435795819734283:0] peer: [6:7574435793240588564:0] socket: 406 qp: -1
2025-11-19T13:59:21.621875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435795819734304:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:21.621882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:21.622614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435795819734207:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.622702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:21.622709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435795819734207:2048] [node 1] ICP22 created new session: [3:7574435795819734307:2048]
2025-11-19T13:59:21.622714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435795819734307:2048] [node 1] ICS09 handshake done sender: [3:7574435795819734273:2111] self: [3:7574435795819734294:0] peer: [1:7574435796663113515:0] socket: 385 qp: -1
2025-11-19T13:59:21.622718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435795819734307:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:21.622724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:21.624899Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435797132836235:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:21.625039Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435795262259326:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:21.625129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435795262259272:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.625142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:21.625147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435795262259272:2062] [node 4] ICP22 created new session: [5:7574435795262259343:2062]
2025-11-19T13:59:21.625152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435795262259343:2062] [node 4] ICS09 handshake done sender: [5:7574435795262259326:2116] self: [5:7574435795262259329:0] peer: [4:7574435797132836234:0] socket: 392 qp: -1
2025-11-19T13:59:21.625158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435795262259343:2062] [node 4] ICS10 traffic start
2025-11-19T13:59:21.625164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:21.625962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435797132836231:2082] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.625979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:21.625984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435797132836231:2082] [node 5] ICP22 created new session: [4:7574435797132836265:2082]
2025-11-19T13:59:21.625988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435797132836265:2082] [node 5] ICS09 handshake done sender: [4:7574435797132836235:2111] self: [4:7574435797132836234:0] peer: [5:7574435795262259329:0] socket: 408 qp: -1
2025-11-19T13:59:21.625993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435797132836265:2082] [node 5] ICS10 traffic start
2025-11-19T13:59:21.626000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:21.626095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435797132836168:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.626101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:21.626105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435797132836168:2050] [node 3] ICP22 created new session: [4:7574435797132836267:2050]
2025-11-19T13:59:21.626109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435797132836267:2050] [node 3] ICS09 handshake done sender: [4:7574435797132836245:2116] self: [4:7574435797132836260:0] peer: [3:7574435795819734279:0] socket: 418 qp: -1
2025-11-19T13:59:21.626113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435797132836267:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:21.626117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:21.620861Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435793240588559:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:21.621060Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435793240588556:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:21.623104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435793240588478:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.623113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:21.623118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435793240588478:2048] [node 1] ICP22 created new session: [6:7574435793240588578:2048]
2025-11-19T13:59:21.623122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435793240588578:2048] [node 1] ICS09 handshake done sender: [6:7574435793240588559:2118] self: [6:7574435793240588574:0] peer: [1:7574435796663113532:0] socket: 407 qp: -1
2025-11-19T13:59:21.623127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435793240588578:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:21.623133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:21.623484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435793240588480:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.623492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:21.623497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435793240588480:2050] [node 3] ICP22 created new session: [6:7574435793240588580:2050]
2025-11-19T13:59:21.623501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435793240588580:2050] [node 3] ICS09 handshake done sender: [6:7574435793240588556:2115] self: [6:7574435793240588564:0] peer: [3:7574435795819734283:0] socket: 414 qp: -1
2025-11-19T13:59:21.623504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435793240588580:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:21.623509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:21.635393Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435795262259341:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.635426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435795262259247:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:21.635551Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435795819734288:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:21.635560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435795819734288:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:21.636441Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435795819734288:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:21.636533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435795819734269:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:21.636549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:21.636555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435795819734269:2108] [node 5] ICP22 created new session: [3:7574435795819734310:2108]
2025-11-19T13:59:21.636560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435795819734310:2108] [node 5] ICS09 handshake done sender: [3:7574435795819734288:2117] self: [3:7574435795819734276:0] peer: [5:7574435795262259345:0] socket: 396 qp: -1
2025-11-19T13:59:21.636565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435795819734310:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:21.636574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:21.636888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435795262259346:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36885
2025-11-19T13:59:21.637970Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435795262259247:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13274) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435795262259341:2121] held: no
2025-11-19T13:59:21.637994Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435795262259341:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:21.646987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435795262259247:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:21.647008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:21.647017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435795262259247:2050] [node 3] ICP22 created new session: [5:7574435795262259347:2050]
2025-11-19T13:59:21.647023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435795262259347:2050] [node 3] ICS09 handshake done sender: [5:7574435795262259341:2121] self: [5:7574435795262259345:0] peer: [3:7574435795819734276:0] socket: 411 qp: -1
2025-11-19T13:59:21.647031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435795262259347:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:21.647039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:21.667939Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:21.668034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435794397669002:2048] [node 1] ICS01 socket: 379 reason# EndOfStream
2025-11-19T13:59:21.668044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435794397668916:2048] [node 1] ICP30 unregister session Session# [2:7574435794397669002:2048] VirtualId# [2:7574435794397668996:0]
2025-11-19T13:59:21.668053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435794397669002:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.668094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:21.668122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:21.675902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:21.675993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435795819734307:2048] [node 1] ICS01 socket: 385 reason# EndOfStream
2025-11-19T13:59:21.676003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435795819734207:2048] [node 1] ICP30 unregister session Session# [3:7574435795819734307:2048] VirtualId# [3:7574435795819734294:0]
2025-11-19T13:59:21.676012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435795819734307:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.676051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:21.676074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:21.694011Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:21.694119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435795262259330:2048] [node 1] ICS01 socket: 387 reason# EndOfStream
2025-11-19T13:59:21.694129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435795262259245:2048] [node 1] ICP30 unregister session Session# [5:7574435795262259330:2048] VirtualId# [5:7574435795262259324:0]
2025-11-19T13:59:21.694137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435795262259330:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.694193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:21.694219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:21.694304Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:21.694364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435797132836255:2048] [node 1] ICS01 socket: 417 reason# EndOfStream
2025-11-19T13:59:21.694370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435797132836166:2048] [node 1] ICP30 unregister session Session# [4:7574435797132836255:2048] VirtualId# [4:7574435797132836248:0]
2025-11-19T13:59:21.694374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435797132836255:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.694398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:21.694419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:21.696325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:21.696404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435793240588578:2048] [node 1] ICS01 socket: 407 reason# EndOfStream
2025-11-19T13:59:21.696411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435793240588478:2048] [node 1] ICP30 unregister session Session# [6:7574435793240588578:2048] VirtualId# [6:7574435793240588574:0]
2025-11-19T13:59:21.696417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435793240588578:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.696467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:21.696489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:21.704032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:21.704137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435795819734295:2049] [node 2] ICS01 socket: 403 reason# EndOfStream
2025-11-19T13:59:21.704148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435795819734208:2049] [node 2] ICP30 unregister session Session# [3:7574435795819734295:2049] VirtualId# [3:7574435795819734289:0]
2025-11-19T13:59:21.704156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435795819734295:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.704211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:21.704234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:21.707653Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:21.707777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435795262259338:2049] [node 2] ICS01 socket: 427 reason# EndOfStream
2025-11-19T13:59:21.707785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435795262259246:2049] [node 2] ICP30 unregister session Session# [5:7574435795262259338:2049] VirtualId# [5:7574435795262259333:0]
2025-11-19T13:59:21.707792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435795262259338:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.707852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:21.707879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:21.707951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:21.708012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435797132836250:2049] [node 2] ICS01 socket: 401 reason# EndOfStream
2025-11-19T13:59:21.708016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435797132836167:2049] [node 2] ICP30 unregister session Session# [4:7574435797132836250:2049] VirtualId# [4:7574435797132836247:0]
2025-11-19T13:59:21.708021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435797132836250:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.708049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:21.708070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:21.709668Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:21.709752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435793240588576:2049] [node 2] ICS01 socket: 416 reason# EndOfStream
2025-11-19T13:59:21.709759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435793240588479:2049] [node 2] ICP30 unregister session Session# [6:7574435793240588576:2049] VirtualId# [6:7574435793240588560:0]
2025-11-19T13:59:21.709767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435793240588576:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.709817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:21.709840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:21.728013Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:21.728104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435793240588580:2050] [node 3] ICS01 socket: 414 reason# EndOfStream
2025-11-19T13:59:21.728114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435793240588480:2050] [node 3] ICP30 unregister session Session# [6:7574435793240588580:2050] VirtualId# [6:7574435793240588564:0]
2025-11-19T13:59:21.728120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435793240588580:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.728175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:21.728201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:21.728981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:21.729055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435795262259347:2050] [node 3] ICS01 socket: 411 reason# EndOfStream
2025-11-19T13:59:21.729061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435795262259247:2050] [node 3] ICP30 unregister session Session# [5:7574435795262259347:2050] VirtualId# [5:7574435795262259345:0]
2025-11-19T13:59:21.729070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435795262259347:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.729117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:21.729138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:21.774137Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:21.774231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435793240588568:2061] [node 4] ICS01 socket: 415 reason# EndOfStream
2025-11-19T13:59:21.774240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435793240588499:2061] [node 4] ICP30 unregister session Session# [6:7574435793240588568:2061] VirtualId# [6:7574435793240588562:0]
2025-11-19T13:59:21.774247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435793240588568:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:21.774308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:21.774333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:22.119972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435799442419528:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:22.119990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435799442419529:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:22.179971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435797907898531:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:22.179986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435797907898532:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:22.233343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435801421444361:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:22.233355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435801421444362:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:22.298743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435798788824082:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:22.298755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435798788824083:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:22.298759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435798788824084:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:22.399855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435800357789503:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:22.399872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435800357789504:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:22.399876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435800357789505:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:22.483861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435798924292731:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:22.483875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435798924292732:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:22.483879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435798924292733:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:22.528130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435800357789521:2061] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.528634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435798788824143:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27747
2025-11-19T13:59:22.527642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435797907898592:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:22.527876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435799442419591:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:22.527978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435799442419592:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:22.528064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435799442419593:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.530576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435800357789534:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44087
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.530960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435797907898596:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40205
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.531559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435797907898603:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40205
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.533694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435800357789546:2078] [node 6] ICP01 ready to work
2025-11-19T13:59:22.533756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435798924292767:2077] [node 5] ICP01 ready to work
2025-11-19T13:59:22.534353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435799442419608:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36715
2025-11-19T13:59:22.536560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435799442419607:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36715
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-19T13:59:22.537623Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435797907898596:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-11-19T13:59:22.536253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435801421444421:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50788
2025-11-19T13:59:22.542837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435798788824145:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:22.543506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435797907898606:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:22.543953Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435799442419607:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.544672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435797907898608:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40205
2025-11-19T13:59:22.546288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435798924292757:2069] [node 4] ICP01 ready to work
2025-11-19T13:59:22.545542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435801421444423:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50788
2025-11-19T13:59:22.546499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435801421444422:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50788
2025-11-19T13:59:22.546896Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435799442419595:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.546912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435799442419595:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:22.547357Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435797907898603:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.547486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435797907898610:2115] [node 5] ICP01 ready to work
2025-11-19T13:59:22.547827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435797907898610:2115] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.555877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435800357789521:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(27494185ns) {Configure:104(27488248ns) {ProcessConfigured:135(25482845ns) {StartInitialHandshake:157(25481777ns) {DropHandshakes:469(270ns) {DropIncomingHandshake:441(23ns) DropOutgoingHandshake:456(19ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:476(1988ns) {SwitchToState:178(1235ns)}}}}}
2025-11-19T13:59:22.557502Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435797907898608:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.557308Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435801421444422:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.558100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435801421444430:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50788
2025-11-19T13:59:22.559429Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435798788824083:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16542909ns) {Configure:104(16541466ns) {ProcessConfigured:135(16127730ns) {StartInitialHandshake:157(16127529ns) {DropHandshakes:469(892ns) {DropIncomingHandshake:441(179ns) DropOutgoingHandshake:456(146ns)} GenerateSessionVirtualId:988(124ns) PrepareNewSessionHandshake:476(980ns) {SwitchToState:178(329ns)}}}}}
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.559966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435798788824151:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27747
::ffff:127.1.0.0
2025-11-19T13:59:22.560058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435799442419610:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36715
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.559296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435800357789546:2078] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25533518ns) {Configure:104(25532525ns) {ProcessConfigured:135(25193630ns) {StartInitialHandshake:157(25193237ns) {DropHandshakes:469(821ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(317ns) PrepareNewSessionHandshake:476(1367ns) {SwitchToState:178(717ns)}}}}}
2025-11-19T13:59:22.561401Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435800357789536:2074] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.561420Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435800357789536:2074] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:22.561790Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435801421444421:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.561845Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435801421444430:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.564985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435797907898531:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.565033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435797907898531:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(27343910ns) {IncomingHandshake:261(27342715ns) {DropIncomingHandshake:441(99ns) PrepareNewSessionHandshake:476(580ns) {SwitchToState:178(145ns)} GenerateSessionVirtualId:988(198ns) IssueIncomingHandshakeReply:193(27337489ns) {SwitchToState:178(527ns)}}}
2025-11-19T13:59:22.571772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435799442419611:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36715
2025-11-19T13:59:22.573012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435798788824145:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(29654926ns) {Configure:104(27103658ns) {ProcessConfigured:135(26784855ns) {StartInitialHandshake:157(26783437ns) {DropHandshakes:469(621ns) {DropIncomingHandshake:441(162ns) DropOutgoingHandshake:456(103ns)} GenerateSessionVirtualId:988(139ns) PrepareNewSessionHandshake:476(1167ns) {SwitchToState:178(49ns)}}}}}
2025-11-19T13:59:22.573277Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435800357789503:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(44728241ns) {Configure:104(44728044ns) {ProcessConfigured:135(43011656ns) {StartInitialHandshake:157(43011106ns) {DropHandshakes:469(205ns) {DropIncomingHandshake:441(22ns) DropOutgoingHandshake:456(36ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(727ns) {SwitchToState:178(561ns)}}}}}
2025-11-19T13:59:22.573327Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435800357789534:2073] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.573363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435800357789503:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.573521Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435798924292757:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(27169214ns) {Configure:104(27168948ns) {ProcessConfigured:135(26818655ns) {StartInitialHandshake:157(26818526ns) {DropHandshakes:469(203ns) {DropIncomingHandshake:441(23ns) DropOutgoingHandshake:456(42ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(1047ns) {SwitchToState:178(21ns)}}}}}
2025-11-19T13:59:22.573751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435801421444428:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:22.573804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435801421444361:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.574314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435801421444427:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:22.574375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435801421444429:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.575948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435798788824149:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:22.579727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435797907898612:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40205
2025-11-19T13:59:22.579776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435797907898613:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40205
2025-11-19T13:59:22.583560Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435800357789536:2074] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:22.585183Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435798788824151:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.585061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435800357789504:2049] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.585084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:22.585091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435800357789504:2049] [node 2] ICP22 created new session: [5:7574435800357789584:2049]
2025-11-19T13:59:22.585099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435800357789584:2049] [node 2] ICS09 handshake done sender: [5:7574435800357789536:2074] self: [5:7574435800357789527:0] peer: [2:7574435797907898611:0] socket: 395 qp: -1
2025-11-19T13:59:22.585107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435800357789584:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:22.585115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:22.586382Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435799442419611:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.586526Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435799442419597:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.586546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435799442419597:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:22.586586Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435799442419599:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.586597Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435799442419599:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:22.586684Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435798788824143:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.586707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435798788824083:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.586800Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435799442419608:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.586870Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435797907898613:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.594395Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435797907898606:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(50832269ns) {Configure:104(50831796ns) {ProcessConfigured:135(50522355ns) {StartInitialHandshake:157(50521939ns) {DropHandshakes:469(380ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(973ns) {SwitchToState:178(788ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.595055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435800357789587:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44087
2025-11-19T13:59:22.595127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435800357789588:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44087
2025-11-19T13:59:22.592891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435801421444428:2112] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19010695ns) {Configure:104(19010135ns) {ProcessConfigured:135(18673918ns) {StartInitialHandshake:157(18655636ns) {DropHandshakes:469(534ns) {DropIncomingHandshake:441(25ns) DropOutgoingHandshake:456(152ns)} GenerateSessionVirtualId:988(731ns) PrepareNewSessionHandshake:476(1255ns) {SwitchToState:178(908ns)}} IncomingHandshake:261(9329ns) {DropIncomingHandshake:441(201ns) PrepareNewSessionHandshake:476(226ns) {SwitchToState:178(151ns)} GenerateSessionVirtualId:988(495ns) IssueIncomingHandshakeReply:193(236ns)}}}}
2025-11-19T13:59:22.595427Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435800357789587:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.595456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435800357789521:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.594692Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435801421444429:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20259227ns) {Configure:104(20258929ns) {ProcessConfigured:135(20025808ns) {StartInitialHandshake:157(19999811ns) {DropHandshakes:469(269ns) {DropIncomingHandshake:441(30ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(287ns) PrepareNewSessionHandshake:476(959ns) {SwitchToState:178(549ns)}} IncomingHandshake:261(17942ns) {DropIncomingHandshake:441(460ns) PrepareNewSessionHandshake:476(201ns) {SwitchToState:178(166ns)} GenerateSessionVirtualId:988(679ns) IssueIncomingHandshakeReply:193(90ns)}}}}
2025-11-19T13:59:22.594895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435801421444427:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(20489318ns) {Configure:104(20488660ns) {ProcessConfigured:135(20199050ns) {StartInitialHandshake:157(20198546ns) {DropHandshakes:469(487ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(98ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(603ns) {SwitchToState:178(550ns)}}}}}
2025-11-19T13:59:22.595203Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435798788824149:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19184527ns) {Configure:104(19183685ns) {ProcessConfigured:135(18858045ns) {StartInitialHandshake:157(18857828ns) {DropHandshakes:469(594ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:476(1045ns) {SwitchToState:178(28ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.597763Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435798788824157:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.597773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435798788824157:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.596750Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435797907898594:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.596770Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435797907898594:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:22.597142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435797907898617:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40205
2025-11-19T13:59:22.597639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435797907898618:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40205
2025-11-19T13:59:22.598662Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435797907898603:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:22.598953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435797907898610:2115] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.598967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:22.598975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435797907898610:2115] [node 5] ICP22 created new session: [2:7574435797907898619:2115]
2025-11-19T13:59:22.598982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435797907898619:2115] [node 5] ICS09 handshake done sender: [2:7574435797907898603:2112] self: [2:7574435797907898611:0] peer: [5:7574435800357789527:0] socket: 396 qp: -1
2025-11-19T13:59:22.598989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435797907898619:2115] [node 5] ICS10 traffic start
2025-11-19T13:59:22.598999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:22.599291Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435798788824157:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:22.598884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435800357789590:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44087
2025-11-19T13:59:22.599441Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435800357789587:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:22.599508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435800357789521:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.599519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:22.599526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435800357789521:2061] [node 4] ICP22 created new session: [5:7574435800357789591:2061]
2025-11-19T13:59:22.599532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435800357789591:2061] [node 4] ICS09 handshake done sender: [5:7574435800357789587:2116] self: [5:7574435800357789589:0] peer: [4:7574435798788824148:0] socket: 416 qp: -1
2025-11-19T13:59:22.599538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435800357789591:2061] [node 4] ICS10 traffic start
2025-11-19T13:59:22.599557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:22.599761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435798788824145:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.599783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:22.599789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435798788824145:2109] [node 5] ICP22 created new session: [4:7574435798788824164:2109]
2025-11-19T13:59:22.599794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435798788824164:2109] [node 5] ICS09 handshake done sender: [4:7574435798788824157:2114] self: [4:7574435798788824148:0] peer: [5:7574435800357789589:0] socket: 415 qp: -1
2025-11-19T13:59:22.599798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435798788824164:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:22.599805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:22.599547Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435799442419595:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:22.599854Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435801421444423:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.599899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435801421444362:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.600470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435798924292809:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30303
2025-11-19T13:59:22.601952Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435799442419599:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:22.602047Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435799442419597:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:22.602147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435799442419528:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.602171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:22.602178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435799442419528:2048] [node 2] ICP22 created new session: [1:7574435799442419617:2048]
2025-11-19T13:59:22.602184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435799442419617:2048] [node 2] ICS09 handshake done sender: [1:7574435799442419595:2110] self: [1:7574435799442419594:0] peer: [2:7574435797907898604:0] socket: 386 qp: -1
2025-11-19T13:59:22.602191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435799442419617:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:22.602208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:22.602319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435799442419591:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.602326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:22.602330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435799442419591:2107] [node 5] ICP22 created new session: [1:7574435799442419619:2107]
2025-11-19T13:59:22.602334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435799442419619:2107] [node 5] ICS09 handshake done sender: [1:7574435799442419599:2112] self: [1:7574435799442419598:0] peer: [5:7574435800357789582:0] socket: 388 qp: -1
2025-11-19T13:59:22.602337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435799442419619:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:22.602341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:22.602581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435799442419529:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.602592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:22.602597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435799442419529:2049] [node 3] ICP22 created new session: [1:7574435799442419621:2049]
2025-11-19T13:59:22.602602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435799442419621:2049] [node 3] ICS09 handshake done sender: [1:7574435799442419597:2111] self: [1:7574435799442419596:0] peer: [3:7574435801421444437:0] socket: 387 qp: -1
2025-11-19T13:59:22.602607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435799442419621:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:22.602613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.603047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435798924292810:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30303
2025-11-19T13:59:22.604010Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435797907898612:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.604101Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435797907898596:2109] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.605669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435797907898531:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.605687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:22.605694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435797907898531:2048] [node 1] ICP22 created new session: [2:7574435797907898621:2048]
2025-11-19T13:59:22.605700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435797907898621:2048] [node 1] ICS09 handshake done sender: [2:7574435797907898596:2109] self: [2:7574435797907898604:0] peer: [1:7574435799442419594:0] socket: 390 qp: -1
2025-11-19T13:59:22.605705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435797907898621:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:22.605713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:22.605113Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435799442419610:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.605142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435799442419529:2049] [node 3] ICP18 handshake (actor [1:7574435799442419610:2117]) is too old
2025-11-19T13:59:22.606535Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435797907898594:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:22.605491Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435798788824084:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(38017420ns) {Configure:104(38016412ns) {ProcessConfigured:135(37636242ns) {StartInitialHandshake:157(37635102ns) {DropHandshakes:469(881ns) {DropIncomingHandshake:441(126ns) DropOutgoingHandshake:456(175ns)} GenerateSessionVirtualId:988(805ns) PrepareNewSessionHandshake:476(599ns) {SwitchToState:178(244ns)}}}}}
2025-11-19T13:59:22.605717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435798788824167:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27747
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.606889Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435798924292809:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.607031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435798924292767:2077] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.607053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435798924292811:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30303
2025-11-19T13:59:22.607537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435797907898592:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.607557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:22.607563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435797907898592:2107] [node 4] ICP22 created new session: [2:7574435797907898625:2107]
2025-11-19T13:59:22.607569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435797907898625:2107] [node 4] ICS09 handshake done sender: [2:7574435797907898594:2108] self: [2:7574435797907898593:0] peer: [4:7574435798788824161:0] socket: 384 qp: -1
2025-11-19T13:59:22.607576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435797907898625:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:22.607583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.615863Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435797907898600:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.615898Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435797907898600:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:22.616300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435801421444361:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27535) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435801421444421:2107] held: no
2025-11-19T13:59:22.616390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435798788824169:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27747
2025-11-19T13:59:22.616632Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435798924292810:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.616682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435798924292731:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.616500Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435801421444441:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.616506Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435801421444441:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:22.617565Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435801421444441:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:22.618065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435801421444428:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.618088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:22.618096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435801421444428:2112] [node 5] ICP22 created new session: [3:7574435801421444449:2112]
2025-11-19T13:59:22.618102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435801421444449:2112] [node 5] ICS09 handshake done sender: [3:7574435801421444441:2116] self: [3:7574435801421444438:0] peer: [5:7574435800357789594:0] socket: 423 qp: -1
2025-11-19T13:59:22.618110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435801421444449:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:22.618133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:22.616179Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435800357789588:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.616230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435800357789505:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.616305Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435800357789579:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.616311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435800357789579:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:22.616346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435800357789593:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44087
2025-11-19T13:59:22.619954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435798788824145:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(20179983ns) {HandleHandshakeStatus:329(20179193ns) {DropIncomingHandshake:441(4529ns) DropOutgoingHandshake:456(89ns) SwitchToState:178(82ns) ProcessPendingSessionEvents:495(15551074ns) {ForwardSessionEventToSession:612(15536583ns) ForwardSessionEventToSession:612(2023ns) ForwardSessionEventToSession:612(1352ns) ForwardSessionEventToSession:612(786ns) ForwardSessionEventToSession:612(681ns) ForwardSessionEventToSession:612(1215ns) ForwardSessionEventToSession:612(607ns) ForwardSessionEventToSession:612(574ns) ForwardSessionEventToSession:612(628ns) ForwardSessionEventToSession:612(1256ns)}}}
2025-11-19T13:59:22.620028Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435798788824167:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.620174Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435798788824143:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:22.622009Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435799442419603:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.622030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435799442419603:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.627973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435801421444452:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50788
2025-11-19T13:59:22.628724Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435801421444421:2107] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.629007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435801421444361:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.629033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:22.629040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435801421444361:2048] [node 1] ICP22 created new session: [3:7574435801421444453:2048]
2025-11-19T13:59:22.629046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435801421444453:2048] [node 1] ICS09 handshake done sender: [3:7574435801421444421:2107] self: [3:7574435801421444437:0] peer: [1:7574435799442419596:0] socket: 399 qp: -1
2025-11-19T13:59:22.629054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435801421444453:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:22.629064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:22.628184Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435800357789505:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435800357789588:2117] held: no
2025-11-19T13:59:22.628331Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435800357789534:2073] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:22.628935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435800357789595:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44087
2025-11-19T13:59:22.629513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435800357789503:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.629524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:22.629530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435800357789503:2048] [node 1] ICP22 created new session: [5:7574435800357789596:2048]
2025-11-19T13:59:22.629536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435800357789596:2048] [node 1] ICS09 handshake done sender: [5:7574435800357789534:2073] self: [5:7574435800357789582:0] peer: [1:7574435799442419598:0] socket: 391 qp: -1
2025-11-19T13:59:22.629541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435800357789596:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:22.629546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:22.629795Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435800357789588:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:22.630037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435800357789505:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.630044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:22.630058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435800357789505:2050] [node 3] ICP22 created new session: [5:7574435800357789599:2050]
2025-11-19T13:59:22.630062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435800357789599:2050] [node 3] ICS09 handshake done sender: [5:7574435800357789588:2117] self: [5:7574435800357789594:0] peer: [3:7574435801421444438:0] socket: 424 qp: -1
2025-11-19T13:59:22.630066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435800357789599:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:22.630072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.637945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435801421444455:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50788
2025-11-19T13:59:22.638100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435798788824083:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.638126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:22.638134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435798788824083:2049] [node 2] ICP22 created new session: [4:7574435798788824170:2049]
2025-11-19T13:59:22.638141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435798788824170:2049] [node 2] ICS09 handshake done sender: [4:7574435798788824143:2108] self: [4:7574435798788824161:0] peer: [2:7574435797907898593:0] socket: 385 qp: -1
2025-11-19T13:59:22.638149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435798788824170:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:22.638157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.639396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435798924292814:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30303
2025-11-19T13:59:22.639468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435798924292815:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30303
2025-11-19T13:59:22.639480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435798924292816:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30303
2025-11-19T13:59:22.639779Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435800357789579:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:22.640072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435798788824082:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.640133Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435798788824083:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:22.640448Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435798924292814:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.640477Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435798924292809:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:22.640507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435798924292733:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.640657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435798924292767:2077] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.640669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:22.640674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435798924292767:2077] [node 5] ICP22 created new session: [6:7574435798924292818:2077]
2025-11-19T13:59:22.640679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435798924292818:2077] [node 5] ICS09 handshake done sender: [6:7574435798924292809:2115] self: [6:7574435798924292812:0] peer: [5:7574435800357789547:0] socket: 420 qp: -1
2025-11-19T13:59:22.640683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435798924292818:2077] [node 5] ICS10 traffic start
2025-11-19T13:59:22.640730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:22.640441Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435799442419601:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.640454Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435799442419601:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:22.641006Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435797907898600:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:22.641167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435797907898532:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.641184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:22.641191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435797907898532:2049] [node 3] ICP22 created new session: [2:7574435797907898629:2049]
2025-11-19T13:59:22.641196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435797907898629:2049] [node 3] ICS09 handshake done sender: [2:7574435797907898600:2111] self: [2:7574435797907898599:0] peer: [3:7574435801421444447:0] socket: 393 qp: -1
2025-11-19T13:59:22.641200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435797907898629:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:22.641205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:22.641273Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435801421444444:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.641284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435801421444444:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.642036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435801421444456:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50788
2025-11-19T13:59:22.639937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435800357789546:2078] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.639957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:22.639964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435800357789546:2078] [node 6] ICP22 created new session: [5:7574435800357789602:2078]
2025-11-19T13:59:22.639970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435800357789602:2078] [node 6] ICS09 handshake done sender: [5:7574435800357789579:2113] self: [5:7574435800357789547:0] peer: [6:7574435798924292812:0] socket: 419 qp: -1
2025-11-19T13:59:22.639977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435800357789602:2078] [node 6] ICS10 traffic start
2025-11-19T13:59:22.639985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:22.643052Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435801421444444:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:22.643208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435801421444362:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3485) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435801421444423:2109] held: no
2025-11-19T13:59:22.643231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435801421444429:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.643244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:22.643250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435801421444429:2110] [node 6] ICP22 created new session: [3:7574435801421444458:2110]
2025-11-19T13:59:22.643255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435801421444458:2110] [node 6] ICS09 handshake done sender: [3:7574435801421444444:2117] self: [3:7574435801421444439:0] peer: [6:7574435798924292817:0] socket: 400 qp: -1
2025-11-19T13:59:22.643261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435801421444458:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:22.643267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:22.643560Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435799442419601:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:22.643666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435799442419592:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.643682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:22.643687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435799442419592:2108] [node 4] ICP22 created new session: [1:7574435799442419626:2108]
2025-11-19T13:59:22.643693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435799442419626:2108] [node 4] ICS09 handshake done sender: [1:7574435799442419601:2113] self: [1:7574435799442419600:0] peer: [4:7574435798788824172:0] socket: 430 qp: -1
2025-11-19T13:59:22.643697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435799442419626:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:22.643703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:22.644165Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435801421444423:2109] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:22.645547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435801421444362:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.645563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:22.645571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435801421444362:2049] [node 2] ICP22 created new session: [3:7574435801421444461:2049]
2025-11-19T13:59:22.645578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435801421444461:2049] [node 2] ICS09 handshake done sender: [3:7574435801421444423:2109] self: [3:7574435801421444447:0] peer: [2:7574435797907898599:0] socket: 401 qp: -1
2025-11-19T13:59:22.645583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435801421444461:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:22.645590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:22.650579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435798788824082:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27535) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435798788824167:2117] held: no
2025-11-19T13:59:22.651053Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435801421444456:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.651535Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435799442419603:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:22.651624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435799442419593:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.651641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:22.651652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435799442419593:2109] [node 6] ICP22 created new session: [1:7574435799442419629:2109]
2025-11-19T13:59:22.651658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435799442419629:2109] [node 6] ICS09 handshake done sender: [1:7574435799442419603:2114] self: [1:7574435799442419602:0] peer: [6:7574435798924292813:0] socket: 431 qp: -1
2025-11-19T13:59:22.651665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435799442419629:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:22.651674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.672101Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435798924292733:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435798924292814:2118] held: no
2025-11-19T13:59:22.672173Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435798924292731:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27535) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435798924292810:2116] held: no
2025-11-19T13:59:22.672237Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435798924292811:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.672267Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435798924292815:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.672485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435798924292767:2077] [node 5] ICP35 event processing took too much time PendingConnection:278(31793736ns) {HandleHandshakeStatus:329(31793443ns) {DropIncomingHandshake:441(168ns) DropOutgoingHandshake:456(3265ns) SwitchToState:178(42ns) ProcessPendingSessionEvents:495(11973670ns) {ForwardSessionEventToSession:612(11960574ns) ForwardSessionEventToSession:612(2521ns) ForwardSessionEventToSession:612(734ns) ForwardSessionEventToSession:612(1053ns) ForwardSessionEventToSession:612(527ns) ForwardSessionEventToSession:612(1292ns) ForwardSessionEventToSession:612(654ns) ForwardSessionEventToSession:612(497ns) ForwardSessionEventToSession:612(519ns) ForwardSessionEventToSession:612(1372ns)}}}
2025-11-19T13:59:22.672504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435798924292821:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30303
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.672935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435798924292757:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.672956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435798924292732:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.673895Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435798924292810:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:22.673263Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435797907898615:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.673276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435797907898615:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:22.675587Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435797907898615:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:22.675712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435797907898606:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.675738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:22.675766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435797907898606:2113] [node 6] ICP22 created new session: [2:7574435797907898632:2113]
2025-11-19T13:59:22.675774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435797907898632:2113] [node 6] ICS09 handshake done sender: [2:7574435797907898615:2118] self: [2:7574435797907898607:0] peer: [6:7574435798924292824:0] socket: 398 qp: -1
2025-11-19T13:59:22.675782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435797907898632:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:22.675793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:22.673239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435798788824174:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27747
2025-11-19T13:59:22.673323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435798788824175:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27747
2025-11-19T13:59:22.676072Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435798788824162:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.676081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435798788824162:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:22.676650Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435798788824175:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.676673Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435798788824167:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.676859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435798924292820:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30303
2025-11-19T13:59:22.678752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435798924292731:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.678768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:22.678777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435798924292731:2048] [node 1] ICP22 created new session: [6:7574435798924292825:2048]
2025-11-19T13:59:22.678784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435798924292825:2048] [node 1] ICS09 handshake done sender: [6:7574435798924292810:2116] self: [6:7574435798924292813:0] peer: [1:7574435799442419602:0] socket: 432 qp: -1
2025-11-19T13:59:22.678792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435798924292825:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:22.678801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:22.679053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435798924292827:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30303
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.679247Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435801421444446:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.679261Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435801421444446:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:22.679986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435798924292828:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30303
2025-11-19T13:59:22.676951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435798788824176:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27747
2025-11-19T13:59:22.677076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435798788824084:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:22.677178Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435798788824162:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:22.680272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435798788824179:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27747
2025-11-19T13:59:22.681096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435798788824082:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.681109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:22.681118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435798788824082:2048] [node 1] ICP22 created new session: [4:7574435798788824180:2048]
2025-11-19T13:59:22.681124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435798788824180:2048] [node 1] ICS09 handshake done sender: [4:7574435798788824167:2117] self: [4:7574435798788824172:0] peer: [1:7574435799442419600:0] socket: 435 qp: -1
2025-11-19T13:59:22.681131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435798788824180:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:22.681141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:22.681725Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435798924292814:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:22.681860Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435798924292732:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3485) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435798924292815:2119] held: no
2025-11-19T13:59:22.681973Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435798788824176:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:22.681996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435798788824149:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.682004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:22.682012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435798788824149:2110] [node 6] ICP22 created new session: [4:7574435798788824182:2110]
2025-11-19T13:59:22.682017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435798788824182:2110] [node 6] ICS09 handshake done sender: [4:7574435798788824162:2115] self: [4:7574435798788824159:0] peer: [6:7574435798924292823:0] socket: 436 qp: -1
2025-11-19T13:59:22.682023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435798788824182:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:22.682030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:22.682161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435798788824149:2110] [node 6] ICP18 handshake (actor [4:7574435798788824176:2121]) is too old
2025-11-19T13:59:22.682373Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435798924292811:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:22.682413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435798924292733:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.682424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:22.682429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435798924292733:2050] [node 3] ICP22 created new session: [6:7574435798924292829:2050]
2025-11-19T13:59:22.682434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435798924292829:2050] [node 3] ICS09 handshake done sender: [6:7574435798924292814:2118] self: [6:7574435798924292817:0] peer: [3:7574435801421444439:0] socket: 440 qp: -1
2025-11-19T13:59:22.682439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435798924292829:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:22.682445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:22.682653Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435801421444446:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:22.682841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435801421444427:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:22.682856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:22.682860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435801421444427:2111] [node 4] ICP22 created new session: [3:7574435801421444465:2111]
2025-11-19T13:59:22.682864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435801421444465:2111] [node 4] ICS09 handshake done sender: [3:7574435801421444446:2118] self: [3:7574435801421444440:0] peer: [4:7574435798788824178:0] socket: 405 qp: -1
2025-11-19T13:59:22.682869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435801421444465:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:22.682877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:22.683209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435798924292757:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.683221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:22.683225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435798924292757:2069] [node 4] ICP22 created new session: [6:7574435798924292831:2069]
2025-11-19T13:59:22.683229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435798924292831:2069] [node 4] ICS09 handshake done sender: [6:7574435798924292811:2117] self: [6:7574435798924292823:0] peer: [4:7574435798788824159:0] socket: 438 qp: -1
2025-11-19T13:59:22.683233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435798924292831:2069] [node 4] ICS10 traffic start
2025-11-19T13:59:22.683237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:22.685099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435798788824084:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435798788824175:2120] held: no
2025-11-19T13:59:22.685144Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435798788824175:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:22.685287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435798788824084:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.685300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:22.685306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435798788824084:2050] [node 3] ICP22 created new session: [4:7574435798788824184:2050]
2025-11-19T13:59:22.685310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435798788824184:2050] [node 3] ICS09 handshake done sender: [4:7574435798788824175:2120] self: [4:7574435798788824178:0] peer: [3:7574435801421444440:0] socket: 397 qp: -1
2025-11-19T13:59:22.685315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435798788824184:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:22.685321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:22.695914Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435798924292815:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:22.696154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435798924292732:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:22.696176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:22.696184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435798924292732:2049] [node 2] ICP22 created new session: [6:7574435798924292833:2049]
2025-11-19T13:59:22.696191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435798924292833:2049] [node 2] ICS09 handshake done sender: [6:7574435798924292815:2119] self: [6:7574435798924292824:0] peer: [2:7574435797907898607:0] socket: 441 qp: -1
2025-11-19T13:59:22.696200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435798924292833:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:22.696214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:22.710988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:22.711086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435797907898621:2048] [node 1] ICS01 socket: 390 reason# EndOfStream
2025-11-19T13:59:22.711096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435797907898531:2048] [node 1] ICP30 unregister session Session# [2:7574435797907898621:2048] VirtualId# [2:7574435797907898604:0]
2025-11-19T13:59:22.711105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435797907898621:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:22.711153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:22.711179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:22.711844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:22.711931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435801421444453:2048] [node 1] ICS01 socket: 399 reason# EndOfStream
2025-11-19T13:59:22.711938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435801421444361:2048] [node 1] ICP30 unregister session Session# [3:7574435801421444453:2048] VirtualId# [3:7574435801421444437:0]
2025-11-19T13:59:22.711945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435801421444453:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:22.711995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:22.712019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:22.715945Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:22.716025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435798788824180:2048] [node 1] ICS01 socket: 435 reason# EndOfStream
2025-11-19T13:59:22.716031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435798788824082:2048] [node 1] ICP30 unregister session Session# [4:7574435798788824180:2048] VirtualId# [4:7574435798788824172:0]
2025-11-19T13:59:22.716035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435798788824180:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:22.716085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:22.716111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:22.719962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:22.720059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435798924292825:2048] [node 1] ICS01 socket: 432 reason# EndOfStream
2025-11-19T13:59:22.720068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435798924292731:2048] [node 1] ICP30 unregister session Session# [6:7574435798924292825:2048] VirtualId# [6:7574435798924292813:0]
2025-11-19T13:59:22.720075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435798924292825:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:22.720140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:22.720175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:22.720362Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:22.720427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435800357789596:2048] [node 1] ICS01 socket: 391 reason# EndOfStream
2025-11-19T13:59:22.720433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435800357789503:2048] [node 1] ICP30 unregister session Session# [5:7574435800357789596:2048] VirtualId# [5:7574435800357789582:0]
2025-11-19T13:59:22.720438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435800357789596:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:22.720473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:22.720496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:22.726689Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:22.726792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435798924292833:2049] [node 2] ICS01 socket: 441 reason# EndOfStream
2025-11-19T13:59:22.726804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435798924292732:2049] [node 2] ICP30 unregister session Session# [6:7574435798924292833:2049] VirtualId# [6:7574435798924292824:0]
2025-11-19T13:59:22.726813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435798924292833:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:22.726878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:22.726905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:22.730483Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:22.730571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435798788824170:2049] [node 2] ICS01 socket: 385 reason# EndOfStream
2025-11-19T13:59:22.730578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435798788824083:2049] [node 2] ICP30 unregister session Session# [4:7574435798788824170:2049] VirtualId# [4:7574435798788824161:0]
2025-11-19T13:59:22.730583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435798788824170:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:22.730629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:22.730665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:22.728635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:22.728733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435800357789584:2049] [node 2] ICS07 socket disconnect 395 reason# EndOfStream
2025-11-19T13:59:22.728743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435800357789584:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:22.728793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:22.728824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435800357789584:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:22.729704Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435800357789504:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3485) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:22.729716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435800357789584:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:22.729723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435800357789504:2049] [node 2] ICP30 unregister session Session# [5:7574435800357789584:2049] VirtualId# [5:7574435800357789527:0]
2025-11-19T13:59:22.729742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:22.729748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435800357789504:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3485) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:22.729752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3485) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:22.739707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:22.739820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435800357789599:2050] [node 3] ICS01 socket: 424 reason# EndOfStream
2025-11-19T13:59:22.739829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435800357789505:2050] [node 3] ICP30 unregister session Session# [5:7574435800357789599:2050] VirtualId# [5:7574435800357789594:0]
2025-11-19T13:59:22.739838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435800357789599:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:22.739898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:22.739927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:22.742319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:22.742394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435798924292829:2050] [node 3] ICS01 socket: 440 reason# EndOfStream
2025-11-19T13:59:22.742403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435798924292733:2050] [node 3] ICP30 unregister session Session# [6:7574435798924292829:2050] VirtualId# [6:7574435798924292817:0]
2025-11-19T13:59:22.742408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435798924292829:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:22.742451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:22.742474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:22.740323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435800357789504:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:22.751940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:22.752052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435798924292831:2069] [node 4] ICS01 socket: 438 reason# EndOfStream
2025-11-19T13:59:22.752061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435798924292757:2069] [node 4] ICP30 unregister session Session# [6:7574435798924292831:2069] VirtualId# [6:7574435798924292823:0]
2025-11-19T13:59:22.752070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435798924292831:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:22.752160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:22.752192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:22.971562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435799918152231:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:22.971577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435799918152232:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:23.008317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435805419372270:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:23.008337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435805419372271:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:23.043554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435802137638255:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:23.043570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435802137638256:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:23.080119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435804566334451:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:23.080134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435804566334452:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:23.080138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435804566334453:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:23.060194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435802069372726:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:23.060208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435802069372727:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:23.060212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435802069372728:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:23.103525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435804898566264:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:23.103537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435804898566265:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:23.103542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435804898566266:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:23.142144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435804566334480:2075] [node 4] ICP01 ready to work
2025-11-19T13:59:23.141122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435804213119588:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:23.142303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435804213119596:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:23.144946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435802069372773:2092] [node 6] ICP01 ready to work
2025-11-19T13:59:23.145334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435802069372779:2097] [node 5] ICP01 ready to work
2025-11-19T13:59:23.146141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435804566334495:2083] [node 6] ICP01 ready to work
2025-11-19T13:59:23.146219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435804213119602:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.147451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435804213119605:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51278
2025-11-19T13:59:23.151012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435804213119606:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51278
2025-11-19T13:59:23.156094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435804898566312:2087] [node 5] ICP01 ready to work
2025-11-19T13:59:23.159959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435805419372334:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.165445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435802069372779:2097] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(18977560ns) {Configure:104(18965096ns) {ProcessConfigured:135(16924634ns) {StartInitialHandshake:157(16923674ns) {DropHandshakes:469(227ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(27ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:476(561ns) {SwitchToState:178(232ns)}}}}}
2025-11-19T13:59:23.165283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435804566334529:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26936
2025-11-19T13:59:23.165306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435804566334530:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26936
2025-11-19T13:59:23.165661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435804898566265:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16853111ns) {Configure:104(16852244ns) {ProcessConfigured:135(16578706ns) {StartInitialHandshake:157(16578314ns) {DropHandshakes:469(448ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(826ns) {SwitchToState:178(492ns)}}}}}
2025-11-19T13:59:23.167253Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435799918152231:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(21960101ns) {Configure:104(21959407ns) {ProcessConfigured:135(21650220ns) {StartInitialHandshake:157(21649818ns) {DropHandshakes:469(787ns) {DropIncomingHandshake:441(217ns) DropOutgoingHandshake:456(202ns)} GenerateSessionVirtualId:988(281ns) PrepareNewSessionHandshake:476(1091ns) {SwitchToState:178(871ns)}}}}}
2025-11-19T13:59:23.167834Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435804213119606:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.168312Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435804566334530:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.173419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435802137638323:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38749
2025-11-19T13:59:23.173441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435802137638324:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38749
2025-11-19T13:59:23.173451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435802137638325:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38749
2025-11-19T13:59:23.173461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435802137638326:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38749
2025-11-19T13:59:23.173506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435802137638327:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38749
2025-11-19T13:59:23.174199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435802137638255:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16997425ns) {Configure:104(16996478ns) {ProcessConfigured:135(16588406ns) {StartInitialHandshake:157(16588018ns) {DropHandshakes:469(798ns) {DropIncomingHandshake:441(134ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(239ns) PrepareNewSessionHandshake:476(625ns) {SwitchToState:178(415ns)}}}}}
2025-11-19T13:59:23.174325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435802137638326:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.174378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435802137638256:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.175157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435804898566307:2079] [node 4] ICP01 ready to work
2025-11-19T13:59:23.175533Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435804213119605:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.175640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435804213119609:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51278
2025-11-19T13:59:23.172792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435805419372343:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60701
2025-11-19T13:59:23.172879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435805419372344:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60701
2025-11-19T13:59:23.172889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435805419372345:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60701
2025-11-19T13:59:23.173165Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435805419372343:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.173701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435805419372270:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17801189ns) {Configure:104(17800114ns) {ProcessConfigured:135(17341376ns) {StartInitialHandshake:157(17340742ns) {DropHandshakes:469(1279ns) {DropIncomingHandshake:441(521ns) DropOutgoingHandshake:456(160ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(690ns) {SwitchToState:178(487ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.175707Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435802137638325:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.175732Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435802137638327:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.175894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435802137638332:2116] [node 5] ICP01 ready to work
2025-11-19T13:59:23.175981Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435805419372345:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.181720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435802137638321:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:23.185071Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435802069372726:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16948470ns) {Configure:104(16947072ns) {ProcessConfigured:135(16571432ns) {StartInitialHandshake:157(16570547ns) {DropHandshakes:469(837ns) {DropIncomingHandshake:441(176ns) DropOutgoingHandshake:456(157ns)} GenerateSessionVirtualId:988(322ns) PrepareNewSessionHandshake:476(881ns) {SwitchToState:178(539ns)}}}}}
2025-11-19T13:59:23.190467Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435802137638323:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.190582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435802137638255:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.188531Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435805419372338:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.188550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435805419372338:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.188631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435805419372342:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:23.189112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435805419372341:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:23.191677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802069372804:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
2025-11-19T13:59:23.191773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802069372803:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
2025-11-19T13:59:23.191915Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435804566334529:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.191982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435804566334451:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.191771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435802137638322:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:23.192091Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435805419372344:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.192139Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435802069372803:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.192188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435802069372726:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.192312Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435802069372804:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.192338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435802069372727:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.196769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435804213119611:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51278
2025-11-19T13:59:23.197448Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435804213119609:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.196944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435804898566307:2079] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(21717035ns) {Configure:104(21716252ns) {ProcessConfigured:135(21263267ns) {StartInitialHandshake:157(21262765ns) {DropHandshakes:469(658ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(123ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:476(1302ns) {SwitchToState:178(115ns)}}}}}
2025-11-19T13:59:23.197096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435802137638332:2116] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21017437ns) {Configure:104(21016754ns) {ProcessConfigured:135(20718889ns) {StartInitialHandshake:157(20718777ns) {DropHandshakes:469(366ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(97ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(319ns) {SwitchToState:178(154ns)}}}}}
2025-11-19T13:59:23.199616Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435805419372336:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.199633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435805419372336:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.199368Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435804213119590:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.199389Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435804213119590:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.199950Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435804213119598:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.199962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435804213119598:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.200002Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435804213119592:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.200007Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435804213119592:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.199912Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435802137638324:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.207959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435805419372342:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19180841ns) {Configure:104(19179546ns) {ProcessConfigured:135(18817038ns) {StartInitialHandshake:157(18816129ns) {DropHandshakes:469(885ns) {DropIncomingHandshake:441(131ns) DropOutgoingHandshake:456(148ns)} GenerateSessionVirtualId:988(424ns) PrepareNewSessionHandshake:476(896ns) {SwitchToState:178(413ns)}}}}}
2025-11-19T13:59:23.208084Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435805419372338:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:23.208272Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435805419372341:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19101798ns) {Configure:104(19101638ns) {ProcessConfigured:135(18898532ns) {StartInitialHandshake:157(18883279ns) {DropHandshakes:469(103ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(28ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:476(942ns) {SwitchToState:178(225ns)}} IncomingHandshake:261(6359ns) {DropIncomingHandshake:441(187ns) PrepareNewSessionHandshake:476(205ns) {SwitchToState:178(161ns)} GenerateSessionVirtualId:988(224ns) IssueIncomingHandshakeReply:193(335ns)}}}}
2025-11-19T13:59:23.208175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435802137638342:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38749
2025-11-19T13:59:23.208679Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435802137638326:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:23.208775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435802137638256:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.208793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:23.208800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435802137638256:2049] [node 2] ICP22 created new session: [3:7574435802137638343:2049]
2025-11-19T13:59:23.208806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435802137638343:2049] [node 2] ICS09 handshake done sender: [3:7574435802137638326:2113] self: [3:7574435802137638330:0] peer: [2:7574435805419372337:0] socket: 418 qp: -1
2025-11-19T13:59:23.208815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435802137638343:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:23.208826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.215202Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435804213119590:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:23.214423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804898566342:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28454
2025-11-19T13:59:23.214550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804898566343:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28454
2025-11-19T13:59:23.214582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804898566344:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28454
2025-11-19T13:59:23.215673Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435805419372336:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:23.216094Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435804213119598:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:23.216054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435805419372334:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.216080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:23.216087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435805419372334:2109] [node 4] ICP22 created new session: [2:7574435805419372358:2109]
2025-11-19T13:59:23.216093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435805419372358:2109] [node 4] ICS09 handshake done sender: [2:7574435805419372336:2110] self: [2:7574435805419372335:0] peer: [4:7574435802069372807:0] socket: 407 qp: -1
2025-11-19T13:59:23.216100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435805419372358:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:23.216121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:23.216347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435799918152232:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.216359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:23.216364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435799918152232:2049] [node 3] ICP22 created new session: [1:7574435804213119616:2049]
2025-11-19T13:59:23.216370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435804213119616:2049] [node 3] ICS09 handshake done sender: [1:7574435804213119590:2108] self: [1:7574435804213119589:0] peer: [3:7574435802137638338:0] socket: 390 qp: -1
2025-11-19T13:59:23.216376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435804213119616:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:23.216383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:23.216250Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435804898566342:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.216300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435804898566307:2079] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.224260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435802137638345:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38749
2025-11-19T13:59:23.225504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435805419372271:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.225532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:23.225538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435805419372271:2049] [node 3] ICP22 created new session: [2:7574435805419372361:2049]
2025-11-19T13:59:23.225546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435805419372361:2049] [node 3] ICS09 handshake done sender: [2:7574435805419372338:2111] self: [2:7574435805419372337:0] peer: [3:7574435802137638330:0] socket: 409 qp: -1
2025-11-19T13:59:23.225553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435805419372361:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:23.225561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:23.225623Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435802069372795:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.225637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435802069372795:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.225203Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435804898566343:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.225270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435804898566266:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.233182Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435804213119611:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.235934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435802137638256:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(27119370ns) {HandleHandshakeStatus:329(27118488ns) {DropIncomingHandshake:441(320ns) DropOutgoingHandshake:456(2314ns) SwitchToState:178(201ns) ProcessPendingSessionEvents:495(20186147ns) {ForwardSessionEventToSession:612(20170274ns) ForwardSessionEventToSession:612(2279ns) ForwardSessionEventToSession:612(1034ns) ForwardSessionEventToSession:612(587ns) ForwardSessionEventToSession:612(639ns) ForwardSessionEventToSession:612(1093ns) ForwardSessionEventToSession:612(1444ns) ForwardSessionEventToSession:612(542ns) ForwardSessionEventToSession:612(906ns) ForwardSessionEventToSession:612(1280ns)}}}
2025-11-19T13:59:23.236064Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435802137638323:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.236285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802069372808:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
2025-11-19T13:59:23.236601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802069372809:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.236562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435804566334533:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26936
2025-11-19T13:59:23.237993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435804566334534:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26936
2025-11-19T13:59:23.238011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435804566334535:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26936
2025-11-19T13:59:23.237935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435805419372363:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60701
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.239673Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435804898566344:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.239729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435804898566265:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.240042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804898566348:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28454
2025-11-19T13:59:23.242172Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435802069372803:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:23.236418Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435804213119592:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:23.242468Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435804898566348:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.242517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435804898566264:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.242742Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435804213119604:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.242750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435804213119604:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.243168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435804213119588:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.243180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:23.243187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435804213119588:2107] [node 4] ICP22 created new session: [1:7574435804213119620:2107]
2025-11-19T13:59:23.243192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435804213119620:2107] [node 4] ICS09 handshake done sender: [1:7574435804213119592:2109] self: [1:7574435804213119591:0] peer: [4:7574435802069372806:0] socket: 391 qp: -1
2025-11-19T13:59:23.243199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435804213119620:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:23.243208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.243593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804898566350:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28454
2025-11-19T13:59:23.243806Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435804566334529:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:23.243939Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435804213119604:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:23.244010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435802137638322:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(52153127ns) {Configure:104(52152696ns) {ProcessConfigured:135(51822429ns) {StartInitialHandshake:157(51806434ns) {DropHandshakes:469(650ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(136ns)} GenerateSessionVirtualId:988(342ns) PrepareNewSessionHandshake:476(828ns) {SwitchToState:178(607ns)}} IncomingHandshake:261(6461ns) {DropIncomingHandshake:441(104ns) PrepareNewSessionHandshake:476(231ns) {SwitchToState:178(188ns)} GenerateSessionVirtualId:988(353ns) IssueIncomingHandshakeReply:193(448ns)}}}}
2025-11-19T13:59:23.244144Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435804898566348:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:23.244206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435804898566264:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.244221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:23.244226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435804898566264:2048] [node 1] ICP22 created new session: [6:7574435804898566351:2048]
2025-11-19T13:59:23.244231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435804898566351:2048] [node 1] ICS09 handshake done sender: [6:7574435804898566348:2118] self: [6:7574435804898566349:0] peer: [1:7574435804213119603:0] socket: 436 qp: -1
2025-11-19T13:59:23.244236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435804898566351:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:23.244255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:23.244130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435802137638255:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.244143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:23.244148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435802137638255:2048] [node 1] ICP22 created new session: [3:7574435802137638349:2048]
2025-11-19T13:59:23.244154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435802137638349:2048] [node 1] ICS09 handshake done sender: [3:7574435802137638323:2110] self: [3:7574435802137638338:0] peer: [1:7574435804213119589:0] socket: 392 qp: -1
2025-11-19T13:59:23.244169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435802137638349:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:23.244177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:23.244358Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435804566334534:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.244388Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435804566334535:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.244473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435804213119602:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.244485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:23.244489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435804213119602:2112] [node 6] ICP22 created new session: [1:7574435804213119623:2112]
2025-11-19T13:59:23.244493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435804213119623:2112] [node 6] ICS09 handshake done sender: [1:7574435804213119604:2113] self: [1:7574435804213119603:0] peer: [6:7574435804898566349:0] socket: 435 qp: -1
2025-11-19T13:59:23.244496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435804213119623:2112] [node 6] ICS10 traffic start
2025-11-19T13:59:23.244501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:23.244623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435804213119596:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.244649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:23.244660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435804213119596:2110] [node 5] ICP22 created new session: [1:7574435804213119625:2110]
2025-11-19T13:59:23.244666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435804213119625:2110] [node 5] ICS09 handshake done sender: [1:7574435804213119598:2111] self: [1:7574435804213119597:0] peer: [5:7574435804566334532:0] socket: 394 qp: -1
2025-11-19T13:59:23.244677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435804213119625:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:23.244694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:23.244729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435802137638335:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.244739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435802137638335:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.248232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435802069372726:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.248257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:23.248264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435802069372726:2048] [node 1] ICP22 created new session: [4:7574435802069372810:2048]
2025-11-19T13:59:23.248271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435802069372810:2048] [node 1] ICS09 handshake done sender: [4:7574435802069372803:2114] self: [4:7574435802069372806:0] peer: [1:7574435804213119591:0] socket: 393 qp: -1
2025-11-19T13:59:23.248279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435802069372810:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:23.248289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:23.248575Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435805419372334:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(32493203ns) {HandleHandshakeStatus:329(32492009ns) {DropIncomingHandshake:441(9061ns) DropOutgoingHandshake:456(176ns) SwitchToState:178(280ns) ProcessPendingSessionEvents:495(22724507ns) {ForwardSessionEventToSession:612(22713658ns) ForwardSessionEventToSession:612(1843ns) ForwardSessionEventToSession:612(661ns) ForwardSessionEventToSession:612(649ns) ForwardSessionEventToSession:612(481ns) ForwardSessionEventToSession:612(1036ns) ForwardSessionEventToSession:612(1001ns) ForwardSessionEventToSession:612(534ns) ForwardSessionEventToSession:612(562ns) ForwardSessionEventToSession:612(829ns)}}}
2025-11-19T13:59:23.248632Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435805419372354:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.248636Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435805419372354:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.251965Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435805419372354:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:23.251779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435799918152232:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(35386366ns) {HandleHandshakeStatus:329(35386011ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(232ns) SwitchToState:178(164ns) ProcessPendingSessionEvents:495(23678008ns) {ForwardSessionEventToSession:612(23665139ns) ForwardSessionEventToSession:612(2003ns) ForwardSessionEventToSession:612(935ns) ForwardSessionEventToSession:612(733ns) ForwardSessionEventToSession:612(962ns) ForwardSessionEventToSession:612(1095ns) ForwardSessionEventToSession:612(719ns) ForwardSessionEventToSession:612(649ns) ForwardSessionEventToSession:612(884ns) ForwardSessionEventToSession:612(1324ns)}}}
2025-11-19T13:59:23.252043Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435805419372353:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.252074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435805419372353:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.251952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435804566334538:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26936
2025-11-19T13:59:23.252370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435805419372341:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.252392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:23.252401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435805419372341:2107] [node 5] ICP22 created new session: [2:7574435805419372365:2107]
2025-11-19T13:59:23.252408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435805419372365:2107] [node 5] ICS09 handshake done sender: [2:7574435805419372354:2117] self: [2:7574435805419372350:0] peer: [5:7574435804566334537:0] socket: 438 qp: -1
2025-11-19T13:59:23.252414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435805419372365:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:23.252424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:23.255911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435802069372727:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7661) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435802069372804:2115] held: no
2025-11-19T13:59:23.255994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435802069372727:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(29952047ns) {HandleHandshakeStatus:395(29951220ns) {LogHandshakeFail:472(3461ns) {UpdateErrorStateLog:544(1385ns)} DropOutgoingHandshake:456(612ns) ProcessPendingSessionEvents:495(20060ns) {DropSessionEvent:512(5145ns) DropSessionEvent:512(1324ns) DropSessionEvent:512(2102ns) DropSessionEvent:512(1191ns) DropSessionEvent:512(1803ns) DropSessionEvent:512(1768ns) DropSessionEvent:512(1360ns) DropSessionEvent:512(1407ns) DropSessionEvent:512(1126ns) DropSessionEvent:512(936ns)}}}
2025-11-19T13:59:23.256231Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435802069372804:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.260346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435799918152232:2049] [node 3] ICP18 handshake (actor [1:7574435804213119611:2118]) is too old
2025-11-19T13:59:23.260784Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435802137638340:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.260796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435802137638340:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.261945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435804566334539:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26936
2025-11-19T13:59:23.262053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435804213119602:2112] [node 6] ICP35 event processing took too much time PendingConnection:278(17550884ns) {HandleHandshakeStatus:329(17550375ns) {DropIncomingHandshake:441(3257ns) DropOutgoingHandshake:456(134ns) SwitchToState:178(37ns) ProcessPendingSessionEvents:495(17501324ns) {ForwardSessionEventToSession:612(17489984ns) ForwardSessionEventToSession:612(1970ns) ForwardSessionEventToSession:612(826ns) ForwardSessionEventToSession:612(1113ns) ForwardSessionEventToSession:612(725ns) ForwardSessionEventToSession:612(375ns) ForwardSessionEventToSession:612(705ns) ForwardSessionEventToSession:612(575ns) ForwardSessionEventToSession:612(549ns) ForwardSessionEventToSession:612(1289ns)}}}
2025-11-19T13:59:23.260629Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435802069372795:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:23.262339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435802069372727:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.262353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:23.262359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435802069372727:2049] [node 2] ICP22 created new session: [4:7574435802069372813:2049]
2025-11-19T13:59:23.262365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435802069372813:2049] [node 2] ICS09 handshake done sender: [4:7574435802069372804:2115] self: [4:7574435802069372807:0] peer: [2:7574435805419372335:0] socket: 415 qp: -1
2025-11-19T13:59:23.262371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435802069372813:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:23.262390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:23.262636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435802069372773:2092] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.262659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:23.262674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435802069372773:2092] [node 6] ICP22 created new session: [4:7574435802069372815:2092]
2025-11-19T13:59:23.262679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435802069372815:2092] [node 6] ICS09 handshake done sender: [4:7574435802069372795:2110] self: [4:7574435802069372787:0] peer: [6:7574435804898566345:0] socket: 404 qp: -1
2025-11-19T13:59:23.262685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435802069372815:2092] [node 6] ICS10 traffic start
2025-11-19T13:59:23.262691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.263013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435804566334540:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26936
2025-11-19T13:59:23.264106Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435805419372271:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(55113186ns) {HandleHandshakeStatus:329(55112882ns) {DropIncomingHandshake:441(4211ns) DropOutgoingHandshake:456(152ns) SwitchToState:178(51ns) ProcessPendingSessionEvents:495(19457590ns) {ForwardSessionEventToSession:612(19442606ns) ForwardSessionEventToSession:612(1866ns) ForwardSessionEventToSession:612(1159ns) ForwardSessionEventToSession:612(774ns) ForwardSessionEventToSession:612(629ns) ForwardSessionEventToSession:612(1143ns) ForwardSessionEventToSession:612(779ns) ForwardSessionEventToSession:612(728ns) ForwardSessionEventToSession:612(1104ns) ForwardSessionEventToSession:612(836ns)}}}
2025-11-19T13:59:23.268193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804898566353:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28454
2025-11-19T13:59:23.268257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435804566334451:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.268305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:23.268318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435804566334451:2048] [node 1] ICP22 created new session: [5:7574435804566334541:2048]
2025-11-19T13:59:23.268328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435804566334541:2048] [node 1] ICS09 handshake done sender: [5:7574435804566334529:2115] self: [5:7574435804566334532:0] peer: [1:7574435804213119597:0] socket: 402 qp: -1
2025-11-19T13:59:23.268338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435804566334541:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:23.268367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:23.268394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435804566334453:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.268427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435804566334453:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(23989835ns) {IncomingHandshake:261(23989478ns) {DropIncomingHandshake:441(117ns) PrepareNewSessionHandshake:476(296ns) {SwitchToState:178(72ns)} GenerateSessionVirtualId:988(558ns) IssueIncomingHandshakeReply:193(23985682ns) {SwitchToState:178(296ns)}}}
2025-11-19T13:59:23.268464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435804566334452:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.268484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435804566334452:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(24040598ns) {IncomingHandshake:261(24040500ns) {DropIncomingHandshake:441(26ns) PrepareNewSessionHandshake:476(442ns) {SwitchToState:178(323ns)} GenerateSessionVirtualId:988(161ns) IssueIncomingHandshakeReply:193(24037637ns) {SwitchToState:178(312ns)}}}
2025-11-19T13:59:23.268524Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435804566334535:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:23.264771Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435802137638340:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:23.268817Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435802137638335:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:23.269166Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435804213119596:2110] [node 5] ICP35 event processing took too much time PendingConnection:278(52950440ns) {HandleHandshakeStatus:329(52949874ns) {DropIncomingHandshake:441(6590ns) DropOutgoingHandshake:456(204ns) SwitchToState:178(71ns) ProcessPendingSessionEvents:495(16878940ns) {ForwardSessionEventToSession:612(16866279ns) ForwardSessionEventToSession:612(2158ns) ForwardSessionEventToSession:612(919ns) ForwardSessionEventToSession:612(707ns) ForwardSessionEventToSession:612(805ns) ForwardSessionEventToSession:612(1087ns) ForwardSessionEventToSession:612(789ns) ForwardSessionEventToSession:612(607ns) ForwardSessionEventToSession:612(664ns) ForwardSessionEventToSession:612(1495ns)}}}
2025-11-19T13:59:23.268904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435802137638255:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(24744236ns) {HandleHandshakeStatus:329(24743558ns) {DropIncomingHandshake:441(211ns) DropOutgoingHandshake:456(1516ns) SwitchToState:178(68ns) ProcessPendingSessionEvents:495(16762846ns) {ForwardSessionEventToSession:612(16747596ns) ForwardSessionEventToSession:612(2559ns) ForwardSessionEventToSession:612(931ns) ForwardSessionEventToSession:612(877ns) ForwardSessionEventToSession:612(706ns) ForwardSessionEventToSession:612(1316ns) ForwardSessionEventToSession:612(880ns) ForwardSessionEventToSession:612(1298ns) ForwardSessionEventToSession:612(744ns) ForwardSessionEventToSession:612(1134ns)}}}
2025-11-19T13:59:23.269326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435802137638321:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.269345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:23.269354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435802137638321:2109] [node 6] ICP22 created new session: [3:7574435802137638353:2109]
2025-11-19T13:59:23.269361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435802137638353:2109] [node 6] ICS09 handshake done sender: [3:7574435802137638335:2117] self: [3:7574435802137638334:0] peer: [6:7574435804898566346:0] socket: 424 qp: -1
2025-11-19T13:59:23.269367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435802137638353:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:23.269375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:23.269595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435804566334452:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.269614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:23.269620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435804566334452:2049] [node 2] ICP22 created new session: [5:7574435804566334543:2049]
2025-11-19T13:59:23.269626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435804566334543:2049] [node 2] ICS09 handshake done sender: [5:7574435804566334535:2119] self: [5:7574435804566334537:0] peer: [2:7574435805419372350:0] socket: 442 qp: -1
2025-11-19T13:59:23.269633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435804566334543:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:23.269650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:23.269780Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435804566334534:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:23.269771Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435805419372363:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.269506Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435804898566264:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(25270204ns) {HandleHandshakeStatus:329(25269758ns) {DropIncomingHandshake:441(211ns) DropOutgoingHandshake:456(7030ns) SwitchToState:178(69ns) ProcessPendingSessionEvents:495(17179106ns) {ForwardSessionEventToSession:612(17168337ns) ForwardSessionEventToSession:612(1210ns) ForwardSessionEventToSession:612(886ns) ForwardSessionEventToSession:612(740ns) ForwardSessionEventToSession:612(605ns) ForwardSessionEventToSession:612(754ns) ForwardSessionEventToSession:612(639ns) ForwardSessionEventToSession:612(825ns) ForwardSessionEventToSession:612(795ns) ForwardSessionEventToSession:612(838ns)}}}
2025-11-19T13:59:23.269656Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435804898566342:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.274045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802069372818:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
2025-11-19T13:59:23.271063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435802137638332:2116] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.271081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:23.271087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435802137638332:2116] [node 5] ICP22 created new session: [3:7574435802137638355:2116]
2025-11-19T13:59:23.271092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435802137638355:2116] [node 5] ICS09 handshake done sender: [3:7574435802137638340:2118] self: [3:7574435802137638333:0] peer: [5:7574435804566334536:0] socket: 437 qp: -1
2025-11-19T13:59:23.271097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435802137638355:2116] [node 5] ICS10 traffic start
2025-11-19T13:59:23.271103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:23.276088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435805419372270:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.276071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435804898566307:2079] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.276091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:23.276099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435804898566307:2079] [node 4] ICP22 created new session: [6:7574435804898566355:2079]
2025-11-19T13:59:23.276106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435804898566355:2079] [node 4] ICS09 handshake done sender: [6:7574435804898566342:2115] self: [6:7574435804898566345:0] peer: [4:7574435802069372787:0] socket: 412 qp: -1
2025-11-19T13:59:23.276112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435804898566355:2079] [node 4] ICS10 traffic start
2025-11-19T13:59:23.276120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:23.277824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435805419372341:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(25429192ns) {HandleHandshakeStatus:329(25428453ns) {DropIncomingHandshake:441(4896ns) DropOutgoingHandshake:456(334ns) SwitchToState:178(235ns) ProcessPendingSessionEvents:495(25325300ns) {ForwardSessionEventToSession:612(25313590ns) ForwardSessionEventToSession:612(1511ns) ForwardSessionEventToSession:612(770ns) ForwardSessionEventToSession:612(1104ns) ForwardSessionEventToSession:612(528ns) ForwardSessionEventToSession:612(932ns) ForwardSessionEventToSession:612(607ns) ForwardSessionEventToSession:612(582ns) ForwardSessionEventToSession:612(569ns) ForwardSessionEventToSession:612(1309ns)}}}
2025-11-19T13:59:23.278082Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435804213119607:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.278095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435804213119607:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.279009Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435805419372353:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.279216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435805419372342:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.279229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:23.279237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435805419372342:2108] [node 6] ICP22 created new session: [2:7574435805419372370:2108]
2025-11-19T13:59:23.279243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435805419372370:2108] [node 6] ICS09 handshake done sender: [2:7574435805419372353:2116] self: [2:7574435805419372349:0] peer: [6:7574435804898566347:0] socket: 427 qp: -1
2025-11-19T13:59:23.279249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435805419372370:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:23.279268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:23.279390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435805419372369:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60701
2025-11-19T13:59:23.279576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804898566354:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28454
2025-11-19T13:59:23.280055Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435804213119607:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:23.279772Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435804566334451:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(31879818ns) {HandleHandshakeStatus:329(31879118ns) {DropIncomingHandshake:441(871ns) DropOutgoingHandshake:456(9926ns) SwitchToState:178(323ns) ProcessPendingSessionEvents:495(2165050ns) {ForwardSessionEventToSession:612(2153073ns) ForwardSessionEventToSession:612(1818ns) ForwardSessionEventToSession:612(929ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(773ns) ForwardSessionEventToSession:612(1733ns) ForwardSessionEventToSession:612(993ns) ForwardSessionEventToSession:612(506ns) ForwardSessionEventToSession:612(541ns) ForwardSessionEventToSession:612(690ns)}}}
2025-11-19T13:59:23.279813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435804566334453:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.279826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:23.279830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435804566334453:2050] [node 3] ICP22 created new session: [5:7574435804566334545:2050]
2025-11-19T13:59:23.279835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435804566334545:2050] [node 3] ICS09 handshake done sender: [5:7574435804566334534:2118] self: [5:7574435804566334536:0] peer: [3:7574435802137638333:0] socket: 441 qp: -1
2025-11-19T13:59:23.279840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435804566334545:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:23.279847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:23.280236Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435802069372818:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.280272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435802069372728:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.280454Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435802137638347:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.280268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435799918152231:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.280288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:23.280294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435799918152231:2048] [node 2] ICP22 created new session: [1:7574435804213119628:2048]
2025-11-19T13:59:23.280301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435804213119628:2048] [node 2] ICS09 handshake done sender: [1:7574435804213119607:2116] self: [1:7574435804213119600:0] peer: [2:7574435805419372367:0] socket: 439 qp: -1
2025-11-19T13:59:23.280305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435804213119628:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:23.280312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:23.280465Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435802137638347:2121] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.280622Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435805419372363:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:23.280897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435805419372270:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.280914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:23.280919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435805419372270:2048] [node 1] ICP22 created new session: [2:7574435805419372372:2048]
2025-11-19T13:59:23.280925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435805419372372:2048] [node 1] ICS09 handshake done sender: [2:7574435805419372363:2118] self: [2:7574435805419372367:0] peer: [1:7574435804213119600:0] socket: 443 qp: -1
2025-11-19T13:59:23.280929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435805419372372:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:23.280937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:23.282582Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435802137638347:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:23.283917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435802137638322:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.283941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:23.283951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435802137638322:2107] [node 4] ICP22 created new session: [3:7574435802137638359:2107]
2025-11-19T13:59:23.283959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435802137638359:2107] [node 4] ICS09 handshake done sender: [3:7574435802137638347:2121] self: [3:7574435802137638339:0] peer: [4:7574435802069372819:0] socket: 413 qp: -1
2025-11-19T13:59:23.283966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435802137638359:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:23.283974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.284322Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435804898566343:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:23.284551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804898566357:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28454
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.284796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435805419372374:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60701
2025-11-19T13:59:23.284764Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435804566334540:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.284848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435804566334480:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.287019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435804898566266:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.287042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:23.287051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435804898566266:2050] [node 3] ICP22 created new session: [6:7574435804898566360:2050]
2025-11-19T13:59:23.287059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435804898566360:2050] [node 3] ICS09 handshake done sender: [6:7574435804898566343:2116] self: [6:7574435804898566346:0] peer: [3:7574435802137638334:0] socket: 430 qp: -1
2025-11-19T13:59:23.287065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435804898566360:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:23.287087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:23.287201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435804898566266:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:23.287347Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435804898566344:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:23.289688Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435802069372728:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23959) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435802069372818:2119] held: no
2025-11-19T13:59:23.289800Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435802069372801:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.289809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435802069372801:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.291657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435804566334550:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26936
2025-11-19T13:59:23.292073Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435804566334540:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:23.291987Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435805419372270:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:23.292148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435804566334480:2075] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.292166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:23.292173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435804566334480:2075] [node 4] ICP22 created new session: [5:7574435804566334551:2075]
2025-11-19T13:59:23.292186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435804566334551:2075] [node 4] ICS09 handshake done sender: [5:7574435804566334540:2122] self: [5:7574435804566334548:0] peer: [4:7574435802069372790:0] socket: 453 qp: -1
2025-11-19T13:59:23.292193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435804566334551:2075] [node 4] ICS10 traffic start
2025-11-19T13:59:23.292202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:23.291866Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435802069372801:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:23.292771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435802069372779:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.292781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:23.292787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435802069372779:2097] [node 5] ICP22 created new session: [4:7574435802069372821:2097]
2025-11-19T13:59:23.292793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435802069372821:2097] [node 5] ICS09 handshake done sender: [4:7574435802069372801:2113] self: [4:7574435802069372790:0] peer: [5:7574435804566334548:0] socket: 452 qp: -1
2025-11-19T13:59:23.292798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435802069372821:2097] [node 5] ICS10 traffic start
2025-11-19T13:59:23.292805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.299619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804898566359:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28454
2025-11-19T13:59:23.299654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435804898566265:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.299670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:23.299677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435804898566265:2049] [node 2] ICP22 created new session: [6:7574435804898566362:2049]
2025-11-19T13:59:23.299684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435804898566362:2049] [node 2] ICS09 handshake done sender: [6:7574435804898566344:2117] self: [6:7574435804898566347:0] peer: [2:7574435805419372349:0] socket: 431 qp: -1
2025-11-19T13:59:23.299691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435804898566362:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:23.299698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:23.306819Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435804898566359:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.306881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435804898566312:2087] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.307103Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435804566334505:2091] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.307117Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435804566334505:2091] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.308078Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435804566334505:2091] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.308624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802069372823:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
2025-11-19T13:59:23.308678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802069372824:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11542
2025-11-19T13:59:23.309175Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435802069372818:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:23.309468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435802069372779:2097] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:23.309590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435802069372728:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.309605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:23.309612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435802069372728:2050] [node 3] ICP22 created new session: [4:7574435802069372825:2050]
2025-11-19T13:59:23.309619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435802069372825:2050] [node 3] ICS09 handshake done sender: [4:7574435802069372818:2119] self: [4:7574435802069372819:0] peer: [3:7574435802137638339:0] socket: 422 qp: -1
2025-11-19T13:59:23.309626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435802069372825:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:23.309639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:23.311926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804898566365:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28454
2025-11-19T13:59:23.312030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435804566334495:2083] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.312057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:23.312062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435804566334495:2083] [node 6] ICP22 created new session: [5:7574435804566334554:2083]
2025-11-19T13:59:23.312069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435804566334554:2083] [node 6] ICS09 handshake done sender: [5:7574435804566334505:2091] self: [5:7574435804566334504:0] peer: [6:7574435804898566364:0] socket: 459 qp: -1
2025-11-19T13:59:23.312075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435804566334554:2083] [node 6] ICS10 traffic start
2025-11-19T13:59:23.312095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:23.313206Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435804898566359:2123] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:23.313324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435804898566312:2087] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.313340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:23.313346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435804898566312:2087] [node 5] ICP22 created new session: [6:7574435804898566366:2087]
2025-11-19T13:59:23.313351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435804898566366:2087] [node 5] ICS09 handshake done sender: [6:7574435804898566359:2123] self: [6:7574435804898566364:0] peer: [5:7574435804566334504:0] socket: 400 qp: -1
2025-11-19T13:59:23.313357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435804898566366:2087] [node 5] ICS10 traffic start
2025-11-19T13:59:23.313369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:23.344471Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:23.344562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574435805419372372:2048] [node 1] ICS07 socket disconnect 443 reason# EndOfStream
2025-11-19T13:59:23.344568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435805419372372:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.344608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:23.344625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435805419372372:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:23.347148Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:23.347226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435802137638349:2048] [node 1] ICS07 socket disconnect 392 reason# EndOfStream
2025-11-19T13:59:23.347232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435802137638349:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.347280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:23.347296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435802137638349:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:23.350161Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435802137638255:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20168) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:23.350174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435802137638349:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:23.350704Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435802137638255:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20168) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:23.350715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435802137638349:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:23.350719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435802137638255:2048] [node 1] ICP30 unregister session Session# [3:7574435802137638349:2048] VirtualId# [3:7574435802137638338:0]
2025-11-19T13:59:23.350737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:23.350742Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435802137638255:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20168) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:23.350745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20168) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:23.355826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:23.355944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435802069372810:2048] [node 1] ICS01 socket: 393 reason# EndOfStream
2025-11-19T13:59:23.355952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435802069372726:2048] [node 1] ICP30 unregister session Session# [4:7574435802069372810:2048] VirtualId# [4:7574435802069372806:0]
2025-11-19T13:59:23.355960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435802069372810:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.356022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:23.356049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:23.356111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:23.356184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435804898566351:2048] [node 1] ICS07 socket disconnect 436 reason# EndOfStream
2025-11-19T13:59:23.356189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435804898566351:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.356241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:23.356262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435804898566351:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:23.356652Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:23.356735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435804566334541:2048] [node 1] ICS07 socket disconnect 402 reason# EndOfStream
2025-11-19T13:59:23.356739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435804566334541:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.356795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:23.356815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435804566334541:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:23.357213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435804898566264:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20168) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:23.357240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435804898566351:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:23.357249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435804898566264:2048] [node 1] ICP30 unregister session Session# [6:7574435804898566351:2048] VirtualId# [6:7574435804898566349:0]
2025-11-19T13:59:23.357289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:23.357294Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435804898566264:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20168) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:23.357297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20168) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:23.357739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435804566334451:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20168) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:23.357796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435804566334541:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:23.357800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435804566334451:2048] [node 1] ICP30 unregister session Session# [5:7574435804566334541:2048] VirtualId# [5:7574435804566334532:0]
2025-11-19T13:59:23.357833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:23.357837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435804566334451:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20168) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:23.357844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20168) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:23.361965Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:23.362050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435802137638343:2049] [node 2] ICS01 socket: 418 reason# EndOfStream
2025-11-19T13:59:23.362059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435802137638256:2049] [node 2] ICP30 unregister session Session# [3:7574435802137638343:2049] VirtualId# [3:7574435802137638330:0]
2025-11-19T13:59:23.362065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435802137638343:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.362102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:23.362110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574435802137638255:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:23.362121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:23.366534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:23.366614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435804898566362:2049] [node 2] ICS01 socket: 431 reason# EndOfStream
2025-11-19T13:59:23.366624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435804898566265:2049] [node 2] ICP30 unregister session Session# [6:7574435804898566362:2049] VirtualId# [6:7574435804898566347:0]
2025-11-19T13:59:23.366631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435804898566362:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.366688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:23.366712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:23.367786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435804898566264:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:23.366745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:23.366807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435802069372813:2049] [node 2] ICS01 socket: 415 reason# EndOfStream
2025-11-19T13:59:23.366811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435802069372727:2049] [node 2] ICP30 unregister session Session# [4:7574435802069372813:2049] VirtualId# [4:7574435802069372807:0]
2025-11-19T13:59:23.366815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435802069372813:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.366838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:23.366858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:23.367990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435804566334451:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:23.368021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:23.368087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435804566334543:2049] [node 2] ICS01 socket: 442 reason# EndOfStream
2025-11-19T13:59:23.368093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435804566334452:2049] [node 2] ICP30 unregister session Session# [5:7574435804566334543:2049] VirtualId# [5:7574435804566334537:0]
2025-11-19T13:59:23.368099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435804566334543:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.368156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:23.368180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:23.399926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:23.400043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435804566334545:2050] [node 3] ICS01 socket: 441 reason# EndOfStream
2025-11-19T13:59:23.400054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435804566334453:2050] [node 3] ICP30 unregister session Session# [5:7574435804566334545:2050] VirtualId# [5:7574435804566334536:0]
2025-11-19T13:59:23.400062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435804566334545:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.400146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:23.400179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:23.400018Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:23.400109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435804898566360:2050] [node 3] ICS01 socket: 430 reason# EndOfStream
2025-11-19T13:59:23.400119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435804898566266:2050] [node 3] ICP30 unregister session Session# [6:7574435804898566360:2050] VirtualId# [6:7574435804898566346:0]
2025-11-19T13:59:23.400126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435804898566360:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.400191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:23.400217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:23.413111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T13:59:23.413225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435804566334551:2075] [node 4] ICS01 socket: 453 reason# EndOfStream
2025-11-19T13:59:23.413237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435804566334480:2075] [node 4] ICP30 unregister session Session# [5:7574435804566334551:2075] VirtualId# [5:7574435804566334548:0]
2025-11-19T13:59:23.413244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435804566334551:2075] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.413307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:23.413332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:23.415042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:23.415125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435804898566355:2079] [node 4] ICS01 socket: 412 reason# EndOfStream
2025-11-19T13:59:23.415133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435804898566307:2079] [node 4] ICP30 unregister session Session# [6:7574435804898566355:2079] VirtualId# [6:7574435804898566345:0]
2025-11-19T13:59:23.415140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435804898566355:2079] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.415190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:23.415214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:23.525991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435801992300824:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:23.526006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435801992300825:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:23.556339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435804314941510:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:23.556359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435804314941511:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:23.572300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435803641302981:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:23.572315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435803641302982:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:23.600208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435802142665649:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:23.600221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435802142665650:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:23.600225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435802142665651:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:23.640193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435802292662167:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:23.640209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435802292662168:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:23.640214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435802292662169:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:23.668049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435804515464145:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:23.668066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435804515464146:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:23.668070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435804515464147:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:23.720309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435804515464186:2087] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.729424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435802292662231:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3189
2025-11-19T13:59:23.732798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435804515464203:2091] [node 4] ICP01 ready to work
2025-11-19T13:59:23.736709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435801992300888:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:23.736794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435801992300889:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:23.736858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435801992300890:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:23.757581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435803641303045:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:23.760502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435804314941574:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:23.760587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435804314941575:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:23.768916Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435801992300825:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(20821151ns) {Configure:104(20818831ns) {ProcessConfigured:135(20485849ns) {StartInitialHandshake:157(20485641ns) {DropHandshakes:469(898ns) {DropIncomingHandshake:441(154ns) DropOutgoingHandshake:456(149ns)} GenerateSessionVirtualId:988(342ns) PrepareNewSessionHandshake:476(1310ns) {SwitchToState:178(1055ns)}}}}}
2025-11-19T13:59:23.768020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435804515464203:2091] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19210403ns) {Configure:104(19207713ns) {ProcessConfigured:135(18903637ns) {StartInitialHandshake:157(18902800ns) {DropHandshakes:469(652ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:476(1523ns) {SwitchToState:178(1085ns)}}}}}
2025-11-19T13:59:23.773083Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435802292662231:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.773293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435802292662233:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:23.773652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435802292662233:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.777432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435803641303052:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:23.777538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435803641303053:2113] [node 4] ICP01 ready to work
2025-11-19T13:59:23.778486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435803641303049:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-19T13:59:23.778498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435803641303050:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-19T13:59:23.781365Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435804515464188:2088] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.781384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435804515464188:2088] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.791075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435802292662235:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3189
2025-11-19T13:59:23.795507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435804314941586:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.795508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435802142665714:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:23.795592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435802142665715:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:23.796717Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435801992300888:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(27757335ns) {Configure:104(27756467ns) {ProcessConfigured:135(27462598ns) {StartInitialHandshake:157(27462241ns) {DropHandshakes:469(633ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(132ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:476(732ns) {SwitchToState:178(138ns)}}}}}
2025-11-19T13:59:23.797271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435801992300887:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14651
2025-11-19T13:59:23.797514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435801992300901:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14651
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.799230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435803641303062:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-19T13:59:23.798326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435804314941587:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9782
2025-11-19T13:59:23.801676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435801992300902:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14651
2025-11-19T13:59:23.801923Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435801992300901:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.802497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435804314941590:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9782
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.802619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802142665712:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18788
2025-11-19T13:59:23.802692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802142665713:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18788
2025-11-19T13:59:23.803424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435802292662240:2113] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.804462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804515464224:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35826
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.804737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435802292662241:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3189
2025-11-19T13:59:23.805023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435802292662244:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3189
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.808050Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435803641303050:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.808134Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435804515464188:2088] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:23.808228Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435803641303049:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.808132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435804314941591:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9782
2025-11-19T13:59:23.808315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435804515464186:2087] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.808330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:23.808336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435804515464186:2087] [node 5] ICP22 created new session: [6:7574435804515464225:2087]
2025-11-19T13:59:23.808342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435804515464225:2087] [node 5] ICS09 handshake done sender: [6:7574435804515464188:2088] self: [6:7574435804515464187:0] peer: [5:7574435802292662234:0] socket: 397 qp: -1
2025-11-19T13:59:23.808349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435804515464225:2087] [node 5] ICS10 traffic start
2025-11-19T13:59:23.808357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:23.808374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435803641302982:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.808402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435803641303063:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-19T13:59:23.806437Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435802142665713:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.807677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802142665729:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18788
2025-11-19T13:59:23.809073Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435801992300902:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.811835Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435804314941587:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.811886Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435804314941577:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.811889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435804314941577:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.812497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435804314941592:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9782
2025-11-19T13:59:23.812585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435803641303066:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-19T13:59:23.812628Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435804314941577:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:23.812738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435804314941511:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.812757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:23.812764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435804314941511:2049] [node 3] ICP22 created new session: [2:7574435804314941595:2049]
2025-11-19T13:59:23.812770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435804314941595:2049] [node 3] ICS09 handshake done sender: [2:7574435804314941577:2110] self: [2:7574435804314941576:0] peer: [3:7574435803641303065:0] socket: 402 qp: -1
2025-11-19T13:59:23.812778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435804314941595:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:23.812787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:23.814382Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435803641302982:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13862) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435803641303049:2111] held: no
2025-11-19T13:59:23.814415Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435803641303049:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:23.814520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435803641302982:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.814533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:23.814538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435803641302982:2049] [node 2] ICP22 created new session: [3:7574435803641303067:2049]
2025-11-19T13:59:23.814543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435803641303067:2049] [node 2] ICS09 handshake done sender: [3:7574435803641303049:2111] self: [3:7574435803641303065:0] peer: [2:7574435804314941576:0] socket: 404 qp: -1
2025-11-19T13:59:23.814549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435803641303067:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:23.814556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.816429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435801992300887:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.817619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435801992300907:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14651
2025-11-19T13:59:23.816539Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435802292662235:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.816568Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435802292662231:2109] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:23.835099Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435802142665712:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.835141Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435802142665729:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.833270Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435803641303062:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.835432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435802292662233:2110] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.835453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:23.835459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435802292662233:2110] [node 6] ICP22 created new session: [5:7574435802292662248:2110]
2025-11-19T13:59:23.835466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435802292662248:2110] [node 6] ICS09 handshake done sender: [5:7574435802292662231:2109] self: [5:7574435802292662234:0] peer: [6:7574435804515464187:0] socket: 398 qp: -1
2025-11-19T13:59:23.835476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435802292662248:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:23.835487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:23.835650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435802292662169:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.836262Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435803641303047:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.836272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435803641303047:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.835239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435802142665650:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.839581Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435804314941579:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.839595Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435804314941579:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.840861Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435803641303047:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:23.841026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435803641303045:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.841048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:23.841054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435803641303045:2108] [node 5] ICP22 created new session: [3:7574435803641303072:2108]
2025-11-19T13:59:23.841060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435803641303072:2108] [node 5] ICS09 handshake done sender: [3:7574435803641303047:2109] self: [3:7574435803641303046:0] peer: [5:7574435802292662250:0] socket: 409 qp: -1
2025-11-19T13:59:23.841066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435803641303072:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:23.841075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:23.841604Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435804314941579:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:23.843442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435804314941574:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.843459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:23.843466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435804314941574:2107] [node 4] ICP22 created new session: [2:7574435804314941599:2107]
2025-11-19T13:59:23.843472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435804314941599:2107] [node 4] ICS09 handshake done sender: [2:7574435804314941579:2111] self: [2:7574435804314941578:0] peer: [4:7574435802142665731:0] socket: 403 qp: -1
2025-11-19T13:59:23.843478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435804314941599:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:23.843487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.844012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802142665733:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18788
2025-11-19T13:59:23.844375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802142665734:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18788
2025-11-19T13:59:23.844427Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435802142665712:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:23.844492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435802142665650:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.844504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:23.844511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435802142665650:2049] [node 2] ICP22 created new session: [4:7574435802142665735:2049]
2025-11-19T13:59:23.844516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435802142665735:2049] [node 2] ICS09 handshake done sender: [4:7574435802142665712:2110] self: [4:7574435802142665731:0] peer: [2:7574435804314941578:0] socket: 405 qp: -1
2025-11-19T13:59:23.844522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435802142665735:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:23.844529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:23.849274Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435803641303063:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.849300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435803641302981:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.850547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435803641303076:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1632
2025-11-19T13:59:23.850828Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435803641303063:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:23.848902Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435801992300907:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.849866Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435801992300894:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.849871Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435801992300894:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.850646Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435801992300894:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:23.850970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435803641302981:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.850986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:23.850991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435803641302981:2048] [node 1] ICP22 created new session: [3:7574435803641303077:2048]
2025-11-19T13:59:23.850997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435803641303077:2048] [node 1] ICS09 handshake done sender: [3:7574435803641303063:2118] self: [3:7574435803641303075:0] peer: [1:7574435801992300893:0] socket: 432 qp: -1
2025-11-19T13:59:23.851004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435803641303077:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:23.851011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:23.860758Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435804314941591:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.860881Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435804314941590:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.860973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435804314941574:2107] [node 4] ICP18 handshake (actor [2:7574435804314941591:2117]) is too old
2025-11-19T13:59:23.860990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435804314941510:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.861207Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435802142665734:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.861374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435802142665651:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.861539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435801992300825:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.861573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:23.861579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435801992300825:2049] [node 3] ICP22 created new session: [1:7574435801992300912:2049]
2025-11-19T13:59:23.861585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435801992300912:2049] [node 3] ICS09 handshake done sender: [1:7574435801992300894:2112] self: [1:7574435801992300893:0] peer: [3:7574435803641303075:0] socket: 419 qp: -1
2025-11-19T13:59:23.861591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435801992300912:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:23.861598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:23.862031Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435803641303060:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.862042Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435803641303060:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.863415Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435803641303060:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.865878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802142665739:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18788
2025-11-19T13:59:23.866402Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435801992300892:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.866420Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435801992300892:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.867910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435803641303053:2113] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.867931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:23.867939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435803641303053:2113] [node 4] ICP22 created new session: [3:7574435803641303080:2113]
2025-11-19T13:59:23.867946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435803641303080:2113] [node 4] ICS09 handshake done sender: [3:7574435803641303060:2116] self: [3:7574435803641303059:0] peer: [4:7574435802142665738:0] socket: 417 qp: -1
2025-11-19T13:59:23.867952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435803641303080:2113] [node 4] ICS10 traffic start
2025-11-19T13:59:23.867961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.868406Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435801992300892:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:23.865992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435802292662169:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24582) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435802292662235:2111] held: no
2025-11-19T13:59:23.866170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435802292662251:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3189
2025-11-19T13:59:23.868314Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435802142665734:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.872692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435804314941603:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9782
2025-11-19T13:59:23.872221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435802142665651:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.872238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:23.872243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435802142665651:2050] [node 3] ICP22 created new session: [4:7574435802142665741:2050]
2025-11-19T13:59:23.872249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435802142665741:2050] [node 3] ICS09 handshake done sender: [4:7574435802142665734:2119] self: [4:7574435802142665738:0] peer: [3:7574435803641303059:0] socket: 443 qp: -1
2025-11-19T13:59:23.872254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435802142665741:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:23.872262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:23.872417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802142665740:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18788
2025-11-19T13:59:23.871915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435802292662252:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3189
2025-11-19T13:59:23.872022Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435802292662241:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.872044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435802292662253:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3189
2025-11-19T13:59:23.872135Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435802292662235:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.876513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804515464228:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35826
2025-11-19T13:59:23.876557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804515464229:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35826
2025-11-19T13:59:23.876647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435801992300824:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.876673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:23.876679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435801992300824:2048] [node 2] ICP22 created new session: [1:7574435801992300916:2048]
2025-11-19T13:59:23.876686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435801992300916:2048] [node 2] ICS09 handshake done sender: [1:7574435801992300892:2111] self: [1:7574435801992300891:0] peer: [2:7574435804314941602:0] socket: 418 qp: -1
2025-11-19T13:59:23.876692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435801992300916:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:23.876699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:23.877234Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435802142665740:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.877265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435802142665649:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.877389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435802292662169:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.877405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:23.877411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435802292662169:2050] [node 3] ICP22 created new session: [5:7574435802292662255:2050]
2025-11-19T13:59:23.877417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435802292662255:2050] [node 3] ICS09 handshake done sender: [5:7574435802292662235:2111] self: [5:7574435802292662250:0] peer: [3:7574435803641303046:0] socket: 411 qp: -1
2025-11-19T13:59:23.877422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435802292662255:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:23.877429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:23.877562Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435802292662253:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.877610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435802292662167:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.877654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435802292662240:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.883431Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435804314941592:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.883513Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435804314941590:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:23.883670Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435802142665723:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.883681Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435802142665723:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.884464Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435802142665721:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.884473Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435802142665721:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.879835Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435804515464229:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.879897Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435804515464224:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.879952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435804515464147:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.879973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435804515464203:2091] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.883714Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435804515464228:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.883742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435804515464146:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.885366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804515464233:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35826
2025-11-19T13:59:23.886358Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435803641303058:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.886375Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435803641303058:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.885542Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435801992300898:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.885554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435801992300898:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.886074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435801992300896:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.886077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435801992300896:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.886919Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435802292662252:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.887935Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435802142665723:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:23.888021Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435801992300898:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:23.888138Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435801992300896:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:23.888255Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435802142665721:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:23.887601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804515464234:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35826
2025-11-19T13:59:23.887620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804515464235:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35826
2025-11-19T13:59:23.888439Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435804515464224:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:23.888490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435804515464203:2091] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.888513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:23.888519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435804515464203:2091] [node 4] ICP22 created new session: [6:7574435804515464236:2091]
2025-11-19T13:59:23.888525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435804515464236:2091] [node 4] ICS09 handshake done sender: [6:7574435804515464224:2115] self: [6:7574435804515464231:0] peer: [4:7574435802142665720:0] socket: 427 qp: -1
2025-11-19T13:59:23.888531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435804515464236:2091] [node 4] ICS10 traffic start
2025-11-19T13:59:23.888539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:23.889120Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435803641303058:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.889544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435802292662168:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.889577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435802292662259:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3189
2025-11-19T13:59:23.889689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435801992300889:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.889706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:23.889711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435801992300889:2108] [node 4] ICP22 created new session: [1:7574435801992300921:2108]
2025-11-19T13:59:23.889716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435801992300921:2108] [node 4] ICS09 handshake done sender: [1:7574435801992300898:2114] self: [1:7574435801992300897:0] peer: [4:7574435802142665743:0] socket: 415 qp: -1
2025-11-19T13:59:23.889721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435801992300921:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:23.889729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:23.889815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435801992300888:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.889822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:23.889826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435801992300888:2107] [node 5] ICP22 created new session: [1:7574435801992300923:2107]
2025-11-19T13:59:23.889830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435801992300923:2107] [node 5] ICS09 handshake done sender: [1:7574435801992300896:2113] self: [1:7574435801992300895:0] peer: [5:7574435802292662257:0] socket: 420 qp: -1
2025-11-19T13:59:23.889833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435801992300923:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:23.889837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.891355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435802292662260:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3189
2025-11-19T13:59:23.891485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435802142665715:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.891503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:23.891508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435802142665715:2109] [node 5] ICP22 created new session: [4:7574435802142665747:2109]
2025-11-19T13:59:23.891513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435802142665747:2109] [node 5] ICS09 handshake done sender: [4:7574435802142665723:2115] self: [4:7574435802142665722:0] peer: [5:7574435802292662258:0] socket: 446 qp: -1
2025-11-19T13:59:23.891518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435802142665747:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:23.891526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:23.891609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435802142665714:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.891616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:23.891619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435802142665714:2108] [node 6] ICP22 created new session: [4:7574435802142665749:2108]
2025-11-19T13:59:23.891623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435802142665749:2108] [node 6] ICS09 handshake done sender: [4:7574435802142665721:2114] self: [4:7574435802142665720:0] peer: [6:7574435804515464231:0] socket: 424 qp: -1
2025-11-19T13:59:23.891626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435802142665749:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:23.891630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:23.892035Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435802292662253:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:23.892109Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435802292662241:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:23.892165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435802292662240:2113] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.892176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:23.892181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435802292662240:2113] [node 4] ICP22 created new session: [5:7574435802292662262:2113]
2025-11-19T13:59:23.892185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435802292662262:2113] [node 4] ICS09 handshake done sender: [5:7574435802292662253:2120] self: [5:7574435802292662258:0] peer: [4:7574435802142665722:0] socket: 431 qp: -1
2025-11-19T13:59:23.892190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435802292662262:2113] [node 4] ICS10 traffic start
2025-11-19T13:59:23.892203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:23.892316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435802292662167:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.892322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:23.892326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435802292662167:2048] [node 1] ICP22 created new session: [5:7574435802292662264:2048]
2025-11-19T13:59:23.892330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435802292662264:2048] [node 1] ICS09 handshake done sender: [5:7574435802292662241:2114] self: [5:7574435802292662257:0] peer: [1:7574435801992300895:0] socket: 425 qp: -1
2025-11-19T13:59:23.892333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435802292662264:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:23.892338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:23.892272Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435804314941581:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.892285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435804314941581:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.893469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435803641303052:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.893493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:23.893500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435803641303052:2110] [node 6] ICP22 created new session: [3:7574435803641303083:2110]
2025-11-19T13:59:23.893507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435803641303083:2110] [node 6] ICS09 handshake done sender: [3:7574435803641303058:2115] self: [3:7574435803641303057:0] peer: [6:7574435804515464230:0] socket: 410 qp: -1
2025-11-19T13:59:23.893514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435803641303083:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:23.893523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.896176Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435804515464147:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24582) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435804515464229:2117] held: no
2025-11-19T13:59:23.896230Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435804515464229:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:23.896396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435802142665746:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18788
2025-11-19T13:59:23.896547Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435802142665740:2121] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:23.896598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435802142665649:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.896611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:23.896617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435802142665649:2048] [node 1] ICP22 created new session: [4:7574435802142665751:2048]
2025-11-19T13:59:23.896623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435802142665751:2048] [node 1] ICS09 handshake done sender: [4:7574435802142665740:2121] self: [4:7574435802142665743:0] peer: [1:7574435801992300897:0] socket: 449 qp: -1
2025-11-19T13:59:23.896628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435802142665751:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:23.896634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.897272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435804314941510:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.897289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:23.897294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435804314941510:2048] [node 1] ICP22 created new session: [2:7574435804314941605:2048]
2025-11-19T13:59:23.897299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435804314941605:2048] [node 1] ICS09 handshake done sender: [2:7574435804314941590:2116] self: [2:7574435804314941602:0] peer: [1:7574435801992300891:0] socket: 421 qp: -1
2025-11-19T13:59:23.897305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435804314941605:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:23.897312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:23.897363Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435804314941510:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:23.897446Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435804314941589:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.897450Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435804314941589:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.897976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435804314941604:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9782
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.899867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435802292662266:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3189
2025-11-19T13:59:23.899975Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435804314941581:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:23.900123Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435804314941589:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:23.900275Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435802292662252:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.900145Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435804515464234:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.900161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435804515464147:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.900169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:23.900175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435804515464147:2050] [node 3] ICP22 created new session: [6:7574435804515464240:2050]
2025-11-19T13:59:23.900181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435804515464240:2050] [node 3] ICS09 handshake done sender: [6:7574435804515464229:2117] self: [6:7574435804515464230:0] peer: [3:7574435803641303057:0] socket: 450 qp: -1
2025-11-19T13:59:23.900186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435804515464240:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:23.900193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:23.900423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804515464239:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35826
2025-11-19T13:59:23.900335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435802292662168:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.900348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:23.900352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435802292662168:2049] [node 2] ICP22 created new session: [5:7574435802292662267:2049]
2025-11-19T13:59:23.900357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435802292662267:2049] [node 2] ICS09 handshake done sender: [5:7574435802292662252:2119] self: [5:7574435802292662261:0] peer: [2:7574435804314941588:0] socket: 445 qp: -1
2025-11-19T13:59:23.900361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435802292662267:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:23.900368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.901364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435801992300925:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14651
2025-11-19T13:59:23.901509Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435804314941604:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.901520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435804314941575:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.901532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:23.901537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435804314941575:2108] [node 6] ICP22 created new session: [2:7574435804314941610:2108]
2025-11-19T13:59:23.901542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435804314941610:2108] [node 6] ICS09 handshake done sender: [2:7574435804314941581:2112] self: [2:7574435804314941580:0] peer: [6:7574435804515464232:0] socket: 414 qp: -1
2025-11-19T13:59:23.901546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435804314941610:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:23.901552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:23.901621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435804314941586:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.901626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:23.901629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435804314941586:2113] [node 5] ICP22 created new session: [2:7574435804314941612:2113]
2025-11-19T13:59:23.901632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435804314941612:2113] [node 5] ICS09 handshake done sender: [2:7574435804314941589:2115] self: [2:7574435804314941588:0] peer: [5:7574435802292662261:0] socket: 441 qp: -1
2025-11-19T13:59:23.901636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435804314941612:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:23.901640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:23.901952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435804314941586:2113] [node 5] ICP18 handshake (actor [2:7574435804314941604:2120]) is too old
2025-11-19T13:59:23.901220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435804515464145:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:23.901304Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435804515464228:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:23.906636Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435801992300900:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.906649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435801992300900:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:23.909572Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435801992300900:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:23.909692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435801992300890:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:23.909705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:23.909711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435801992300890:2109] [node 6] ICP22 created new session: [1:7574435801992300927:2109]
2025-11-19T13:59:23.909717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435801992300927:2109] [node 6] ICS09 handshake done sender: [1:7574435801992300900:2115] self: [1:7574435801992300899:0] peer: [6:7574435804515464242:0] socket: 416 qp: -1
2025-11-19T13:59:23.909722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435801992300927:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:23.909731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:23.916003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435804515464146:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.916034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:23.916043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435804515464146:2049] [node 2] ICP22 created new session: [6:7574435804515464243:2049]
2025-11-19T13:59:23.916050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435804515464243:2049] [node 2] ICS09 handshake done sender: [6:7574435804515464228:2116] self: [6:7574435804515464232:0] peer: [2:7574435804314941580:0] socket: 400 qp: -1
2025-11-19T13:59:23.916058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435804515464243:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:23.916067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:23.916149Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435804515464146:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:23.916397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435804515464245:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35826
2025-11-19T13:59:23.916567Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435801992300925:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:23.916599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435801992300890:2109] [node 6] ICP18 handshake (actor [1:7574435801992300925:2119]) is too old
2025-11-19T13:59:23.918146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435804515464145:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15161) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435804515464234:2119] held: no
2025-11-19T13:59:23.918177Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435804515464234:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:23.918269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435804515464145:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:23.918282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:23.918288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435804515464145:2048] [node 1] ICP22 created new session: [6:7574435804515464246:2048]
2025-11-19T13:59:23.918293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435804515464246:2048] [node 1] ICS09 handshake done sender: [6:7574435804515464234:2119] self: [6:7574435804515464242:0] peer: [1:7574435801992300899:0] socket: 459 qp: -1
2025-11-19T13:59:23.918299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435804515464246:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:23.918306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:23.948285Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:23.948375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435804314941605:2048] [node 1] ICS01 socket: 421 reason# EndOfStream
2025-11-19T13:59:23.948384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435804314941510:2048] [node 1] ICP30 unregister session Session# [2:7574435804314941605:2048] VirtualId# [2:7574435804314941602:0]
2025-11-19T13:59:23.948390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435804314941605:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.948423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:23.948443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:23.957136Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:23.957235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435803641303077:2048] [node 1] ICS07 socket disconnect 432 reason# EndOfStream
2025-11-19T13:59:23.957241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435803641303077:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.957276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:23.957293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435803641303077:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:23.958829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435803641302981:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15161) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:23.958846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435803641303077:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:23.958851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435803641302981:2048] [node 1] ICP30 unregister session Session# [3:7574435803641303077:2048] VirtualId# [3:7574435803641303075:0]
2025-11-19T13:59:23.958866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:23.958871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435803641302981:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15161) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:23.958875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15161) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:23.971853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574435803641302981:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:23.977463Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:23.977556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435802292662264:2048] [node 1] ICS01 socket: 425 reason# EndOfStream
2025-11-19T13:59:23.977564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435802292662167:2048] [node 1] ICP30 unregister session Session# [5:7574435802292662264:2048] VirtualId# [5:7574435802292662257:0]
2025-11-19T13:59:23.977572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435802292662264:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.977626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:23.977648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:23.984107Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:23.984169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435804515464246:2048] [node 1] ICS01 socket: 459 reason# EndOfStream
2025-11-19T13:59:23.984174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435804515464145:2048] [node 1] ICP30 unregister session Session# [6:7574435804515464246:2048] VirtualId# [6:7574435804515464242:0]
2025-11-19T13:59:23.984179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435804515464246:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.983848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:23.983944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435802142665751:2048] [node 1] ICS01 socket: 449 reason# EndOfStream
2025-11-19T13:59:23.983952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435802142665649:2048] [node 1] ICP30 unregister session Session# [4:7574435802142665751:2048] VirtualId# [4:7574435802142665743:0]
2025-11-19T13:59:23.983958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435802142665751:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.984012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:23.984035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:23.984960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:23.985025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435803641303067:2049] [node 2] ICS07 socket disconnect 404 reason# EndOfStream
2025-11-19T13:59:23.985029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435803641303067:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.985057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:23.985072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435803641303067:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:23.984204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:23.984230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:23.990041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:23.990116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435802292662267:2049] [node 2] ICS01 socket: 445 reason# EndOfStream
2025-11-19T13:59:23.990124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435802292662168:2049] [node 2] ICP30 unregister session Session# [5:7574435802292662267:2049] VirtualId# [5:7574435802292662261:0]
2025-11-19T13:59:23.990130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435802292662267:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.990171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:23.990195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:23.990303Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:23.990362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435804515464243:2049] [node 2] ICS01 socket: 400 reason# EndOfStream
2025-11-19T13:59:23.990367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435804515464146:2049] [node 2] ICP30 unregister session Session# [6:7574435804515464243:2049] VirtualId# [6:7574435804515464232:0]
2025-11-19T13:59:23.990371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435804515464243:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.990396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:23.990415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:23.990489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:23.990546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435802142665735:2049] [node 2] ICS01 socket: 405 reason# EndOfStream
2025-11-19T13:59:23.990550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435802142665650:2049] [node 2] ICP30 unregister session Session# [4:7574435802142665735:2049] VirtualId# [4:7574435802142665731:0]
2025-11-19T13:59:23.990554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435802142665735:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:23.990577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:23.990595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:24.001098Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:24.001184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435802292662255:2050] [node 3] ICS01 socket: 411 reason# EndOfStream
2025-11-19T13:59:24.001194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435802292662169:2050] [node 3] ICP30 unregister session Session# [5:7574435802292662255:2050] VirtualId# [5:7574435802292662250:0]
2025-11-19T13:59:24.001202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435802292662255:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.001247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:24.001271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:24.001753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T13:59:24.001859Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:24.001920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435804515464240:2050] [node 3] ICS01 socket: 450 reason# EndOfStream
2025-11-19T13:59:24.001925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435804515464147:2050] [node 3] ICP30 unregister session Session# [6:7574435804515464240:2050] VirtualId# [6:7574435804515464230:0]
2025-11-19T13:59:24.001930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435804515464240:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.002303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:24.002326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:24.029429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:24.029524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435804515464236:2091] [node 4] ICS01 socket: 427 reason# EndOfStream
2025-11-19T13:59:24.029534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435804515464203:2091] [node 4] ICP30 unregister session Session# [6:7574435804515464236:2091] VirtualId# [6:7574435804515464231:0]
2025-11-19T13:59:24.029542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435804515464236:2091] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.029600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:24.029624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:24.130839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435806516270196:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:24.130864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435806516270197:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:24.179950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435806460196662:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:24.179967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435806460196663:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:24.220372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435806768627999:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:24.220387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435806768628000:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:24.256961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435808551305048:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:24.256976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435808551305049:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:24.256981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435808551305050:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:24.284881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435806938719765:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:24.284896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435806938719766:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:24.284913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435806938719767:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:24.307679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435809839581748:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:24.307694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435809839581749:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:24.307698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435809839581750:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:24.355023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435809839581762:2059] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.356871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435806460196722:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47459
2025-11-19T13:59:24.358786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435806938719804:2080] [node 6] ICP01 ready to work
2025-11-19T13:59:24.358882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435806938719805:2082] [node 4] ICP01 ready to work
2025-11-19T13:59:24.360191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435809839581791:2063] [node 5] ICP01 ready to work
2025-11-19T13:59:24.363339Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435808551305110:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.363351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435808551305110:2108] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.364138Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435808551305110:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:24.364251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435808551305049:2049] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.364266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:24.364274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435808551305049:2049] [node 2] ICP22 created new session: [4:7574435808551305113:2049]
2025-11-19T13:59:24.364280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435808551305113:2049] [node 2] ICS09 handshake done sender: [4:7574435808551305110:2108] self: [4:7574435808551305109:0] peer: [2:7574435806460196725:0] socket: 402 qp: -1
2025-11-19T13:59:24.364286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435808551305113:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:24.364294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:24.362683Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435806460196722:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.362855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435806460196724:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:24.363237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435806460196724:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:24.364512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435806460196726:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47459
2025-11-19T13:59:24.363925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435806516270259:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:24.364020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435806516270260:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:24.367856Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435806460196722:2107] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:24.368214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435806460196724:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.368225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:24.368234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435806460196724:2108] [node 4] ICP22 created new session: [2:7574435806460196727:2108]
2025-11-19T13:59:24.368239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435806460196727:2108] [node 4] ICS09 handshake done sender: [2:7574435806460196722:2107] self: [2:7574435806460196725:0] peer: [4:7574435808551305109:0] socket: 403 qp: -1
2025-11-19T13:59:24.368248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435806460196727:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:24.368269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.372111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435809839581762:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(17000613ns) {Configure:104(16999284ns) {ProcessConfigured:135(11920083ns) {StartInitialHandshake:157(11919153ns) {DropHandshakes:469(437ns) {DropIncomingHandshake:441(162ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(1787ns) {SwitchToState:178(1077ns)}}}}}
2025-11-19T13:59:24.373058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435809839581800:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12095
2025-11-19T13:59:24.376109Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435809839581800:2095] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.376090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435806768628061:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:24.376685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435806768628062:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50252
2025-11-19T13:59:24.376698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435806768628063:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50252
2025-11-19T13:59:24.376556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435806516270266:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.376155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435808551305115:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17969
2025-11-19T13:59:24.376176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435808551305116:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17969
2025-11-19T13:59:24.377441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435809839581791:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:24.379086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435809839581824:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12095
2025-11-19T13:59:24.379058Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435806768628063:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.385244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435806516270197:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(25248911ns) {Configure:104(25248177ns) {ProcessConfigured:135(24910113ns) {StartInitialHandshake:157(24909287ns) {DropHandshakes:469(469ns) {DropIncomingHandshake:441(87ns) DropOutgoingHandshake:456(77ns)} GenerateSessionVirtualId:988(248ns) PrepareNewSessionHandshake:476(1538ns) {SwitchToState:178(807ns)}}}}}
2025-11-19T13:59:24.385707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435806516270267:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19993
2025-11-19T13:59:24.386401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435806460196732:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:24.387893Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435808551305115:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.388093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435808551305118:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:24.388468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435808551305118:2111] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.393562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435806768628067:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:24.393767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435806768628069:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:24.395035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435806768628070:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50252
2025-11-19T13:59:24.395845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435806460196662:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16210586ns) {Configure:104(16208731ns) {ProcessConfigured:135(15883536ns) {StartInitialHandshake:157(15882948ns) {DropHandshakes:469(632ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(282ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:476(1260ns) {SwitchToState:178(393ns)}}}}}
2025-11-19T13:59:24.396000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435806460196734:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.398327Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435806768628062:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.398447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435806768628082:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50252
2025-11-19T13:59:24.398934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435806516270196:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(22267597ns) {Configure:104(22267180ns) {ProcessConfigured:135(21947699ns) {StartInitialHandshake:157(21947287ns) {DropHandshakes:469(379ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(96ns) PrepareNewSessionHandshake:476(1404ns) {SwitchToState:178(327ns)}}}}}
2025-11-19T13:59:24.410725Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435806938719810:2088] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.410740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435806938719810:2088] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.412471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435808551305125:2112] [node 6] ICP01 ready to work
2025-11-19T13:59:24.412518Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435808551305116:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.411971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435806938719837:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38430
2025-11-19T13:59:24.411991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435806938719838:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38430
2025-11-19T13:59:24.413264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435808551305122:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17969
2025-11-19T13:59:24.413290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435808551305123:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17969
2025-11-19T13:59:24.413331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435808551305124:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17969
2025-11-19T13:59:24.414635Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435806938719807:2086] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.414650Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435806938719807:2086] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:24.418370Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435806938719810:2088] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:24.419932Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435809839581824:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.419979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435809839581750:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:24.420180Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435806768628065:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.420193Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435806768628065:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.419860Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435808551305123:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.420187Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435808551305048:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25056486ns) {Configure:104(25055064ns) {ProcessConfigured:135(24704715ns) {StartInitialHandshake:157(24703804ns) {DropHandshakes:469(496ns) {DropIncomingHandshake:441(81ns) DropOutgoingHandshake:456(76ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:476(942ns) {SwitchToState:178(508ns)}}}}}
2025-11-19T13:59:24.420928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435809839581830:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12095
2025-11-19T13:59:24.420950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435809839581831:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12095
2025-11-19T13:59:24.431974Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435806516270267:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.433857Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435806938719838:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.434018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435806768628084:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50252
2025-11-19T13:59:24.435662Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435806768628065:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:24.435825Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435806768628074:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.435837Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435806768628074:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:24.438858Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435808551305115:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:24.434307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435806938719767:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:24.435789Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435806938719807:2086] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:24.435978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435806938719805:2082] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.436015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:24.436023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435806938719805:2082] [node 4] ICP22 created new session: [5:7574435806938719849:2082]
2025-11-19T13:59:24.436032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435806938719849:2082] [node 4] ICS09 handshake done sender: [5:7574435806938719810:2088] self: [5:7574435806938719809:0] peer: [4:7574435808551305119:0] socket: 405 qp: -1
2025-11-19T13:59:24.436041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435806938719849:2082] [node 4] ICS10 traffic start
2025-11-19T13:59:24.436078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:24.436235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435806938719804:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.436242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:24.436246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435806938719804:2080] [node 6] ICP22 created new session: [5:7574435806938719851:2080]
2025-11-19T13:59:24.436250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435806938719851:2080] [node 6] ICS09 handshake done sender: [5:7574435806938719807:2086] self: [5:7574435806938719806:0] peer: [6:7574435809839581825:0] socket: 404 qp: -1
2025-11-19T13:59:24.436254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435806938719851:2080] [node 6] ICS10 traffic start
2025-11-19T13:59:24.436258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:24.442007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435808551305050:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:24.443643Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435809839581831:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.443680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435809839581762:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.444834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435806516270277:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19993
2025-11-19T13:59:24.444856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435806516270278:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19993
2025-11-19T13:59:24.445488Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435806516270278:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.445642Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435806938719837:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.445670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435806938719765:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:24.446122Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435806516270265:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.446134Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435806516270265:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.446850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435806938719855:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38430
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.449907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435806460196744:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47459
2025-11-19T13:59:24.450530Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435806516270265:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:24.450908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435806768628061:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.450941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:24.450949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435806768628061:2108] [node 6] ICP22 created new session: [3:7574435806768628086:2108]
2025-11-19T13:59:24.450957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435806768628086:2108] [node 6] ICS09 handshake done sender: [3:7574435806768628065:2111] self: [3:7574435806768628064:0] peer: [6:7574435809839581829:0] socket: 418 qp: -1
2025-11-19T13:59:24.450967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435806768628086:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:24.450990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:24.451237Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435806768628072:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.451243Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435806768628072:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:24.452685Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435806768628074:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:24.452889Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435806768628072:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:24.453001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435806768628069:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.453020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:24.453026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435806768628069:2112] [node 5] ICP22 created new session: [3:7574435806768628091:2112]
2025-11-19T13:59:24.453032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435806768628091:2112] [node 5] ICS09 handshake done sender: [3:7574435806768628074:2115] self: [3:7574435806768628073:0] peer: [5:7574435806938719848:0] socket: 427 qp: -1
2025-11-19T13:59:24.453039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435806768628091:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:24.453048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:24.453172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435806768628067:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.453178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:24.453184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435806768628067:2107] [node 4] ICP22 created new session: [3:7574435806768628093:2107]
2025-11-19T13:59:24.453188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435806768628093:2107] [node 4] ICS09 handshake done sender: [3:7574435806768628072:2114] self: [3:7574435806768628071:0] peer: [4:7574435808551305134:0] socket: 426 qp: -1
2025-11-19T13:59:24.453191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435806768628093:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:24.453197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.453753Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435806938719837:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.454871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435809839581833:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12095
2025-11-19T13:59:24.455125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435809839581834:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12095
2025-11-19T13:59:24.455271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435806516270279:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19993
2025-11-19T13:59:24.455296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435806516270260:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.455312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:24.455318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435806516270260:2108] [node 5] ICP22 created new session: [1:7574435806516270282:2108]
2025-11-19T13:59:24.455324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435806516270282:2108] [node 5] ICS09 handshake done sender: [1:7574435806516270265:2111] self: [1:7574435806516270263:0] peer: [5:7574435806938719854:0] socket: 421 qp: -1
2025-11-19T13:59:24.455330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435806516270282:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:24.455337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:24.455716Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435806460196744:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.455397Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435806768628070:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.458137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435806938719765:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.458155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:24.458160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435806938719765:2048] [node 1] ICP22 created new session: [5:7574435806938719857:2048]
2025-11-19T13:59:24.458166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435806938719857:2048] [node 1] ICS09 handshake done sender: [5:7574435806938719837:2111] self: [5:7574435806938719854:0] peer: [1:7574435806516270263:0] socket: 423 qp: -1
2025-11-19T13:59:24.458173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435806938719857:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:24.458180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:24.458273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435806938719767:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19652) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435806938719838:2112] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.459219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435806938719856:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38430
2025-11-19T13:59:24.459351Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435806938719838:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:24.459408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435806938719767:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.459420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:24.459426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435806938719767:2050] [node 3] ICP22 created new session: [5:7574435806938719859:2050]
2025-11-19T13:59:24.459430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435806938719859:2050] [node 3] ICS09 handshake done sender: [5:7574435806938719838:2112] self: [5:7574435806938719848:0] peer: [3:7574435806768628073:0] socket: 434 qp: -1
2025-11-19T13:59:24.459435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435806938719859:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:24.459441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:24.457562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435808551305118:2111] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.457590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:24.457599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435808551305118:2111] [node 5] ICP22 created new session: [4:7574435808551305136:2111]
2025-11-19T13:59:24.457606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435808551305136:2111] [node 5] ICS09 handshake done sender: [4:7574435808551305115:2109] self: [4:7574435808551305119:0] peer: [5:7574435806938719809:0] socket: 407 qp: -1
2025-11-19T13:59:24.457612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435808551305136:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:24.457621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:24.457796Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435808551305122:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.457941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435808551305048:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:24.457958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435808551305138:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17969
2025-11-19T13:59:24.462865Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435808551305127:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.462884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435808551305127:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:24.464432Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435809839581830:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.464462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435809839581750:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19652) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435809839581824:2113] held: no
2025-11-19T13:59:24.464490Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435809839581800:2095] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:24.464600Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435809839581824:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:24.465076Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435806460196738:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.465093Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435806460196738:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:24.464680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435809839581791:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.464696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:24.464703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435809839581791:2063] [node 5] ICP22 created new session: [6:7574435809839581835:2063]
2025-11-19T13:59:24.464708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435809839581835:2063] [node 5] ICS09 handshake done sender: [6:7574435809839581800:2095] self: [6:7574435809839581825:0] peer: [5:7574435806938719806:0] socket: 406 qp: -1
2025-11-19T13:59:24.464715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435809839581835:2063] [node 5] ICS10 traffic start
2025-11-19T13:59:24.464722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:24.464886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435809839581750:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.464893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:24.464898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435809839581750:2050] [node 3] ICP22 created new session: [6:7574435809839581837:2050]
2025-11-19T13:59:24.464902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435809839581837:2050] [node 3] ICS09 handshake done sender: [6:7574435809839581824:2113] self: [6:7574435809839581829:0] peer: [3:7574435806768628064:0] socket: 419 qp: -1
2025-11-19T13:59:24.464907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435809839581837:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:24.464912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:24.465088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435809839581749:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:24.464503Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435806768628082:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.464535Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435806768628084:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.464638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435806768627999:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:24.464730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435806768628000:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:24.464742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435806768628067:2107] [node 4] ICP18 handshake (actor [3:7574435806768628084:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.465826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435806460196746:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47459
2025-11-19T13:59:24.465937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435806460196747:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47459
2025-11-19T13:59:24.466952Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435806460196736:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.466969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435806460196736:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:24.467138Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435806460196738:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:24.467680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435806460196663:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.467699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:24.467705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435806460196663:2049] [node 3] ICP22 created new session: [2:7574435806460196749:2049]
2025-11-19T13:59:24.467710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435806460196749:2049] [node 3] ICS09 handshake done sender: [2:7574435806460196738:2114] self: [2:7574435806460196737:0] peer: [3:7574435806768628097:0] socket: 430 qp: -1
2025-11-19T13:59:24.467717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435806460196749:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:24.467724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.469662Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435806460196736:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:24.470137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435806460196732:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.470148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:24.470152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435806460196732:2110] [node 6] ICP22 created new session: [2:7574435806460196753:2110]
2025-11-19T13:59:24.470157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435806460196753:2110] [node 6] ICS09 handshake done sender: [2:7574435806460196736:2113] self: [2:7574435806460196735:0] peer: [6:7574435809839581839:0] socket: 429 qp: -1
2025-11-19T13:59:24.470161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435806460196753:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:24.470166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.469577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435809839581841:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12095
2025-11-19T13:59:24.469830Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435809839581830:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:24.469883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435809839581749:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.469904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:24.469910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435809839581749:2049] [node 2] ICP22 created new session: [6:7574435809839581842:2049]
2025-11-19T13:59:24.469916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435809839581842:2049] [node 2] ICS09 handshake done sender: [6:7574435809839581830:2117] self: [6:7574435809839581839:0] peer: [2:7574435806460196735:0] socket: 432 qp: -1
2025-11-19T13:59:24.469921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435809839581842:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:24.469928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:24.471893Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435806516270260:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(16563023ns) {HandleHandshakeStatus:329(16561632ns) {DropIncomingHandshake:441(3971ns) DropOutgoingHandshake:456(116ns) SwitchToState:178(201ns) ProcessPendingSessionEvents:495(8785876ns) {ForwardSessionEventToSession:612(8770668ns) ForwardSessionEventToSession:612(1522ns) ForwardSessionEventToSession:612(1399ns) ForwardSessionEventToSession:612(660ns) ForwardSessionEventToSession:612(488ns) ForwardSessionEventToSession:612(1364ns) ForwardSessionEventToSession:612(661ns) ForwardSessionEventToSession:612(651ns) ForwardSessionEventToSession:612(483ns) ForwardSessionEventToSession:612(920ns)}}}
2025-11-19T13:59:24.472214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435806938719861:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38430
2025-11-19T13:59:24.475994Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435808551305123:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:24.475972Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435806516270277:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.476069Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435806516270269:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.476073Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435806516270269:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.479466Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435806460196746:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.479492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435806460196663:2049] [node 3] ICP18 handshake (actor [2:7574435806460196746:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.480615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435809839581844:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12095
2025-11-19T13:59:24.480645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435809839581845:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12095
2025-11-19T13:59:24.477280Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435806516270262:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.477290Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435806516270262:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:24.478242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435806516270284:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19993
2025-11-19T13:59:24.479668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435808551305050:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19652) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435808551305123:2114] held: no
2025-11-19T13:59:24.479699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435808551305050:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.479709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:24.479715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435808551305050:2050] [node 3] ICP22 created new session: [4:7574435808551305141:2050]
2025-11-19T13:59:24.479721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435808551305141:2050] [node 3] ICS09 handshake done sender: [4:7574435808551305123:2114] self: [4:7574435808551305134:0] peer: [3:7574435806768628071:0] socket: 436 qp: -1
2025-11-19T13:59:24.479728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435808551305141:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:24.479737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.481888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435806768628098:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50252
2025-11-19T13:59:24.482125Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435806938719861:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.482168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435806938719766:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.482314Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435806516270284:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.482367Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435806516270269:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.482287Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435808551305127:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:24.482565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435808551305143:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17969
2025-11-19T13:59:24.482331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435806768628099:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50252
2025-11-19T13:59:24.482937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435808551305125:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.482974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:24.482982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435808551305125:2112] [node 6] ICP22 created new session: [4:7574435808551305144:2112]
2025-11-19T13:59:24.482989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435808551305144:2112] [node 6] ICS09 handshake done sender: [4:7574435808551305127:2116] self: [4:7574435808551305126:0] peer: [6:7574435809839581832:0] socket: 438 qp: -1
2025-11-19T13:59:24.482996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435808551305144:2112] [node 6] ICS10 traffic start
2025-11-19T13:59:24.483003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:24.482598Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435806460196740:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.482606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435806460196740:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.483457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435806938719863:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38430
2025-11-19T13:59:24.483625Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435806516270262:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:24.484090Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435806516270279:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.484106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435806516270197:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.484116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:24.484121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435806516270197:2049] [node 3] ICP22 created new session: [1:7574435806516270289:2049]
2025-11-19T13:59:24.484126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435806516270289:2049] [node 3] ICS09 handshake done sender: [1:7574435806516270269:2113] self: [1:7574435806516270256:0] peer: [3:7574435806768628096:0] socket: 422 qp: -1
2025-11-19T13:59:24.484131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435806516270289:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:24.484137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:24.484240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435806516270259:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.484245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:24.484249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435806516270259:2107] [node 4] ICP22 created new session: [1:7574435806516270291:2107]
2025-11-19T13:59:24.484252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435806516270291:2107] [node 4] ICS09 handshake done sender: [1:7574435806516270262:2109] self: [1:7574435806516270261:0] peer: [4:7574435808551305139:0] socket: 420 qp: -1
2025-11-19T13:59:24.484256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435806516270291:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:24.484260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:24.485411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435806516270197:2049] [node 3] ICP18 handshake (actor [1:7574435806516270284:2119]) is too old
2025-11-19T13:59:24.485441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435806516270259:2107] [node 4] ICP18 handshake (actor [1:7574435806516270279:2118]) is too old
2025-11-19T13:59:24.486014Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435806460196740:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:24.486242Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435806938719861:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:24.483795Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435806768628070:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:24.486540Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435806768628082:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:24.486392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435806938719766:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.486406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:24.486413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435806938719766:2049] [node 2] ICP22 created new session: [5:7574435806938719864:2049]
2025-11-19T13:59:24.486425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435806938719864:2049] [node 2] ICS09 handshake done sender: [5:7574435806938719861:2119] self: [5:7574435806938719862:0] peer: [2:7574435806460196739:0] socket: 463 qp: -1
2025-11-19T13:59:24.486431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435806938719864:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:24.486439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:24.487199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435806460196734:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.487216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:24.487226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435806460196734:2111] [node 5] ICP22 created new session: [2:7574435806460196756:2111]
2025-11-19T13:59:24.487232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435806460196756:2111] [node 5] ICS09 handshake done sender: [2:7574435806460196740:2115] self: [2:7574435806460196739:0] peer: [5:7574435806938719862:0] socket: 461 qp: -1
2025-11-19T13:59:24.487236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435806460196756:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:24.487244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:24.487639Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435808551305122:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:24.487942Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435806460196732:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:24.488231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435806768628000:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.488248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:24.488253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435806768628000:2049] [node 2] ICP22 created new session: [3:7574435806768628100:2049]
2025-11-19T13:59:24.488259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435806768628100:2049] [node 2] ICS09 handshake done sender: [3:7574435806768628082:2118] self: [3:7574435806768628097:0] peer: [2:7574435806460196737:0] socket: 431 qp: -1
2025-11-19T13:59:24.488264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435806768628100:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:24.488271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:24.488678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435808551305048:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.488692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:24.488697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435808551305048:2048] [node 1] ICP22 created new session: [4:7574435808551305146:2048]
2025-11-19T13:59:24.488702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435808551305146:2048] [node 1] ICS09 handshake done sender: [4:7574435808551305122:2113] self: [4:7574435808551305139:0] peer: [1:7574435806516270261:0] socket: 424 qp: -1
2025-11-19T13:59:24.488706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435808551305146:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:24.488721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:24.489581Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435809839581831:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:24.489718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435809839581762:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12614) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435809839581831:2118] held: no
2025-11-19T13:59:24.489748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435809839581762:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.489757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:24.489762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435809839581762:2059] [node 4] ICP22 created new session: [6:7574435809839581846:2059]
2025-11-19T13:59:24.489767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435809839581846:2059] [node 4] ICS09 handshake done sender: [6:7574435809839581831:2118] self: [6:7574435809839581832:0] peer: [4:7574435808551305126:0] socket: 441 qp: -1
2025-11-19T13:59:24.489772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435809839581846:2059] [node 4] ICS10 traffic start
2025-11-19T13:59:24.489779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:24.490846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435806768628000:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:24.490943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435806768627999:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.490952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:24.490956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435806768627999:2048] [node 1] ICP22 created new session: [3:7574435806768628102:2048]
2025-11-19T13:59:24.490961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435806768628102:2048] [node 1] ICS09 handshake done sender: [3:7574435806768628070:2113] self: [3:7574435806768628096:0] peer: [1:7574435806516270256:0] socket: 425 qp: -1
2025-11-19T13:59:24.490966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435806768628102:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:24.490972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:24.492390Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435806768627999:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.493210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435806460196758:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47459
2025-11-19T13:59:24.494516Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435806460196758:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.494550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435806460196662:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:24.494684Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435806516270275:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.494695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435806516270275:2115] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.495275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435806460196760:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47459
2025-11-19T13:59:24.495386Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435809839581844:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.495417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435809839581748:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:24.495711Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435806516270275:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:24.495972Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435806460196758:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:24.496073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435806460196662:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.496090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:24.496095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435806460196662:2048] [node 1] ICP22 created new session: [2:7574435806460196761:2048]
2025-11-19T13:59:24.496100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435806460196761:2048] [node 1] ICS09 handshake done sender: [2:7574435806460196758:2119] self: [2:7574435806460196759:0] peer: [1:7574435806516270268:0] socket: 411 qp: -1
2025-11-19T13:59:24.496106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435806460196761:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:24.496112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:24.497547Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435806516270271:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:24.497557Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435806516270271:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:24.498064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435806516270196:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.498088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:24.498094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435806516270196:2048] [node 2] ICP22 created new session: [1:7574435806516270295:2048]
2025-11-19T13:59:24.498099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435806516270295:2048] [node 2] ICS09 handshake done sender: [1:7574435806516270275:2115] self: [1:7574435806516270268:0] peer: [2:7574435806460196759:0] socket: 410 qp: -1
2025-11-19T13:59:24.498105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435806516270295:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:24.498113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:24.499367Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435806516270271:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:24.499486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435806516270266:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:24.499503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:24.499509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435806516270266:2110] [node 6] ICP22 created new session: [1:7574435806516270298:2110]
2025-11-19T13:59:24.499514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435806516270298:2110] [node 6] ICS09 handshake done sender: [1:7574435806516270271:2114] self: [1:7574435806516270270:0] peer: [6:7574435809839581848:0] socket: 465 qp: -1
2025-11-19T13:59:24.499518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435806516270298:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:24.499525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.500673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435809839581748:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6478) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435809839581844:2122] held: no
2025-11-19T13:59:24.500741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435809839581849:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12095
2025-11-19T13:59:24.501169Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435809839581844:2122] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:24.501232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435809839581748:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:24.501243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:24.501249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435809839581748:2048] [node 1] ICP22 created new session: [6:7574435809839581850:2048]
2025-11-19T13:59:24.501254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435809839581850:2048] [node 1] ICS09 handshake done sender: [6:7574435809839581844:2122] self: [6:7574435809839581848:0] peer: [1:7574435806516270270:0] socket: 467 qp: -1
2025-11-19T13:59:24.501260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435809839581850:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:24.501268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:24.508315Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:24.508419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435806460196761:2048] [node 1] ICS01 socket: 411 reason# EndOfStream
2025-11-19T13:59:24.508428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435806460196662:2048] [node 1] ICP30 unregister session Session# [2:7574435806460196761:2048] VirtualId# [2:7574435806460196759:0]
2025-11-19T13:59:24.508435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435806460196761:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.508470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:24.508489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:24.512012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:24.512111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435806768628102:2048] [node 1] ICS01 socket: 425 reason# EndOfStream
2025-11-19T13:59:24.512120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435806768627999:2048] [node 1] ICP30 unregister session Session# [3:7574435806768628102:2048] VirtualId# [3:7574435806768628096:0]
2025-11-19T13:59:24.512129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435806768628102:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.512178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:24.512205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:24.521739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:24.521801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435809839581850:2048] [node 1] ICS01 socket: 467 reason# EndOfStream
2025-11-19T13:59:24.521806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435809839581748:2048] [node 1] ICP30 unregister session Session# [6:7574435809839581850:2048] VirtualId# [6:7574435809839581848:0]
2025-11-19T13:59:24.521811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435809839581850:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.521820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:24.521855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:24.521993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:24.522055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435808551305146:2048] [node 1] ICS01 socket: 424 reason# EndOfStream
2025-11-19T13:59:24.522060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435808551305048:2048] [node 1] ICP30 unregister session Session# [4:7574435808551305146:2048] VirtualId# [4:7574435808551305139:0]
2025-11-19T13:59:24.522065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435808551305146:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.522094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:24.522116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:24.521489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:24.521591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435806938719857:2048] [node 1] ICS01 socket: 423 reason# EndOfStream
2025-11-19T13:59:24.521601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435806938719765:2048] [node 1] ICP30 unregister session Session# [5:7574435806938719857:2048] VirtualId# [5:7574435806938719854:0]
2025-11-19T13:59:24.521608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435806938719857:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.521672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:24.521697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:24.525322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:24.525412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435806768628100:2049] [node 2] ICS07 socket disconnect 431 reason# EndOfStream
2025-11-19T13:59:24.525419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435806768628100:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.525463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:24.525481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435806768628100:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:24.528734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:24.528821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435808551305113:2049] [node 2] ICS01 socket: 402 reason# EndOfStream
2025-11-19T13:59:24.528830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435808551305049:2049] [node 2] ICP30 unregister session Session# [4:7574435808551305113:2049] VirtualId# [4:7574435808551305109:0]
2025-11-19T13:59:24.528837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435808551305113:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.528888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:24.528919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:24.528953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:24.529013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435809839581842:2049] [node 2] ICS01 socket: 432 reason# EndOfStream
2025-11-19T13:59:24.529018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435809839581749:2049] [node 2] ICP30 unregister session Session# [6:7574435809839581842:2049] VirtualId# [6:7574435809839581839:0]
2025-11-19T13:59:24.529022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435809839581842:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.529050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:24.529072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:24.529101Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:24.529162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435806938719864:2049] [node 2] ICS01 socket: 463 reason# EndOfStream
2025-11-19T13:59:24.529166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435806938719766:2049] [node 2] ICP30 unregister session Session# [5:7574435806938719864:2049] VirtualId# [5:7574435806938719862:0]
2025-11-19T13:59:24.529170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435806938719864:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.529195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:24.529214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:24.536758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:24.536850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435806938719859:2050] [node 3] ICS01 socket: 434 reason# EndOfStream
2025-11-19T13:59:24.536860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435806938719767:2050] [node 3] ICP30 unregister session Session# [5:7574435806938719859:2050] VirtualId# [5:7574435806938719848:0]
2025-11-19T13:59:24.536866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435806938719859:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.536922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:24.536945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:24.537931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:24.538002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435809839581837:2050] [node 3] ICS01 socket: 419 reason# EndOfStream
2025-11-19T13:59:24.538015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435809839581750:2050] [node 3] ICP30 unregister session Session# [6:7574435809839581837:2050] VirtualId# [6:7574435809839581829:0]
2025-11-19T13:59:24.538022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435809839581837:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.538070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:24.538092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:24.554592Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:24.554693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435809839581846:2059] [node 4] ICS01 socket: 441 reason# EndOfStream
2025-11-19T13:59:24.554704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435809839581762:2059] [node 4] ICP30 unregister session Session# [6:7574435809839581846:2059] VirtualId# [6:7574435809839581832:0]
2025-11-19T13:59:24.554712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435809839581846:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:24.554753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:24.554777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:24.736056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435807685972681:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:24.736075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435807685972682:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:24.816158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435807542573023:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:24.816180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435807542573024:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:24.847869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435809937912248:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:24.847889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435809937912249:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:24.875889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435806396366033:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:24.875904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435806396366034:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:24.875909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435806396366035:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:24.927896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435808930163428:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:24.927912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435808930163429:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:24.927917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435808930163430:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:24.955847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435807799326870:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:24.955875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435807799326871:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:24.962457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435807799326869:2048] [node 1] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.974765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435807542573083:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47181
2025-11-19T13:59:24.975198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435807799326884:2061] [node 5] ICP01 ready to work
2025-11-19T13:59:24.976950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435807685972748:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:24.977047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435807685972749:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:24.977596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435807542573085:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:24.979289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435807799326889:2063] [node 4] ICP01 ready to work
2025-11-19T13:59:24.979554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435809937912281:2079] [node 5] ICP01 ready to work
2025-11-19T13:59:24.980094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435808930163475:2090] [node 6] ICP01 ready to work
2025-11-19T13:59:24.980331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435806396366078:2090] [node 5] ICP01 ready to work
2025-11-19T13:59:24.987870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435807685972745:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.995212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435807542573096:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.993522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435807799326922:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54900
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.995053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435809937912307:2088] [node 4] ICP01 ready to work
2025-11-19T13:59:24.995711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435809937912308:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13381
2025-11-19T13:59:24.995725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435809937912309:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13381
2025-11-19T13:59:24.996514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435808930163500:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12815
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:24.997279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435808930163501:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12815
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.004084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435808930163428:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17856364ns) {Configure:104(17855719ns) {ProcessConfigured:135(17581980ns) {StartInitialHandshake:157(17581551ns) {DropHandshakes:469(462ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:476(1417ns) {SwitchToState:178(610ns)}}}}}
2025-11-19T13:59:25.005152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435807685972760:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33829
2025-11-19T13:59:25.005172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435807685972761:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33829
2025-11-19T13:59:25.006539Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435808930163501:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.007903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435807799326870:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19939338ns) {Configure:104(19938036ns) {ProcessConfigured:135(19597374ns) {StartInitialHandshake:157(19596829ns) {DropHandshakes:469(465ns) {DropIncomingHandshake:441(93ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(333ns) PrepareNewSessionHandshake:476(1367ns) {SwitchToState:178(1135ns)}}}}}
2025-11-19T13:59:25.009611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435812094294229:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54900
2025-11-19T13:59:25.009692Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435809937912281:2079] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(29771467ns) {Configure:104(21652902ns) {ProcessConfigured:135(21355853ns) {StartInitialHandshake:157(21355489ns) {DropHandshakes:469(532ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:476(799ns) {SwitchToState:178(610ns)}}}}}
2025-11-19T13:59:25.009771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435814232879623:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13381
2025-11-19T13:59:25.011490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435808930163502:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12815
2025-11-19T13:59:25.011784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435806396366095:2098] [node 6] ICP01 ready to work
2025-11-19T13:59:25.010960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435809937912286:2076] [node 6] ICP01 ready to work
2025-11-19T13:59:25.011809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435808930163486:2101] [node 4] ICP01 ready to work
2025-11-19T13:59:25.016078Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435807799326922:2093] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.016092Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435809937912309:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.015928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435807542573089:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.021724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435806396366033:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21796736ns) {Configure:104(21795920ns) {ProcessConfigured:135(21436899ns) {StartInitialHandshake:157(21436682ns) {DropHandshakes:469(775ns) {DropIncomingHandshake:441(87ns) DropOutgoingHandshake:456(161ns)} GenerateSessionVirtualId:988(261ns) PrepareNewSessionHandshake:476(916ns) {SwitchToState:178(68ns)}}}}}
2025-11-19T13:59:25.022363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435810691333398:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50465
2025-11-19T13:59:25.022409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435810691333399:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50465
2025-11-19T13:59:25.023132Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435810691333398:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.021043Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435807542573085:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(43381473ns) {Configure:104(43380457ns) {ProcessConfigured:135(43068681ns) {StartInitialHandshake:157(43067888ns) {DropHandshakes:469(597ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:476(1366ns) {SwitchToState:178(294ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.033761Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435808930163500:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.034146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435808930163428:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.031920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435811980940058:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33829
2025-11-19T13:59:25.031871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435809937912286:2076] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20789280ns) {Configure:104(20788117ns) {ProcessConfigured:135(20352744ns) {StartInitialHandshake:157(20352072ns) {DropHandshakes:469(794ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(288ns)} GenerateSessionVirtualId:988(171ns) PrepareNewSessionHandshake:476(1301ns) {SwitchToState:178(313ns)}}}}}
2025-11-19T13:59:25.034317Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435807685972751:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.034322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435807685972751:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.034961Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435807685972751:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:25.035048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435807685972748:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.035061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:25.035070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435807685972748:2109] [node 5] ICP22 created new session: [1:7574435811980940060:2109]
2025-11-19T13:59:25.035076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435811980940060:2109] [node 5] ICS09 handshake done sender: [1:7574435807685972751:2111] self: [1:7574435807685972750:0] peer: [5:7574435813225130805:0] socket: 411 qp: -1
2025-11-19T13:59:25.035083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435811980940060:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:25.035121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:25.033266Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435806396366035:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21402641ns) {Configure:104(21402064ns) {ProcessConfigured:135(21112639ns) {StartInitialHandshake:157(21112188ns) {DropHandshakes:469(486ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(242ns) PrepareNewSessionHandshake:476(903ns) {SwitchToState:178(282ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.036165Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435806396366078:2090] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(55778402ns) {Configure:104(55777880ns) {ProcessConfigured:135(55401414ns) {StartInitialHandshake:157(55401049ns) {DropHandshakes:469(501ns) {DropIncomingHandshake:441(82ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(117ns) PrepareNewSessionHandshake:476(520ns) {SwitchToState:178(42ns)}}}}}
2025-11-19T13:59:25.036096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435813225130807:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12815
2025-11-19T13:59:25.036599Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435808930163501:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:25.036693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435808930163428:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.036709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:25.036715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435808930163428:2048] [node 1] ICP22 created new session: [5:7574435813225130808:2048]
2025-11-19T13:59:25.036720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435813225130808:2048] [node 1] ICS09 handshake done sender: [5:7574435808930163501:2114] self: [5:7574435813225130805:0] peer: [1:7574435807685972750:0] socket: 425 qp: -1
2025-11-19T13:59:25.036727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435813225130808:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:25.036743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.039540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435814232879627:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13381
2025-11-19T13:59:25.039862Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435814232879627:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.039896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435809937912248:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.040008Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435814232879623:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.040869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435811837540399:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47181
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.041286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435811837540400:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47181
2025-11-19T13:59:25.043841Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435810691333399:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.043871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435806396366035:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.044119Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435809937912313:2105] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.044127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435809937912313:2105] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.044660Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435809937912313:2105] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:25.044740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435809937912307:2088] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.044755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:25.044760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435809937912307:2088] [node 4] ICP22 created new session: [3:7574435814232879631:2088]
2025-11-19T13:59:25.044765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435814232879631:2088] [node 4] ICS09 handshake done sender: [3:7574435809937912313:2105] self: [3:7574435809937912312:0] peer: [4:7574435810691333410:0] socket: 423 qp: -1
2025-11-19T13:59:25.044771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435814232879631:2088] [node 4] ICS10 traffic start
2025-11-19T13:59:25.044778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:25.047190Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435811837540400:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.047221Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435807542573083:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.047252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435807542573023:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.048033Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435808930163502:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.048082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435813225130811:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12815
2025-11-19T13:59:25.048101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435808930163429:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.049252Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435807685972760:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.049115Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435807542573098:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.049127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435807542573098:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.049971Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435807542573098:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:25.050147Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435807685972742:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.050155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435807685972742:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.050196Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435811837540399:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.050208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435811837540401:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47181
2025-11-19T13:59:25.050221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435807542573096:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.050233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:25.050241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435807542573096:2112] [node 5] ICP22 created new session: [2:7574435811837540405:2112]
2025-11-19T13:59:25.050247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435811837540405:2112] [node 5] ICS09 handshake done sender: [2:7574435807542573098:2113] self: [2:7574435807542573097:0] peer: [5:7574435813225130812:0] socket: 420 qp: -1
2025-11-19T13:59:25.050253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435811837540405:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:25.050270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:25.050355Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435807685972761:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.050506Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435809937912308:2102] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.050529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435809937912249:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.051830Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435807685972755:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.048187Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435812094294229:2101] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.051848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435807685972755:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.048218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435807799326884:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.050045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435807799326869:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(48375589ns) {Configure:104(48375187ns) {ProcessConfigured:135(48060017ns) {StartInitialHandshake:157(48059820ns) {DropHandshakes:469(533ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(88ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:476(739ns) {SwitchToState:178(130ns)}}}}}
2025-11-19T13:59:25.050074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435807799326869:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.050300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435812094294246:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54900
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.052136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435810691333411:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50465
2025-11-19T13:59:25.052200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435810691333412:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50465
2025-11-19T13:59:25.051842Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435808930163495:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.051857Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435808930163495:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.052469Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435813225130811:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.052501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435808930163430:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.059883Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435812094294246:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.059932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435807799326889:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.055935Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435810691333411:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.061568Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435814232879622:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.061584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435814232879622:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.063185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435808930163429:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19896) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435808930163502:2115] held: no
2025-11-19T13:59:25.060356Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435810691333399:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:25.061651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435806396366035:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.061666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:25.061672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435806396366035:2050] [node 3] ICP22 created new session: [4:7574435810691333414:2050]
2025-11-19T13:59:25.061678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435810691333414:2050] [node 3] ICS09 handshake done sender: [4:7574435810691333399:2112] self: [4:7574435810691333410:0] peer: [3:7574435809937912312:0] socket: 436 qp: -1
2025-11-19T13:59:25.061685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435810691333414:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:25.061694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:25.060540Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435807542573092:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.060553Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435807542573092:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.061372Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435807685972753:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.061388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435807685972753:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.066422Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435807542573092:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:25.066554Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435811837540401:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.066565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435807542573024:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.066581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:25.066588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435807542573024:2049] [node 3] ICP22 created new session: [2:7574435811837540410:2049]
2025-11-19T13:59:25.066595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435811837540410:2049] [node 3] ICS09 handshake done sender: [2:7574435807542573092:2111] self: [2:7574435807542573091:0] peer: [3:7574435814232879634:0] socket: 414 qp: -1
2025-11-19T13:59:25.066600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435811837540410:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:25.066608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.071022Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435809937912249:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19896) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435809937912308:2102] held: no
2025-11-19T13:59:25.071164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435812094294249:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54900
2025-11-19T13:59:25.068142Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435807685972755:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:25.067956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435807542573096:2112] [node 5] ICP35 event processing took too much time PendingConnection:278(17698904ns) {HandleHandshakeStatus:329(17698105ns) {DropIncomingHandshake:441(2636ns) DropOutgoingHandshake:456(329ns) SwitchToState:178(57ns) ProcessPendingSessionEvents:495(6671350ns) {ForwardSessionEventToSession:612(6659878ns) ForwardSessionEventToSession:612(1732ns) ForwardSessionEventToSession:612(1077ns) ForwardSessionEventToSession:612(557ns) ForwardSessionEventToSession:612(492ns) ForwardSessionEventToSession:612(883ns) ForwardSessionEventToSession:612(655ns) ForwardSessionEventToSession:612(842ns) ForwardSessionEventToSession:612(653ns) ForwardSessionEventToSession:612(950ns)}}}
2025-11-19T13:59:25.069569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435807685972682:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.069585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:25.069592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435807685972682:2049] [node 3] ICP22 created new session: [1:7574435811980940066:2049]
2025-11-19T13:59:25.069599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435811980940066:2049] [node 3] ICS09 handshake done sender: [1:7574435807685972755:2113] self: [1:7574435807685972754:0] peer: [3:7574435814232879628:0] socket: 443 qp: -1
2025-11-19T13:59:25.069605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435811980940066:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:25.069613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:25.071707Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435812094294249:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.068572Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435810691333403:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.071760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435807799326870:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.068586Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435810691333403:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.072683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435812094294251:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54900
2025-11-19T13:59:25.073789Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435812094294249:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:25.073920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435807799326870:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.073937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:25.073942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435807799326870:2049] [node 2] ICP22 created new session: [6:7574435812094294252:2049]
2025-11-19T13:59:25.073947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435812094294252:2049] [node 2] ICS09 handshake done sender: [6:7574435812094294249:2118] self: [6:7574435812094294250:0] peer: [2:7574435807542573086:0] socket: 459 qp: -1
2025-11-19T13:59:25.073953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435812094294252:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:25.073959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:25.071930Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435811837540396:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.071940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435811837540396:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.073628Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435811837540396:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:25.075646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435807542573085:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.075657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:25.075664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435807542573085:2108] [node 6] ICP22 created new session: [2:7574435811837540416:2108]
2025-11-19T13:59:25.075670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435811837540416:2108] [node 6] ICS09 handshake done sender: [2:7574435811837540396:2114] self: [2:7574435807542573086:0] peer: [6:7574435812094294250:0] socket: 456 qp: -1
2025-11-19T13:59:25.075675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435811837540416:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:25.075681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:25.073179Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435807685972742:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:25.073261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435807685972681:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.073276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:25.073282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435807685972681:2048] [node 2] ICP22 created new session: [1:7574435811980940070:2048]
2025-11-19T13:59:25.073287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435811980940070:2048] [node 2] ICS09 handshake done sender: [1:7574435807685972742:2107] self: [1:7574435807685972741:0] peer: [2:7574435811837540403:0] socket: 408 qp: -1
2025-11-19T13:59:25.073292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435811980940070:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:25.073298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.078106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435810691333416:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50465
2025-11-19T13:59:25.078125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435810691333417:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50465
2025-11-19T13:59:25.077427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435811980940072:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33829
2025-11-19T13:59:25.078726Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435807685972753:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:25.078835Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435811980940058:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.078878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435807685972749:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.078887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:25.078894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435807685972749:2110] [node 6] ICP22 created new session: [1:7574435811980940074:2110]
2025-11-19T13:59:25.078900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435811980940074:2110] [node 6] ICS09 handshake done sender: [1:7574435807685972753:2112] self: [1:7574435807685972752:0] peer: [6:7574435812094294247:0] socket: 412 qp: -1
2025-11-19T13:59:25.078905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435811980940074:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:25.078912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.083860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435813225130815:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12815
2025-11-19T13:59:25.083939Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435807685972759:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.080268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435811837540415:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47181
2025-11-19T13:59:25.083280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435807542573023:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9604) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435807542573083:2107] held: no
2025-11-19T13:59:25.083950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435807685972759:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.083302Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435807542573083:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:25.084681Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435807685972759:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:25.084753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435807685972745:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.084768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:25.084773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435807685972745:2108] [node 4] ICP22 created new session: [1:7574435811980940078:2108]
2025-11-19T13:59:25.084778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435811980940078:2108] [node 4] ICS09 handshake done sender: [1:7574435807685972759:2114] self: [1:7574435807685972758:0] peer: [4:7574435810691333420:0] socket: 462 qp: -1
2025-11-19T13:59:25.084783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435811980940078:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:25.084811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:25.080335Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435810691333417:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.080395Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435810691333403:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:25.081738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435806396366033:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.082252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435806396366095:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.082266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:25.082272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435806396366095:2098] [node 6] ICP22 created new session: [4:7574435810691333421:2098]
2025-11-19T13:59:25.082278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435810691333421:2098] [node 6] ICS09 handshake done sender: [4:7574435810691333403:2114] self: [4:7574435810691333402:0] peer: [6:7574435812094294248:0] socket: 439 qp: -1
2025-11-19T13:59:25.082285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435810691333421:2098] [node 6] ICS10 traffic start
2025-11-19T13:59:25.082292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:25.083601Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435810691333416:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.083623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435806396366034:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.088856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435811980940072:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.088890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435807685972749:2110] [node 6] ICP18 handshake (actor [1:7574435811980940072:2118]) is too old
2025-11-19T13:59:25.087923Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435808930163495:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:25.088723Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435807799326889:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8645) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435812094294246:2117] held: no
2025-11-19T13:59:25.089961Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435806396366033:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9604) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435810691333417:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.090181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435813225130816:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12815
2025-11-19T13:59:25.090228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435808930163475:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.090242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:25.090247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435808930163475:2090] [node 6] ICP22 created new session: [5:7574435813225130817:2090]
2025-11-19T13:59:25.090253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435813225130817:2090] [node 6] ICS09 handshake done sender: [5:7574435808930163495:2110] self: [5:7574435808930163476:0] peer: [6:7574435812094294245:0] socket: 427 qp: -1
2025-11-19T13:59:25.090258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435813225130817:2090] [node 6] ICS10 traffic start
2025-11-19T13:59:25.090265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:25.090355Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435808930163502:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:25.090634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435808930163429:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.090644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:25.090648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435808930163429:2049] [node 2] ICP22 created new session: [5:7574435813225130819:2049]
2025-11-19T13:59:25.090652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435813225130819:2049] [node 2] ICS09 handshake done sender: [5:7574435808930163502:2115] self: [5:7574435813225130812:0] peer: [2:7574435807542573097:0] socket: 426 qp: -1
2025-11-19T13:59:25.090656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435813225130819:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:25.090661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.093924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435814232879635:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13381
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.094440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435810691333424:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50465
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.096187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435812094294255:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54900
2025-11-19T13:59:25.094491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435814232879636:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13381
2025-11-19T13:59:25.096434Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435809937912248:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9604) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435814232879627:2118] held: no
2025-11-19T13:59:25.096477Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435814232879627:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:25.095291Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435811837540398:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.095304Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435811837540398:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.095854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435807542573023:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.095865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:25.095872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435807542573023:2048] [node 1] ICP22 created new session: [2:7574435811837540418:2048]
2025-11-19T13:59:25.095878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435811837540418:2048] [node 1] ICS09 handshake done sender: [2:7574435807542573083:2107] self: [2:7574435811837540403:0] peer: [1:7574435807685972741:0] socket: 409 qp: -1
2025-11-19T13:59:25.095885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435811837540418:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:25.095892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.096644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435813225130821:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12815
2025-11-19T13:59:25.096886Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435810691333417:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.097021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435812094294256:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54900
2025-11-19T13:59:25.097196Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435814232879622:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.099881Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435813225130816:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.099920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435808930163486:2101] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.100011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435812094294257:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54900
2025-11-19T13:59:25.100081Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435811837540398:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:25.101950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435809937912248:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.101969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:25.101977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435809937912248:2048] [node 1] ICP22 created new session: [3:7574435814232879639:2048]
2025-11-19T13:59:25.101983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435814232879639:2048] [node 1] ICS09 handshake done sender: [3:7574435814232879627:2118] self: [3:7574435814232879628:0] peer: [1:7574435807685972754:0] socket: 446 qp: -1
2025-11-19T13:59:25.101991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435814232879639:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:25.101996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:25.102096Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435809937912308:2102] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.102264Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435810691333409:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.102273Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435810691333409:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.102830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435806396366033:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.102841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:25.102847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435806396366033:2048] [node 1] ICP22 created new session: [4:7574435810691333426:2048]
2025-11-19T13:59:25.102853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435810691333426:2048] [node 1] ICS09 handshake done sender: [4:7574435810691333417:2120] self: [4:7574435810691333420:0] peer: [1:7574435807685972758:0] socket: 466 qp: -1
2025-11-19T13:59:25.102860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435810691333426:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:25.102865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:25.102985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435810691333425:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50465
2025-11-19T13:59:25.103170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435809937912281:2079] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.103189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:25.103195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435809937912281:2079] [node 5] ICP22 created new session: [3:7574435814232879641:2079]
2025-11-19T13:59:25.103202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435814232879641:2079] [node 5] ICS09 handshake done sender: [3:7574435814232879622:2115] self: [3:7574435809937912303:0] peer: [5:7574435813225130813:0] socket: 451 qp: -1
2025-11-19T13:59:25.103207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435814232879641:2079] [node 5] ICS10 traffic start
2025-11-19T13:59:25.103212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:25.104172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435807542573089:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.104196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:25.104203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435807542573089:2109] [node 4] ICP22 created new session: [2:7574435811837540421:2109]
2025-11-19T13:59:25.104208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435811837540421:2109] [node 4] ICS09 handshake done sender: [2:7574435811837540398:2115] self: [2:7574435811837540397:0] peer: [4:7574435810691333423:0] socket: 460 qp: -1
2025-11-19T13:59:25.104214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435811837540421:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:25.104221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:25.103572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435809937912249:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.103585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:25.103590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435809937912249:2049] [node 2] ICP22 created new session: [3:7574435814232879643:2049]
2025-11-19T13:59:25.103595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435814232879643:2049] [node 2] ICS09 handshake done sender: [3:7574435809937912308:2102] self: [3:7574435814232879634:0] peer: [2:7574435807542573091:0] socket: 418 qp: -1
2025-11-19T13:59:25.103600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435814232879643:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:25.103606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:25.105898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435808930163430:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17716) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435813225130811:2119] held: no
2025-11-19T13:59:25.106107Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435807799326869:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9604) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435807799326922:2093] held: no
2025-11-19T13:59:25.106134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435807799326884:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3890) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435812094294229:2101] held: no
2025-11-19T13:59:25.106201Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435807799326922:2093] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:25.106367Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435812094294229:2101] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:25.106440Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435812094294246:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.106510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435807799326869:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.106523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:25.106529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435807799326869:2048] [node 1] ICP22 created new session: [6:7574435812094294259:2048]
2025-11-19T13:59:25.106534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435812094294259:2048] [node 1] ICS09 handshake done sender: [6:7574435807799326922:2093] self: [6:7574435812094294247:0] peer: [1:7574435807685972752:0] socket: 415 qp: -1
2025-11-19T13:59:25.106540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435812094294259:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:25.106544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:25.106628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435807799326884:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.106634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:25.106638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435807799326884:2061] [node 5] ICP22 created new session: [6:7574435812094294261:2061]
2025-11-19T13:59:25.106642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435812094294261:2061] [node 5] ICS09 handshake done sender: [6:7574435812094294229:2101] self: [6:7574435812094294245:0] peer: [5:7574435808930163476:0] socket: 432 qp: -1
2025-11-19T13:59:25.106646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435812094294261:2061] [node 5] ICS10 traffic start
2025-11-19T13:59:25.106651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:25.106770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435812094294258:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54900
2025-11-19T13:59:25.107364Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435812094294258:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.107392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435807799326889:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.107400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:25.107407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435807799326889:2063] [node 4] ICP22 created new session: [6:7574435812094294263:2063]
2025-11-19T13:59:25.107411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435812094294263:2063] [node 4] ICS09 handshake done sender: [6:7574435812094294246:2117] self: [6:7574435812094294248:0] peer: [4:7574435810691333402:0] socket: 440 qp: -1
2025-11-19T13:59:25.107416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435812094294263:2063] [node 4] ICS10 traffic start
2025-11-19T13:59:25.107421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:25.107537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435807799326871:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:25.107987Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435813225130811:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:25.108169Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435806396366034:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19896) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435810691333416:2119] held: no
2025-11-19T13:59:25.107686Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435814232879625:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:25.107695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435814232879625:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:25.108419Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435810691333409:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:25.109099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435808930163430:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.109115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:25.109120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435808930163430:2050] [node 3] ICP22 created new session: [5:7574435813225130823:2050]
2025-11-19T13:59:25.109127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435813225130823:2050] [node 3] ICS09 handshake done sender: [5:7574435813225130811:2119] self: [5:7574435813225130813:0] peer: [3:7574435809937912303:0] socket: 452 qp: -1
2025-11-19T13:59:25.109132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435813225130823:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:25.109149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.109950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435812094294266:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54900
2025-11-19T13:59:25.110025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435806396366078:2090] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.110043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:25.110049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435806396366078:2090] [node 5] ICP22 created new session: [4:7574435810691333429:2090]
2025-11-19T13:59:25.110054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435810691333429:2090] [node 5] ICS09 handshake done sender: [4:7574435810691333409:2116] self: [4:7574435806396366079:0] peer: [5:7574435813225130822:0] socket: 468 qp: -1
2025-11-19T13:59:25.110059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435810691333429:2090] [node 5] ICS10 traffic start
2025-11-19T13:59:25.110065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:25.110973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435808930163486:2101] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8645) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435813225130816:2121] held: no
2025-11-19T13:59:25.111194Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435814232879625:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:25.111345Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435812094294258:2123] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:25.111401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435807799326871:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.111413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:25.111418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435807799326871:2050] [node 3] ICP22 created new session: [6:7574435812094294267:2050]
2025-11-19T13:59:25.111422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435812094294267:2050] [node 3] ICS09 handshake done sender: [6:7574435812094294258:2123] self: [6:7574435812094294265:0] peer: [3:7574435814232879624:0] socket: 421 qp: -1
2025-11-19T13:59:25.111427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435812094294267:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:25.111432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:25.112604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435809937912286:2076] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:25.112621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:25.112625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435809937912286:2076] [node 6] ICP22 created new session: [3:7574435814232879646:2076]
2025-11-19T13:59:25.112630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435814232879646:2076] [node 6] ICS09 handshake done sender: [3:7574435814232879625:2117] self: [3:7574435814232879624:0] peer: [6:7574435812094294265:0] socket: 473 qp: -1
2025-11-19T13:59:25.112635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435814232879646:2076] [node 6] ICS10 traffic start
2025-11-19T13:59:25.112641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:25.113106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435813225130825:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12815
2025-11-19T13:59:25.113428Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435813225130816:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:25.113594Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435810691333416:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:25.113878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435808930163486:2101] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.113888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:25.113893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435808930163486:2101] [node 4] ICP22 created new session: [5:7574435813225130826:2101]
2025-11-19T13:59:25.113898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435813225130826:2101] [node 4] ICS09 handshake done sender: [5:7574435813225130816:2121] self: [5:7574435813225130822:0] peer: [4:7574435806396366079:0] socket: 469 qp: -1
2025-11-19T13:59:25.113903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435813225130826:2101] [node 4] ICS10 traffic start
2025-11-19T13:59:25.113910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:25.114181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435806396366034:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:25.114194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:25.114197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435806396366034:2049] [node 2] ICP22 created new session: [4:7574435810691333431:2049]
2025-11-19T13:59:25.114202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435810691333431:2049] [node 2] ICS09 handshake done sender: [4:7574435810691333416:2119] self: [4:7574435810691333423:0] peer: [2:7574435811837540397:0] socket: 465 qp: -1
2025-11-19T13:59:25.114206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435810691333431:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:25.114213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:25.120754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:25.120848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435814232879639:2048] [node 1] ICS01 socket: 446 reason# EndOfStream
2025-11-19T13:59:25.120857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435809937912248:2048] [node 1] ICP30 unregister session Session# [3:7574435814232879639:2048] VirtualId# [3:7574435814232879628:0]
2025-11-19T13:59:25.120864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435814232879639:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:25.120895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:25.120914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:25.121263Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:25.121335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435811837540418:2048] [node 1] ICS01 socket: 409 reason# EndOfStream
2025-11-19T13:59:25.121341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435807542573023:2048] [node 1] ICP30 unregister session Session# [2:7574435811837540418:2048] VirtualId# [2:7574435811837540403:0]
2025-11-19T13:59:25.121345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435811837540418:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:25.121376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:25.121394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:25.168781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:25.168899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435810691333426:2048] [node 1] ICS01 socket: 466 reason# EndOfStream
2025-11-19T13:59:25.168912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435806396366033:2048] [node 1] ICP30 unregister session Session# [4:7574435810691333426:2048] VirtualId# [4:7574435810691333420:0]
2025-11-19T13:59:25.168921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435810691333426:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:25.168976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:25.168999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:25.170724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:25.170817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435813225130808:2048] [node 1] ICS01 socket: 425 reason# EndOfStream
2025-11-19T13:59:25.170825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435808930163428:2048] [node 1] ICP30 unregister session Session# [5:7574435813225130808:2048] VirtualId# [5:7574435813225130805:0]
2025-11-19T13:59:25.170831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435813225130808:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:25.170883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:25.170906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:25.171871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:25.171942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435812094294259:2048] [node 1] ICS01 socket: 415 reason# EndOfStream
2025-11-19T13:59:25.171955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435807799326869:2048] [node 1] ICP30 unregister session Session# [6:7574435812094294259:2048] VirtualId# [6:7574435812094294247:0]
2025-11-19T13:59:25.171960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435812094294259:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:25.172005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:25.172032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:25.176292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:25.176367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435814232879643:2049] [node 2] ICS01 socket: 418 reason# EndOfStream
2025-11-19T13:59:25.176375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435809937912249:2049] [node 2] ICP30 unregister session Session# [3:7574435814232879643:2049] VirtualId# [3:7574435814232879634:0]
2025-11-19T13:59:25.176379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435814232879643:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:25.176411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:25.176428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:25.183991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:25.184092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435812094294252:2049] [node 2] ICS01 socket: 459 reason# EndOfStream
2025-11-19T13:59:25.184100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435807799326870:2049] [node 2] ICP30 unregister session Session# [6:7574435812094294252:2049] VirtualId# [6:7574435812094294250:0]
2025-11-19T13:59:25.184107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435812094294252:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:25.184158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:25.184182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:25.185284Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:25.185355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435813225130819:2049] [node 2] ICS01 socket: 426 reason# EndOfStream
2025-11-19T13:59:25.185361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435808930163429:2049] [node 2] ICP30 unregister session Session# [5:7574435813225130819:2049] VirtualId# [5:7574435813225130812:0]
2025-11-19T13:59:25.185365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435813225130819:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:25.185402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:25.185422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:25.185515Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:25.185574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435810691333431:2049] [node 2] ICS01 socket: 465 reason# EndOfStream
2025-11-19T13:59:25.185579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435806396366034:2049] [node 2] ICP30 unregister session Session# [4:7574435810691333431:2049] VirtualId# [4:7574435810691333423:0]
2025-11-19T13:59:25.185582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435810691333431:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:25.185606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:25.185624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:25.197705Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:25.197814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435813225130823:2050] [node 3] ICS01 socket: 452 reason# EndOfStream
2025-11-19T13:59:25.197826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435808930163430:2050] [node 3] ICP30 unregister session Session# [5:7574435813225130823:2050] VirtualId# [5:7574435813225130813:0]
2025-11-19T13:59:25.197833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435813225130823:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:25.197891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:25.197916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:25.198134Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:25.198212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435812094294267:2050] [node 3] ICS01 socket: 421 reason# EndOfStream
2025-11-19T13:59:25.198222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435807799326871:2050] [node 3] ICP30 unregister session Session# [6:7574435812094294267:2050] VirtualId# [6:7574435812094294265:0]
2025-11-19T13:59:25.198226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435812094294267:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:25.198254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:25.198273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:25.212981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:25.213084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435812094294263:2063] [node 4] ICS01 socket: 440 reason# EndOfStream
2025-11-19T13:59:25.213092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435807799326889:2063] [node 4] ICP30 unregister session Session# [6:7574435812094294263:2063] VirtualId# [6:7574435812094294248:0]
2025-11-19T13:59:25.213100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435812094294263:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:25.213158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:25.213183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:25.534292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435814445223659:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:25.534306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435814445223660:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:25.623142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435813518536895:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:25.623156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435813518536896:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:25.739910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435810312892746:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:25.739923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435810312892747:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:25.856232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435811012946086:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:25.856250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435811012946087:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:25.856255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435811012946088:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:25.964143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435812655336914:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:25.964155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435812655336915:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:25.964160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435812655336916:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:26.027914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435816822508968:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:26.027929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435816822508969:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:26.027935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435816822508970:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:26.122053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435816822509015:2093] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.123168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435815307913443:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30800
2025-11-19T13:59:26.123821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435818740191016:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:26.124020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435816822509022:2096] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.131641Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435815307913443:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.131696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435815307913444:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30800
2025-11-19T13:59:26.131863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435815307913446:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:26.132264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435815307913446:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.132081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435814607860104:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:26.133551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435814607860110:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:26.136513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435818740191027:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:26.137428Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435816822509017:2094] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.137445Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435816822509017:2094] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.144295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435816950304271:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1109
2025-11-19T13:59:26.144360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435816950304272:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1109
2025-11-19T13:59:26.144998Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435816950304272:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.146531Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435816950304271:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.147919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435816950304276:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1109
2025-11-19T13:59:26.150914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435818740191025:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:26.147967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435817813504260:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:26.148047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435817813504261:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:26.148653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435817813504259:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11561
2025-11-19T13:59:26.146535Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435814607860106:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.146549Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435814607860106:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.149372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435814607860109:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:26.149457Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435814607860106:2109] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.152463Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818740191020:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.152471Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435818740191020:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.152916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435818740191032:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51583
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.153224Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435818740191020:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:26.153521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435818740191016:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.153537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:26.153543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435818740191016:2107] [node 4] ICP22 created new session: [1:7574435818740191034:2107]
2025-11-19T13:59:26.153549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435818740191034:2107] [node 4] ICS09 handshake done sender: [1:7574435818740191020:2108] self: [1:7574435818740191017:0] peer: [4:7574435815307913448:0] socket: 417 qp: -1
2025-11-19T13:59:26.153558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435818740191034:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:26.153566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:26.152181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816822509039:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43785
2025-11-19T13:59:26.152342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816822509040:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43785
2025-11-19T13:59:26.151868Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435815307913444:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.152195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435811012946086:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.153107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435815307913449:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30800
2025-11-19T13:59:26.153403Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435815307913444:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:26.153813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435811012946086:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.153821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:26.153824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435811012946086:2048] [node 1] ICP22 created new session: [4:7574435815307913450:2048]
2025-11-19T13:59:26.153828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435815307913450:2048] [node 1] ICS09 handshake done sender: [4:7574435815307913444:2109] self: [4:7574435815307913448:0] peer: [1:7574435818740191017:0] socket: 418 qp: -1
2025-11-19T13:59:26.153832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435815307913450:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:26.153862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:26.155865Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435817813504259:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.159864Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435816822509017:2094] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:26.156549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435815307913452:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30800
2025-11-19T13:59:26.160741Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435815307913443:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:26.159994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435812655336916:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.160038Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435816950304272:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:26.160071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435816950304275:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:26.160377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435816950304275:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.163481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435812655336916:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.163498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:26.163505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435812655336916:2050] [node 3] ICP22 created new session: [5:7574435816950304278:2050]
2025-11-19T13:59:26.163513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435816950304278:2050] [node 3] ICS09 handshake done sender: [5:7574435816950304272:2109] self: [5:7574435816950304273:0] peer: [3:7574435814607860105:0] socket: 426 qp: -1
2025-11-19T13:59:26.163521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435816950304278:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:26.163541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:26.163766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435817813504256:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:26.164112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435814607860104:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.164135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:26.164143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435814607860104:2108] [node 5] ICP22 created new session: [3:7574435814607860120:2108]
2025-11-19T13:59:26.164150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435814607860120:2108] [node 5] ICS09 handshake done sender: [3:7574435814607860106:2109] self: [3:7574435814607860105:0] peer: [5:7574435816950304273:0] socket: 421 qp: -1
2025-11-19T13:59:26.164157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435814607860120:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:26.164191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:26.165607Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435814607860109:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16159215ns) {Configure:104(16158539ns) {ProcessConfigured:135(15917055ns) {StartInitialHandshake:157(15916647ns) {DropHandshakes:469(469ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(45ns)} GenerateSessionVirtualId:988(240ns) PrepareNewSessionHandshake:476(923ns) {SwitchToState:178(253ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.167237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435816950304280:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1109
2025-11-19T13:59:26.167615Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435818740191025:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16624451ns) {Configure:104(16623883ns) {ProcessConfigured:135(16313006ns) {StartInitialHandshake:157(16312618ns) {DropHandshakes:469(485ns) {DropIncomingHandshake:441(93ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:476(656ns) {SwitchToState:178(456ns)}}}}}
2025-11-19T13:59:26.168144Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435816822509024:2098] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.168162Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435816822509024:2098] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.168327Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435816822509039:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.168197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435810312892747:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(22280969ns) {Configure:104(22279257ns) {ProcessConfigured:135(21930503ns) {StartInitialHandshake:157(21930093ns) {DropHandshakes:469(778ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(218ns)} GenerateSessionVirtualId:988(253ns) PrepareNewSessionHandshake:476(603ns) {SwitchToState:178(385ns)}}}}}
2025-11-19T13:59:26.171844Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818740191022:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.171862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435818740191022:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.172590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435813518536895:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.172633Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435813518536895:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16711963ns) {IncomingHandshake:261(16711356ns) {DropIncomingHandshake:441(403ns) PrepareNewSessionHandshake:476(490ns) {SwitchToState:178(300ns)} GenerateSessionVirtualId:988(177ns) IssueIncomingHandshakeReply:193(16704626ns) {SwitchToState:178(957ns)}}}
2025-11-19T13:59:26.172136Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435816950304280:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.172473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435812655336915:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.175470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435818740191027:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(38903271ns) {Configure:104(38902476ns) {ProcessConfigured:135(38591732ns) {StartInitialHandshake:157(38591219ns) {DropHandshakes:469(497ns) {DropIncomingHandshake:441(96ns) DropOutgoingHandshake:456(77ns)} GenerateSessionVirtualId:988(90ns) PrepareNewSessionHandshake:476(574ns) {SwitchToState:178(414ns)}}}}}
2025-11-19T13:59:26.176014Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435817813504265:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.176026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435817813504265:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.176925Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818740191032:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.175738Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435816822509040:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.179822Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435817813504265:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.176114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435816822508970:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.176369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435816822508969:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.176692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435816822509015:2093] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.176709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:26.176717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435816822509015:2093] [node 4] ICP22 created new session: [6:7574435816822509044:2093]
2025-11-19T13:59:26.176724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435816822509044:2093] [node 4] ICS09 handshake done sender: [6:7574435816822509017:2094] self: [6:7574435816822509016:0] peer: [4:7574435815307913447:0] socket: 414 qp: -1
2025-11-19T13:59:26.176730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435816822509044:2093] [node 4] ICS10 traffic start
2025-11-19T13:59:26.176756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:26.180601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816822509047:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43785
2025-11-19T13:59:26.180782Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435816822509024:2098] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:26.180896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435816822509022:2096] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.180910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:26.180915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435816822509022:2096] [node 5] ICP22 created new session: [6:7574435816822509048:2096]
2025-11-19T13:59:26.180921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435816822509048:2096] [node 5] ICS09 handshake done sender: [6:7574435816822509024:2098] self: [6:7574435816822509023:0] peer: [5:7574435816950304277:0] socket: 416 qp: -1
2025-11-19T13:59:26.180927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435816822509048:2096] [node 5] ICS10 traffic start
2025-11-19T13:59:26.180945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.183542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435815307913453:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30800
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.185231Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435816822509047:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.185596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435816822508968:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.184089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435814607860125:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
2025-11-19T13:59:26.184139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435814607860126:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
2025-11-19T13:59:26.187316Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435814607860112:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.187329Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435814607860112:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.187892Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435818740191022:2109] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:26.187521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435815307913446:2110] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.187548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:26.187558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435815307913446:2110] [node 6] ICP22 created new session: [4:7574435815307913454:2110]
2025-11-19T13:59:26.187566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435815307913454:2110] [node 6] ICS09 handshake done sender: [4:7574435815307913443:2108] self: [4:7574435815307913447:0] peer: [6:7574435816822509016:0] socket: 415 qp: -1
2025-11-19T13:59:26.187573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435815307913454:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:26.187601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:26.187975Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435814607860126:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.188003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435810312892746:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.188239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435814445223659:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.188258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:26.188293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435814445223659:2048] [node 2] ICP22 created new session: [1:7574435818740191041:2048]
2025-11-19T13:59:26.188301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435818740191041:2048] [node 2] ICS09 handshake done sender: [1:7574435818740191022:2109] self: [1:7574435818740191018:0] peer: [2:7574435817813504269:0] socket: 420 qp: -1
2025-11-19T13:59:26.188308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435818740191041:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:26.188315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:26.188098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435815307913453:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.190119Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435817813504263:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.190137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435817813504263:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.190666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435815307913446:2110] [node 6] ICP35 event processing took too much time PendingConnection:278(29825675ns) {HandleHandshakeStatus:329(29824713ns) {DropIncomingHandshake:441(1193ns) DropOutgoingHandshake:456(197ns) SwitchToState:178(103ns) ProcessPendingSessionEvents:495(331ns)}}
2025-11-19T13:59:26.191805Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818740191029:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.191824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435818740191029:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.192247Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818740191036:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.192273Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435818740191036:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.192638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435816950304283:2113] [node 4] ICP01 ready to work
2025-11-19T13:59:26.193040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435811012946088:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.193188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435815307913458:2114] [node 5] ICP01 ready to work
2025-11-19T13:59:26.192459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435817813504261:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.192476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:26.192484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435817813504261:2110] [node 5] ICP22 created new session: [2:7574435817813504273:2110]
2025-11-19T13:59:26.192489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435817813504273:2110] [node 5] ICS09 handshake done sender: [2:7574435817813504265:2113] self: [2:7574435817813504264:0] peer: [5:7574435816950304281:0] socket: 433 qp: -1
2025-11-19T13:59:26.192496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435817813504273:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:26.192512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:26.194301Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435816822509015:2093] [node 4] ICP35 event processing took too much time PendingConnection:278(33353221ns) {HandleHandshakeStatus:329(33352953ns) {DropIncomingHandshake:441(600ns) DropOutgoingHandshake:456(433ns) SwitchToState:178(50ns) ProcessPendingSessionEvents:495(6618585ns) {ForwardSessionEventToSession:612(6610622ns) ForwardSessionEventToSession:612(1662ns) ForwardSessionEventToSession:612(1066ns) ForwardSessionEventToSession:612(569ns) ForwardSessionEventToSession:612(569ns) ForwardSessionEventToSession:612(948ns)}}}
2025-11-19T13:59:26.195400Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435817813504254:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.195406Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435817813504254:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.192584Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435814607860125:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.192621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435810312892747:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.196095Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435814607860112:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:26.198673Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435817813504263:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:26.198819Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435817813504254:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:26.198418Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435814607860122:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.198437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435814607860122:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.209063Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435818740191029:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:26.209174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435814445223660:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.209198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:26.209209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435814445223660:2049] [node 3] ICP22 created new session: [1:7574435818740191045:2049]
2025-11-19T13:59:26.209216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435818740191045:2049] [node 3] ICS09 handshake done sender: [1:7574435818740191029:2112] self: [1:7574435818740191019:0] peer: [3:7574435814607860127:0] socket: 424 qp: -1
2025-11-19T13:59:26.209224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435818740191045:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:26.209248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.212501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435817813504260:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.212525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:26.212534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435817813504260:2109] [node 6] ICP22 created new session: [2:7574435817813504280:2109]
2025-11-19T13:59:26.212541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435817813504280:2109] [node 6] ICS09 handshake done sender: [2:7574435817813504263:2112] self: [2:7574435817813504262:0] peer: [6:7574435816822509043:0] socket: 432 qp: -1
2025-11-19T13:59:26.212548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435817813504280:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:26.212556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:26.212657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435813518536896:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.212663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:26.212667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435813518536896:2049] [node 3] ICP22 created new session: [2:7574435817813504282:2049]
2025-11-19T13:59:26.212671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435817813504282:2049] [node 3] ICS09 handshake done sender: [2:7574435817813504254:2108] self: [2:7574435817813504253:0] peer: [3:7574435814607860128:0] socket: 423 qp: -1
2025-11-19T13:59:26.212675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435817813504282:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:26.212678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:26.213822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435817813504278:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11561
2025-11-19T13:59:26.214050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435816950304286:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1109
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.214615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816822509051:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43785
2025-11-19T13:59:26.214835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816822509052:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43785
2025-11-19T13:59:26.214690Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435816950304283:2113] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18872828ns) {Configure:104(18871452ns) {ProcessConfigured:135(18580867ns) {StartInitialHandshake:157(18579738ns) {DropHandshakes:469(704ns) {DropIncomingHandshake:441(82ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:476(1086ns) {SwitchToState:178(270ns)}}}}}
2025-11-19T13:59:26.214445Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435818740191036:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:26.215555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435818740191025:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.215569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:26.215575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435818740191025:2110] [node 6] ICP22 created new session: [1:7574435818740191048:2110]
2025-11-19T13:59:26.215581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435818740191048:2110] [node 6] ICS09 handshake done sender: [1:7574435818740191036:2114] self: [1:7574435818740191031:0] peer: [6:7574435816822509050:0] socket: 442 qp: -1
2025-11-19T13:59:26.215586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435818740191048:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:26.215593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.216028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435811012946087:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20085338ns) {Configure:104(20084283ns) {ProcessConfigured:135(19816835ns) {StartInitialHandshake:157(19816305ns) {DropHandshakes:469(453ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:476(737ns) {SwitchToState:178(392ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.217105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816822509053:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43785
2025-11-19T13:59:26.216295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435812655336914:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(20158796ns) {Configure:104(20158309ns) {ProcessConfigured:135(19952817ns) {StartInitialHandshake:157(19952308ns) {DropHandshakes:469(229ns) {DropIncomingHandshake:441(46ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(147ns) PrepareNewSessionHandshake:476(916ns) {SwitchToState:178(28ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.217431Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435816822509039:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:26.217542Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435816822509040:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:26.218132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435810312892746:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32713) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435814607860126:2116] held: no
2025-11-19T13:59:26.218193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435814607860131:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
2025-11-19T13:59:26.218342Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435816822509047:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:26.218612Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435816950304280:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:26.218684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435812655336915:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.218707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:26.218713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435812655336915:2049] [node 2] ICP22 created new session: [5:7574435816950304291:2049]
2025-11-19T13:59:26.218718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435816950304291:2049] [node 2] ICS09 handshake done sender: [5:7574435816950304280:2112] self: [5:7574435816950304281:0] peer: [2:7574435817813504264:0] socket: 434 qp: -1
2025-11-19T13:59:26.218724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435816950304291:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:26.218740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:26.219988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435817813504279:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11561
2025-11-19T13:59:26.220120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435814607860110:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.220138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:26.220145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435814607860110:2110] [node 6] ICP22 created new session: [3:7574435814607860133:2110]
2025-11-19T13:59:26.220152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435814607860133:2110] [node 6] ICS09 handshake done sender: [3:7574435814607860112:2111] self: [3:7574435814607860111:0] peer: [6:7574435816822509042:0] socket: 427 qp: -1
2025-11-19T13:59:26.220159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435814607860133:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:26.220165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.221070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435818740191050:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51583
2025-11-19T13:59:26.221112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435818740191051:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51583
2025-11-19T13:59:26.220853Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435817813504259:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:26.221202Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435817813504261:2110] [node 5] ICP35 event processing took too much time PendingConnection:278(28719393ns) {HandleHandshakeStatus:329(28718065ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(219ns) SwitchToState:178(80ns) ProcessPendingSessionEvents:495(17297498ns) {ForwardSessionEventToSession:612(17285776ns) ForwardSessionEventToSession:612(1352ns) ForwardSessionEventToSession:612(687ns) ForwardSessionEventToSession:612(531ns) ForwardSessionEventToSession:612(543ns) ForwardSessionEventToSession:612(961ns) ForwardSessionEventToSession:612(532ns) ForwardSessionEventToSession:612(493ns) ForwardSessionEventToSession:612(683ns) ForwardSessionEventToSession:612(1125ns)}}}
2025-11-19T13:59:26.220725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435814607860132:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
2025-11-19T13:59:26.222818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435813518536895:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.222832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:26.222849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435813518536895:2048] [node 1] ICP22 created new session: [2:7574435817813504285:2048]
2025-11-19T13:59:26.222855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435817813504285:2048] [node 1] ICS09 handshake done sender: [2:7574435817813504259:2111] self: [2:7574435817813504269:0] peer: [1:7574435818740191018:0] socket: 422 qp: -1
2025-11-19T13:59:26.222860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435817813504285:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:26.222866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:26.227886Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818740191050:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.228003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435814445223659:2048] [node 2] ICP18 handshake (actor [1:7574435818740191050:2116]) is too old
2025-11-19T13:59:26.227930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435816822508968:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.227952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:26.227959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435816822508968:2048] [node 1] ICP22 created new session: [6:7574435816822509054:2048]
2025-11-19T13:59:26.227976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435816822509054:2048] [node 1] ICS09 handshake done sender: [6:7574435816822509047:2114] self: [6:7574435816822509050:0] peer: [1:7574435818740191031:0] socket: 443 qp: -1
2025-11-19T13:59:26.227983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435816822509054:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:26.227991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:26.228178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435816822508969:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.228185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:26.228192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435816822508969:2049] [node 2] ICP22 created new session: [6:7574435816822509056:2049]
2025-11-19T13:59:26.228196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435816822509056:2049] [node 2] ICS09 handshake done sender: [6:7574435816822509040:2113] self: [6:7574435816822509043:0] peer: [2:7574435817813504262:0] socket: 435 qp: -1
2025-11-19T13:59:26.228201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435816822509056:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:26.228205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:26.228306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435816822508970:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.228311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:26.228314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435816822508970:2050] [node 3] ICP22 created new session: [6:7574435816822509058:2050]
2025-11-19T13:59:26.228319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435816822509058:2050] [node 3] ICS09 handshake done sender: [6:7574435816822509039:2112] self: [6:7574435816822509042:0] peer: [3:7574435814607860111:0] socket: 428 qp: -1
2025-11-19T13:59:26.228323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435816822509058:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:26.228328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:26.229066Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435814607860125:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:26.232280Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435814607860126:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.240417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435815307913465:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30800
2025-11-19T13:59:26.240464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435815307913466:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30800
2025-11-19T13:59:26.240493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435815307913467:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30800
2025-11-19T13:59:26.243618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435810312892747:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.243648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:26.243657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435810312892747:2049] [node 2] ICP22 created new session: [3:7574435814607860136:2049]
2025-11-19T13:59:26.243664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435814607860136:2049] [node 2] ICS09 handshake done sender: [3:7574435814607860125:2115] self: [3:7574435814607860128:0] peer: [2:7574435817813504253:0] socket: 425 qp: -1
2025-11-19T13:59:26.243672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435814607860136:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:26.243681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:26.243893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435810312892746:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.243902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:26.243909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435810312892746:2048] [node 1] ICP22 created new session: [3:7574435814607860138:2048]
2025-11-19T13:59:26.243914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435814607860138:2048] [node 1] ICS09 handshake done sender: [3:7574435814607860126:2116] self: [3:7574435814607860127:0] peer: [1:7574435818740191019:0] socket: 448 qp: -1
2025-11-19T13:59:26.243919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435814607860138:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:26.243924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:26.244505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435814607860110:2110] [node 6] ICP35 event processing took too much time PendingConnection:278(48301225ns) {HandleHandshakeStatus:329(48300643ns) {DropIncomingHandshake:441(472ns) DropOutgoingHandshake:456(79ns) SwitchToState:178(57ns) ProcessPendingSessionEvents:495(21782239ns) {ForwardSessionEventToSession:612(21770958ns) ForwardSessionEventToSession:612(1694ns) ForwardSessionEventToSession:612(872ns) ForwardSessionEventToSession:612(563ns) ForwardSessionEventToSession:612(728ns) ForwardSessionEventToSession:612(1324ns) ForwardSessionEventToSession:612(587ns) ForwardSessionEventToSession:612(524ns) ForwardSessionEventToSession:612(570ns) ForwardSessionEventToSession:612(1014ns)}}}
2025-11-19T13:59:26.244016Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818740191051:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.244862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435812655336915:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(26156385ns) {HandleHandshakeStatus:329(26155769ns) {DropIncomingHandshake:441(408ns) DropOutgoingHandshake:456(74ns) SwitchToState:178(220ns) ProcessPendingSessionEvents:495(22110978ns) {ForwardSessionEventToSession:612(22104690ns) ForwardSessionEventToSession:612(2066ns) ForwardSessionEventToSession:612(987ns) ForwardSessionEventToSession:612(928ns) ForwardSessionEventToSession:612(618ns)}}}
2025-11-19T13:59:26.245110Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435817813504279:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.245252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435813518536896:2049] [node 3] ICP18 handshake (actor [2:7574435817813504279:2117]) is too old
2025-11-19T13:59:26.245512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435813518536895:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22673285ns) {HandleHandshakeStatus:329(22672947ns) {DropIncomingHandshake:441(289ns) DropOutgoingHandshake:456(1918ns) SwitchToState:178(68ns) ProcessPendingSessionEvents:495(22603866ns) {ForwardSessionEventToSession:612(22595226ns) ForwardSessionEventToSession:612(1215ns) ForwardSessionEventToSession:612(569ns) ForwardSessionEventToSession:612(851ns) ForwardSessionEventToSession:612(501ns) ForwardSessionEventToSession:612(491ns) ForwardSessionEventToSession:612(585ns) ForwardSessionEventToSession:612(729ns) ForwardSessionEventToSession:612(480ns) ForwardSessionEventToSession:612(574ns)}}}
2025-11-19T13:59:26.246150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435817813504287:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11561
2025-11-19T13:59:26.246769Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435814607860122:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:26.247972Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435815307913465:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.247995Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435815307913466:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.248068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435811012946087:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.248430Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435815307913453:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:26.248231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435816950304293:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1109
2025-11-19T13:59:26.248356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435814607860109:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.248374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:26.248380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435814607860109:2107] [node 4] ICP22 created new session: [3:7574435814607860140:2107]
2025-11-19T13:59:26.248388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435814607860140:2107] [node 4] ICS09 handshake done sender: [3:7574435814607860122:2113] self: [3:7574435814607860119:0] peer: [4:7574435815307913456:0] socket: 441 qp: -1
2025-11-19T13:59:26.248394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435814607860140:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:26.248406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:26.250767Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435816950304271:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:26.250855Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435817813504287:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.251159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435816950304275:2110] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.251171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:26.251179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435816950304275:2110] [node 6] ICP22 created new session: [5:7574435816950304294:2110]
2025-11-19T13:59:26.251185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435816950304294:2110] [node 6] ICS09 handshake done sender: [5:7574435816950304271:2108] self: [5:7574435816950304277:0] peer: [6:7574435816822509023:0] socket: 419 qp: -1
2025-11-19T13:59:26.251190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435816950304294:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:26.251196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:26.248962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435811012946088:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.248976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:26.248981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435811012946088:2050] [node 3] ICP22 created new session: [4:7574435815307913470:2050]
2025-11-19T13:59:26.248985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435815307913470:2050] [node 3] ICS09 handshake done sender: [4:7574435815307913453:2113] self: [4:7574435815307913456:0] peer: [3:7574435814607860119:0] socket: 444 qp: -1
2025-11-19T13:59:26.248990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435815307913470:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:26.248995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:26.255768Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435817813504271:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.255787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435817813504271:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.256307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435816950304296:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1109
2025-11-19T13:59:26.256354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435816950304297:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1109
2025-11-19T13:59:26.256922Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435816950304297:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.256953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435812655336914:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.257236Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818740191038:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.257246Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435818740191038:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.258159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435816950304299:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1109
2025-11-19T13:59:26.258299Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435818740191038:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:26.258476Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435816950304297:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:26.258577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435812655336914:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.258598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:26.258604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435812655336914:2048] [node 1] ICP22 created new session: [5:7574435816950304300:2048]
2025-11-19T13:59:26.258610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435816950304300:2048] [node 1] ICS09 handshake done sender: [5:7574435816950304297:2119] self: [5:7574435816950304298:0] peer: [1:7574435818740191028:0] socket: 455 qp: -1
2025-11-19T13:59:26.258616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435816950304300:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:26.258638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:26.259445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435818740191027:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.259465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:26.259473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435818740191027:2111] [node 5] ICP22 created new session: [1:7574435818740191054:2111]
2025-11-19T13:59:26.259478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435818740191054:2111] [node 5] ICS09 handshake done sender: [1:7574435818740191038:2115] self: [1:7574435818740191028:0] peer: [5:7574435816950304298:0] socket: 453 qp: -1
2025-11-19T13:59:26.259484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435818740191054:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:26.259491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:26.260295Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435816950304296:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.260335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435816950304283:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.261180Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435815307913460:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.261193Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435815307913460:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.261795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435815307913472:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30800
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.262032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435816950304303:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1109
2025-11-19T13:59:26.262202Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435815307913460:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:26.268588Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435816950304296:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:26.268712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435816950304283:2113] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.268728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:26.268733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435816950304283:2113] [node 4] ICP22 created new session: [5:7574435816950304304:2113]
2025-11-19T13:59:26.268738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435816950304304:2113] [node 4] ICS09 handshake done sender: [5:7574435816950304296:2118] self: [5:7574435816950304302:0] peer: [4:7574435815307913459:0] socket: 473 qp: -1
2025-11-19T13:59:26.268743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435816950304304:2113] [node 4] ICS10 traffic start
2025-11-19T13:59:26.268752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:26.267965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435815307913458:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.267995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:26.268004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435815307913458:2114] [node 5] ICP22 created new session: [4:7574435815307913474:2114]
2025-11-19T13:59:26.268012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435815307913474:2114] [node 5] ICS09 handshake done sender: [4:7574435815307913460:2115] self: [4:7574435815307913459:0] peer: [5:7574435816950304302:0] socket: 451 qp: -1
2025-11-19T13:59:26.268020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435815307913474:2114] [node 5] ICS10 traffic start
2025-11-19T13:59:26.268038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:26.270183Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435815307913466:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:26.270675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435811012946087:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.270695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:26.270711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435811012946087:2049] [node 2] ICP22 created new session: [4:7574435815307913476:2049]
2025-11-19T13:59:26.270716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435815307913476:2049] [node 2] ICS09 handshake done sender: [4:7574435815307913466:2118] self: [4:7574435815307913469:0] peer: [2:7574435817813504270:0] socket: 474 qp: -1
2025-11-19T13:59:26.270721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435815307913476:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:26.269980Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435817813504271:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:26.270336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435817813504256:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.270355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:26.270360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435817813504256:2107] [node 4] ICP22 created new session: [2:7574435817813504290:2107]
2025-11-19T13:59:26.270365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435817813504290:2107] [node 4] ICS09 handshake done sender: [2:7574435817813504271:2115] self: [2:7574435817813504270:0] peer: [4:7574435815307913469:0] socket: 470 qp: -1
2025-11-19T13:59:26.270369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435817813504290:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:26.270376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:26.270755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:26.291108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:26.291174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435816822509054:2048] [node 1] ICS07 socket disconnect 443 reason# EndOfStream
2025-11-19T13:59:26.291179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435816822509054:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.291214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:26.288479Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:26.288583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435814607860138:2048] [node 1] ICS01 socket: 448 reason# EndOfStream
2025-11-19T13:59:26.288594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435810312892746:2048] [node 1] ICP30 unregister session Session# [3:7574435814607860138:2048] VirtualId# [3:7574435814607860127:0]
2025-11-19T13:59:26.288602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435814607860138:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.288635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:26.288669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:26.291231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435816822509054:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:26.291683Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:26.291785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435816950304300:2048] [node 1] ICS01 socket: 455 reason# EndOfStream
2025-11-19T13:59:26.291792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435812655336914:2048] [node 1] ICP30 unregister session Session# [5:7574435816950304300:2048] VirtualId# [5:7574435816950304298:0]
2025-11-19T13:59:26.291797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435816950304300:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.291838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:26.291861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:26.291934Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:26.291991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435815307913450:2048] [node 1] ICS07 socket disconnect 418 reason# EndOfStream
2025-11-19T13:59:26.291995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435815307913450:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.292017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:26.292035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435815307913450:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:26.291742Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435816822508968:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32713) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:26.292356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435816822509054:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:26.292362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435816822508968:2048] [node 1] ICP30 unregister session Session# [6:7574435816822509054:2048] VirtualId# [6:7574435816822509050:0]
2025-11-19T13:59:26.292381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:26.292390Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435816822508968:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32713) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.292393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32713) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.294401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435811012946086:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32713) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:26.294413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435815307913450:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:26.294418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435811012946086:2048] [node 1] ICP30 unregister session Session# [4:7574435815307913450:2048] VirtualId# [4:7574435815307913448:0]
2025-11-19T13:59:26.294431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:26.294435Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435811012946086:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32713) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.294438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32713) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.296484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:26.296554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435814607860136:2049] [node 2] ICS01 socket: 425 reason# EndOfStream
2025-11-19T13:59:26.296561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435810312892747:2049] [node 2] ICP30 unregister session Session# [3:7574435814607860136:2049] VirtualId# [3:7574435814607860128:0]
2025-11-19T13:59:26.296567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435814607860136:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.296590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:26.296608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:26.300532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435811012946086:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:26.303063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435816822508968:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:26.319900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:26.320065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435815307913476:2049] [node 2] ICS01 socket: 474 reason# EndOfStream
2025-11-19T13:59:26.320074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435811012946087:2049] [node 2] ICP30 unregister session Session# [4:7574435815307913476:2049] VirtualId# [4:7574435815307913469:0]
2025-11-19T13:59:26.320080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435815307913476:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.320132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:26.320156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:26.319942Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:26.320046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435816822509056:2049] [node 2] ICS01 socket: 435 reason# EndOfStream
2025-11-19T13:59:26.320058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435816822508969:2049] [node 2] ICP30 unregister session Session# [6:7574435816822509056:2049] VirtualId# [6:7574435816822509043:0]
2025-11-19T13:59:26.320067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435816822509056:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.320127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:26.320154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:26.323059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:26.323166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435816950304291:2049] [node 2] ICS07 socket disconnect 434 reason# EndOfStream
2025-11-19T13:59:26.323173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435816950304291:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.323229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:26.323248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435816950304291:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:26.323758Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435812655336915:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10541) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:26.323773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435816950304291:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:26.323779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435812655336915:2049] [node 2] ICP30 unregister session Session# [5:7574435816950304291:2049] VirtualId# [5:7574435816950304281:0]
2025-11-19T13:59:26.323799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:26.323807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435812655336915:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10541) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.323810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10541) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.336270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435812655336915:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:26.347886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:26.347947Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:26.348047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435816950304278:2050] [node 3] ICS07 socket disconnect 426 reason# EndOfStream
2025-11-19T13:59:26.348054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435816950304278:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.348104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:26.348123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435816950304278:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:26.348595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435816822509058:2050] [node 3] ICS01 socket: 428 reason# EndOfStream
2025-11-19T13:59:26.348607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435816822508970:2050] [node 3] ICP30 unregister session Session# [6:7574435816822509058:2050] VirtualId# [6:7574435816822509042:0]
2025-11-19T13:59:26.348613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435816822509058:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.348642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:26.348665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:26.348904Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435812655336916:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7822) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:26.348914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435816950304278:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:26.348918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435812655336916:2050] [node 3] ICP30 unregister session Session# [5:7574435816950304278:2050] VirtualId# [5:7574435816950304273:0]
2025-11-19T13:59:26.348930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:26.348937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435812655336916:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7822) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.348939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7822) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.356845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435812655336916:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:59:26.369845Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:26.369949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435816822509044:2093] [node 4] ICS07 socket disconnect 414 reason# EndOfStream
2025-11-19T13:59:26.369956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435816822509044:2093] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.370003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:26.370025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435816822509044:2093] [node 4] ICS15 start handshake
2025-11-19T13:59:26.373003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435816822509015:2093] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20600) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:26.373019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435816822509044:2093] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:26.373023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435816822509015:2093] [node 4] ICP30 unregister session Session# [6:7574435816822509044:2093] VirtualId# [6:7574435816822509016:0]
2025-11-19T13:59:26.373037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:26.373043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435816822509015:2093] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20600) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.373046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20600) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.383856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435816822509015:2093] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:26.447868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435816059027242:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:26.447883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435816059027243:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:26.439879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435818689891589:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:26.439899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435818689891590:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:26.462925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435817345038520:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:26.462942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435817345038521:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:26.495980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435815477983285:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:26.495990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435815477983286:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:26.495994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435815477983287:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:26.513186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435816736530934:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:26.513197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435816736530935:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:26.513201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435816736530936:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:26.519050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435818689891623:2079] [node 4] ICP01 ready to work
2025-11-19T13:59:26.519072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435818689891624:2080] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.519542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435818689891626:2081] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.519540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435817345038547:2074] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63563
2025-11-19T13:59:26.519659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435816059027271:2074] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19579
2025-11-19T13:59:26.520656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435816059027274:2075] [node 5] ICP01 ready to work
2025-11-19T13:59:26.520340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435817345038549:2075] [node 4] ICP01 ready to work
2025-11-19T13:59:26.520856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435817345038553:2076] [node 5] ICP01 ready to work
2025-11-19T13:59:26.520934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435817345038554:2077] [node 6] ICP01 ready to work
2025-11-19T13:59:26.524115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435815477983315:2070] [node 6] ICP01 ready to work
2025-11-19T13:59:26.525309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435816736530959:2067] [node 4] ICP01 ready to work
2025-11-19T13:59:26.530629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435815477983329:2083] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.535290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435818689891624:2080] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16179428ns) {Configure:104(16178928ns) {ProcessConfigured:135(14430105ns) {StartInitialHandshake:157(14429445ns) {DropHandshakes:469(279ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(35ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:476(726ns) {SwitchToState:178(447ns)}}}}}
2025-11-19T13:59:26.535693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435818689891626:2081] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16098390ns) {Configure:104(16097875ns) {ProcessConfigured:135(14743889ns) {StartInitialHandshake:157(14743244ns) {DropHandshakes:469(299ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(13ns)} GenerateSessionVirtualId:988(49ns) PrepareNewSessionHandshake:476(995ns) {SwitchToState:178(474ns)}}}}}
2025-11-19T13:59:26.535786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435818689891643:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2933
2025-11-19T13:59:26.535842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435818689891644:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2933
2025-11-19T13:59:26.536421Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435817345038547:2074] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.536478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435817345038521:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.538587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435817345038553:2076] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17619534ns) {Configure:104(17619003ns) {ProcessConfigured:135(17374883ns) {StartInitialHandshake:157(17374665ns) {DropHandshakes:469(460ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(72ns) PrepareNewSessionHandshake:476(934ns) {SwitchToState:178(326ns)}}}}}
2025-11-19T13:59:26.539758Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435816059027271:2074] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.537061Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818689891643:2093] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.538478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435818689891590:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18207247ns) {Configure:104(18206264ns) {ProcessConfigured:135(17393908ns) {StartInitialHandshake:157(17393494ns) {DropHandshakes:469(217ns) {DropIncomingHandshake:441(33ns) DropOutgoingHandshake:456(42ns)} GenerateSessionVirtualId:988(145ns) PrepareNewSessionHandshake:476(566ns) {SwitchToState:178(122ns)}}}}}
2025-11-19T13:59:26.538483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435818689891623:2079] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19203746ns) {Configure:104(19201300ns) {ProcessConfigured:135(17419195ns) {StartInitialHandshake:157(17418893ns) {DropHandshakes:469(141ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(149ns) PrepareNewSessionHandshake:476(708ns) {SwitchToState:178(95ns)}}}}}
2025-11-19T13:59:26.540325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435816059027269:2073] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.540340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435816059027269:2073] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.544744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435816059027316:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19579
2025-11-19T13:59:26.544765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435816059027317:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19579
2025-11-19T13:59:26.544909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435816736530935:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24322856ns) {Configure:104(20200022ns) {ProcessConfigured:135(19830286ns) {StartInitialHandshake:157(19829841ns) {DropHandshakes:469(480ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:476(837ns) {SwitchToState:178(235ns)}}}}}
2025-11-19T13:59:26.545115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435816736530959:2067] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19766605ns) {Configure:104(19766332ns) {ProcessConfigured:135(19543686ns) {StartInitialHandshake:157(19543256ns) {DropHandshakes:469(418ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(75ns) PrepareNewSessionHandshake:476(486ns) {SwitchToState:178(116ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.549119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435817345038554:2077] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27794080ns) {Configure:104(20962727ns) {ProcessConfigured:135(20685914ns) {StartInitialHandshake:157(20685313ns) {DropHandshakes:469(535ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(52ns)} GenerateSessionVirtualId:988(306ns) PrepareNewSessionHandshake:476(640ns) {SwitchToState:178(333ns)}}}}}
2025-11-19T13:59:26.547205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435816059027318:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19579
2025-11-19T13:59:26.549860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435816059027290:2087] [node 4] ICP01 ready to work
2025-11-19T13:59:26.550305Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435816059027317:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.491879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435818086603571:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:26.491896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435818086603572:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:26.491901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435818086603573:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:26.520031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435818086603589:2064] [node 5] ICP01 ready to work
2025-11-19T13:59:26.529378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435818086603613:2069] [node 6] ICP01 ready to work
2025-11-19T13:59:26.538183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435818086603589:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(18105600ns) {Configure:104(18104934ns) {ProcessConfigured:135(17164172ns) {StartInitialHandshake:157(17163477ns) {DropHandshakes:469(144ns) {DropIncomingHandshake:441(28ns) DropOutgoingHandshake:456(13ns)} GenerateSessionVirtualId:988(96ns) PrepareNewSessionHandshake:476(844ns) {SwitchToState:178(184ns)}}}}}
2025-11-19T13:59:26.538295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435818086603628:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31528
2025-11-19T13:59:26.548008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435818086603650:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31528
2025-11-19T13:59:26.551888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435816736530971:2076] [node 5] ICP01 ready to work
2025-11-19T13:59:26.549908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435815477983343:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-11-19T13:59:26.552604Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435815477983343:2095] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.556057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818689891644:2094] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.556153Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435816059027316:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.556329Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435818086603628:2094] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.556362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435818086603573:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.560488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435817345038601:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63563
2025-11-19T13:59:26.560676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435818689891670:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2933
2025-11-19T13:59:26.560694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435818689891671:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2933
2025-11-19T13:59:26.562738Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818689891671:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.566111Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435817345038601:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.566172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435817345038520:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.565167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435816059027313:2078] [node 6] ICP01 ready to work
2025-11-19T13:59:26.565680Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435816059027297:2095] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.565689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435816059027297:2095] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.568396Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435817345038583:2100] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.568413Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435817345038583:2100] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.568939Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818689891670:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.570078Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435816059027269:2073] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:26.570252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435816059027243:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.570279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:26.570286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435816059027243:2049] [node 3] ICP22 created new session: [2:7574435816059027326:2049]
2025-11-19T13:59:26.570293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435816059027326:2049] [node 3] ICS09 handshake done sender: [2:7574435816059027269:2073] self: [2:7574435816059027268:0] peer: [3:7574435817345038586:0] socket: 420 qp: -1
2025-11-19T13:59:26.570301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435816059027326:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:26.570321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:26.564656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435816736530936:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(24201285ns) {Configure:104(24200546ns) {ProcessConfigured:135(23895406ns) {StartInitialHandshake:157(23894403ns) {DropHandshakes:469(942ns) {DropIncomingHandshake:441(77ns) DropOutgoingHandshake:456(207ns)} GenerateSessionVirtualId:988(127ns) PrepareNewSessionHandshake:476(1023ns) {SwitchToState:178(378ns)}}}}}
2025-11-19T13:59:26.571561Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435818689891589:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(52451653ns) {Configure:104(52451285ns) {ProcessConfigured:135(50335403ns) {StartInitialHandshake:157(50334450ns) {DropHandshakes:469(521ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(113ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:476(715ns) {SwitchToState:178(345ns)}}}}}
2025-11-19T13:59:26.571678Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435816059027290:2087] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(21746901ns) {Configure:104(21746296ns) {ProcessConfigured:135(21438029ns) {StartInitialHandshake:157(21437744ns) {DropHandshakes:469(912ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(174ns)} GenerateSessionVirtualId:988(277ns) PrepareNewSessionHandshake:476(599ns) {SwitchToState:178(240ns)}}}}}
2025-11-19T13:59:26.575263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435815477983286:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.575302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435815477983286:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(22641356ns) {IncomingHandshake:261(22640641ns) {DropIncomingHandshake:441(223ns) PrepareNewSessionHandshake:476(556ns) {SwitchToState:178(233ns)} GenerateSessionVirtualId:988(159ns) IssueIncomingHandshakeReply:193(22634417ns) {SwitchToState:178(466ns)}}}
2025-11-19T13:59:26.575886Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818689891649:2097] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.575908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435818689891649:2097] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.576910Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435818689891649:2097] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:26.577013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435818689891590:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.577040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:26.577046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435818689891590:2049] [node 3] ICP22 created new session: [1:7574435818689891677:2049]
2025-11-19T13:59:26.577054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435818689891677:2049] [node 3] ICS09 handshake done sender: [1:7574435818689891649:2097] self: [1:7574435818689891630:0] peer: [3:7574435817345038602:0] socket: 441 qp: -1
2025-11-19T13:59:26.577061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435818689891677:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:26.577084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:26.577669Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435816059027297:2095] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:26.577785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435816059027274:2075] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.577805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:26.577811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435816059027274:2075] [node 5] ICP22 created new session: [2:7574435816059027333:2075]
2025-11-19T13:59:26.577817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435816059027333:2075] [node 5] ICS09 handshake done sender: [2:7574435816059027297:2095] self: [2:7574435816059027277:0] peer: [5:7574435815477983358:0] socket: 437 qp: -1
2025-11-19T13:59:26.577823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435816059027333:2075] [node 5] ICS10 traffic start
2025-11-19T13:59:26.577830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.583113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435815477983286:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31564) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435815477983343:2095] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.587558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435817345038605:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63563
2025-11-19T13:59:26.588212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435817345038603:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63563
2025-11-19T13:59:26.588242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435817345038604:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63563
2025-11-19T13:59:26.588301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435817345038521:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31564) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435817345038547:2074] held: no
2025-11-19T13:59:26.588474Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435817345038583:2100] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:26.588513Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435817345038547:2074] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.591703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435815477983363:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-11-19T13:59:26.592543Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435815477983363:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.593799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435815477983285:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.591909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435816736530934:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(50886812ns) {Configure:104(50885434ns) {ProcessConfigured:135(50659586ns) {StartInitialHandshake:157(50658597ns) {DropHandshakes:469(303ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(52ns)} GenerateSessionVirtualId:988(136ns) PrepareNewSessionHandshake:476(1597ns) {SwitchToState:178(672ns)}}}}}
2025-11-19T13:59:26.591988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816736531011:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61779
2025-11-19T13:59:26.592022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816736531012:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61779
2025-11-19T13:59:26.592051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816736531013:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61779
2025-11-19T13:59:26.592097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816736531014:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61779
2025-11-19T13:59:26.595128Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818689891645:2095] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.595142Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435818689891645:2095] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.595411Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435816736531011:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.595415Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435816736531013:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.595437Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435816736531014:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.595447Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435816736531012:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.595496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435816736530934:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.595500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435816736530935:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.595509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435816736530959:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.595611Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435816059027318:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.596211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435818086603652:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31528
2025-11-19T13:59:26.596230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435818086603653:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31528
2025-11-19T13:59:26.596273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435818086603654:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31528
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.596618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435818086603655:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31528
2025-11-19T13:59:26.597584Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435816059027243:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(27309279ns) {HandleHandshakeStatus:329(27308180ns) {DropIncomingHandshake:441(5155ns) DropOutgoingHandshake:456(224ns) SwitchToState:178(70ns) ProcessPendingSessionEvents:495(22167448ns) {ForwardSessionEventToSession:612(22155484ns) ForwardSessionEventToSession:612(2256ns) ForwardSessionEventToSession:612(748ns) ForwardSessionEventToSession:612(608ns) ForwardSessionEventToSession:612(555ns) ForwardSessionEventToSession:612(800ns) ForwardSessionEventToSession:612(593ns) ForwardSessionEventToSession:612(850ns) ForwardSessionEventToSession:612(540ns) ForwardSessionEventToSession:612(1240ns)}}}
2025-11-19T13:59:26.597856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435816059027329:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.597861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435816059027329:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.597438Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435818086603653:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.597466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435818086603571:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.597919Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435818086603652:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.597935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435818086603572:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.598582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435817345038521:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.598602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:26.598609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435817345038521:2049] [node 2] ICP22 created new session: [3:7574435817345038607:2049]
2025-11-19T13:59:26.598615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435817345038607:2049] [node 2] ICS09 handshake done sender: [3:7574435817345038547:2074] self: [3:7574435817345038586:0] peer: [2:7574435816059027268:0] socket: 422 qp: -1
2025-11-19T13:59:26.598623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435817345038607:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:26.598640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:26.598744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435817345038549:2075] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.598750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:26.598754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435817345038549:2075] [node 4] ICP22 created new session: [3:7574435817345038609:2075]
2025-11-19T13:59:26.598759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435817345038609:2075] [node 4] ICS09 handshake done sender: [3:7574435817345038583:2100] self: [3:7574435817345038552:0] peer: [4:7574435818086603651:0] socket: 433 qp: -1
2025-11-19T13:59:26.598762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435817345038609:2075] [node 4] ICS10 traffic start
2025-11-19T13:59:26.598766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:26.599404Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435817345038520:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29963) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435817345038601:2116] held: no
2025-11-19T13:59:26.599471Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435817345038601:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.598064Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435818086603615:2085] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.598070Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435818086603615:2085] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.600015Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435818086603628:2094] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:26.600760Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435818086603615:2085] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:26.600912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435818086603613:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.600924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:26.600928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435818086603613:2069] [node 6] ICP22 created new session: [4:7574435818086603659:2069]
2025-11-19T13:59:26.600932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435818086603659:2069] [node 6] ICS09 handshake done sender: [4:7574435818086603615:2085] self: [4:7574435818086603614:0] peer: [6:7574435816736531017:0] socket: 432 qp: -1
2025-11-19T13:59:26.600937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435818086603659:2069] [node 6] ICS10 traffic start
2025-11-19T13:59:26.600942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.601542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435816059027337:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19579
2025-11-19T13:59:26.602380Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435818086603650:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.602464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435818086603613:2069] [node 6] ICP18 handshake (actor [4:7574435818086603650:2116]) is too old
2025-11-19T13:59:26.596939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435815477983367:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-11-19T13:59:26.597175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435816736530971:2076] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(45201497ns) {Configure:104(45200655ns) {ProcessConfigured:135(44896532ns) {StartInitialHandshake:157(44895390ns) {DropHandshakes:469(696ns) {DropIncomingHandshake:441(78ns) DropOutgoingHandshake:456(172ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:476(735ns) {SwitchToState:178(177ns)}}}}}
2025-11-19T13:59:26.597210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435816736530971:2076] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.599839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816736531020:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61779
2025-11-19T13:59:26.601290Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435816736531012:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:26.601667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435816736530959:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.601676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:26.601680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435816736530959:2067] [node 4] ICP22 created new session: [6:7574435816736531021:2067]
2025-11-19T13:59:26.601685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435816736531021:2067] [node 4] ICS09 handshake done sender: [6:7574435816736531012:2115] self: [6:7574435816736531017:0] peer: [4:7574435818086603614:0] socket: 435 qp: -1
2025-11-19T13:59:26.601689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435816736531021:2067] [node 4] ICS10 traffic start
2025-11-19T13:59:26.601703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:26.597770Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818689891642:2092] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.597783Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435818689891642:2092] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.600399Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435818689891645:2095] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:26.600493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435818689891626:2081] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.600505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:26.600509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435818689891626:2081] [node 5] ICP22 created new session: [1:7574435818689891681:2081]
2025-11-19T13:59:26.600514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435818689891681:2081] [node 5] ICS09 handshake done sender: [1:7574435818689891645:2095] self: [1:7574435818689891628:0] peer: [5:7574435815477983366:0] socket: 439 qp: -1
2025-11-19T13:59:26.600518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435818689891681:2081] [node 5] ICS10 traffic start
2025-11-19T13:59:26.600533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:26.602230Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435818689891590:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(25187505ns) {HandleHandshakeStatus:329(25186692ns) {DropIncomingHandshake:441(10332ns) DropOutgoingHandshake:456(427ns) SwitchToState:178(222ns) ProcessPendingSessionEvents:495(14124017ns) {ForwardSessionEventToSession:612(14112051ns) ForwardSessionEventToSession:612(1466ns) ForwardSessionEventToSession:612(1169ns) ForwardSessionEventToSession:612(735ns) ForwardSessionEventToSession:612(676ns) ForwardSessionEventToSession:612(960ns) ForwardSessionEventToSession:612(595ns) ForwardSessionEventToSession:612(702ns) ForwardSessionEventToSession:612(849ns) ForwardSessionEventToSession:612(1063ns)}}}
2025-11-19T13:59:26.604154Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818689891648:2096] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.604160Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435818689891648:2096] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.604797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435817345038520:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.604812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:26.604817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435817345038520:2048] [node 1] ICP22 created new session: [3:7574435817345038614:2048]
2025-11-19T13:59:26.604823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435817345038614:2048] [node 1] ICS09 handshake done sender: [3:7574435817345038601:2116] self: [3:7574435817345038602:0] peer: [1:7574435818689891630:0] socket: 444 qp: -1
2025-11-19T13:59:26.604827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435817345038614:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:26.604832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:26.605030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435817345038613:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63563
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.605851Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435815477983285:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29963) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435815477983363:2114] held: no
2025-11-19T13:59:26.605876Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435815477983343:2095] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:26.606187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816736531023:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61779
2025-11-19T13:59:26.606371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435815477983286:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.606383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:26.606389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435815477983286:2049] [node 2] ICP22 created new session: [5:7574435815477983369:2049]
2025-11-19T13:59:26.606394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435815477983369:2049] [node 2] ICS09 handshake done sender: [5:7574435815477983343:2095] self: [5:7574435815477983358:0] peer: [2:7574435816059027277:0] socket: 442 qp: -1
2025-11-19T13:59:26.606399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435815477983369:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:26.606405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.607201Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435817345038604:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.607299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435817345038616:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63563
2025-11-19T13:59:26.604074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435816059027328:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.604088Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435816059027328:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.607637Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435818689891648:2096] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.610416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816736531024:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61779
2025-11-19T13:59:26.610453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816736531025:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61779
2025-11-19T13:59:26.610543Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435816736531023:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.610571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435816736530936:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.611354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816736531027:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61779
2025-11-19T13:59:26.611530Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435816736531013:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:26.611633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435816736530934:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.611649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:26.611653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435816736530934:2048] [node 1] ICP22 created new session: [6:7574435816736531028:2048]
2025-11-19T13:59:26.611659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435816736531028:2048] [node 1] ICS09 handshake done sender: [6:7574435816736531013:2116] self: [6:7574435816736531015:0] peer: [1:7574435818689891627:0] socket: 463 qp: -1
2025-11-19T13:59:26.611664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435816736531028:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:26.611670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:26.608878Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435818689891642:2092] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:26.609012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435818689891624:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.609021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:26.609028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435818689891624:2080] [node 6] ICP22 created new session: [1:7574435818689891686:2080]
2025-11-19T13:59:26.609033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435818689891686:2080] [node 6] ICS09 handshake done sender: [1:7574435818689891642:2092] self: [1:7574435818689891627:0] peer: [6:7574435816736531015:0] socket: 438 qp: -1
2025-11-19T13:59:26.609038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435818689891686:2080] [node 6] ICS10 traffic start
2025-11-19T13:59:26.609044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:26.610697Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435817345038600:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.610718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435817345038600:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.611876Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435817345038600:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.612056Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435818086603655:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.612184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435818086603662:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31528
2025-11-19T13:59:26.612101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435817345038554:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.612120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:26.612125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435817345038554:2077] [node 6] ICP22 created new session: [3:7574435817345038619:2077]
2025-11-19T13:59:26.612130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435817345038619:2077] [node 6] ICS09 handshake done sender: [3:7574435817345038600:2115] self: [3:7574435817345038578:0] peer: [6:7574435816736531026:0] socket: 475 qp: -1
2025-11-19T13:59:26.612135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435817345038619:2077] [node 6] ICS10 traffic start
2025-11-19T13:59:26.612141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:26.612618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435818689891623:2079] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.612637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:26.612645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435818689891623:2079] [node 4] ICP22 created new session: [1:7574435818689891688:2079]
2025-11-19T13:59:26.612651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435818689891688:2079] [node 4] ICS09 handshake done sender: [1:7574435818689891648:2096] self: [1:7574435818689891629:0] peer: [4:7574435818086603656:0] socket: 457 qp: -1
2025-11-19T13:59:26.612655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435818689891688:2079] [node 4] ICS10 traffic start
2025-11-19T13:59:26.612665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:26.612945Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435816736531023:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:26.613530Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435815477983317:2074] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.613543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435815477983317:2074] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.613131Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435818086603653:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:26.613631Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435816059027329:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:26.613777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435816059027313:2078] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.613790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:26.613795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435816059027313:2078] [node 6] ICP22 created new session: [2:7574435816059027339:2078]
2025-11-19T13:59:26.613800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435816059027339:2078] [node 6] ICS09 handshake done sender: [2:7574435816059027329:2118] self: [2:7574435816059027325:0] peer: [6:7574435816736531016:0] socket: 423 qp: -1
2025-11-19T13:59:26.613804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435816059027339:2078] [node 6] ICS10 traffic start
2025-11-19T13:59:26.613820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:26.613999Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435817345038613:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.614023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435817345038549:2075] [node 4] ICP18 handshake (actor [3:7574435817345038613:2120]) is too old
2025-11-19T13:59:26.614172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435818086603571:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.614183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:26.614188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435818086603571:2048] [node 1] ICP22 created new session: [4:7574435818086603664:2048]
2025-11-19T13:59:26.614193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435818086603664:2048] [node 1] ICS09 handshake done sender: [4:7574435818086603653:2118] self: [4:7574435818086603656:0] peer: [1:7574435818689891629:0] socket: 465 qp: -1
2025-11-19T13:59:26.614196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435818086603664:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:26.614201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:26.614316Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435816059027337:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.614335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435816059027242:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.619784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435818086603573:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.619817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:26.619827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435818086603573:2050] [node 3] ICP22 created new session: [4:7574435818086603666:2050]
2025-11-19T13:59:26.619833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435818086603666:2050] [node 3] ICS09 handshake done sender: [4:7574435818086603628:2094] self: [4:7574435818086603651:0] peer: [3:7574435817345038552:0] socket: 440 qp: -1
2025-11-19T13:59:26.619842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435818086603666:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:26.619863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.620611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435815477983372:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-11-19T13:59:26.620715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435815477983373:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-11-19T13:59:26.620767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435815477983374:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-11-19T13:59:26.620935Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435815477983363:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:26.621038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435815477983285:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.621051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:26.621057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435815477983285:2048] [node 1] ICP22 created new session: [5:7574435815477983375:2048]
2025-11-19T13:59:26.621064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435815477983375:2048] [node 1] ICS09 handshake done sender: [5:7574435815477983363:2114] self: [5:7574435815477983366:0] peer: [1:7574435818689891628:0] socket: 443 qp: -1
2025-11-19T13:59:26.621068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435815477983375:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:26.621074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:26.627913Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435815477983373:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.628732Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818689891674:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.628744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435818689891674:2118] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.629141Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435815477983317:2074] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:26.629269Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435817345038616:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.628523Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435816059027328:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:26.629017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435816059027290:2087] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.629028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:26.629033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435816059027290:2087] [node 4] ICP22 created new session: [2:7574435816059027343:2087]
2025-11-19T13:59:26.629038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435816059027343:2087] [node 4] ICS09 handshake done sender: [2:7574435816059027328:2117] self: [2:7574435816059027319:0] peer: [4:7574435818086603657:0] socket: 428 qp: -1
2025-11-19T13:59:26.629042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435816059027343:2087] [node 4] ICS10 traffic start
2025-11-19T13:59:26.629048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:26.633342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435816736530936:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.633362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:26.633368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435816736530936:2050] [node 3] ICP22 created new session: [6:7574435816736531031:2050]
2025-11-19T13:59:26.633374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435816736531031:2050] [node 3] ICS09 handshake done sender: [6:7574435816736531023:2120] self: [6:7574435816736531026:0] peer: [3:7574435817345038578:0] socket: 476 qp: -1
2025-11-19T13:59:26.633379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435816736531031:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:26.633385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:26.633441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435816736530936:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:26.633573Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435816736531014:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.633987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435816736531033:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61779
2025-11-19T13:59:26.634386Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435816736531011:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:26.633787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435818086603571:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(19587465ns) {HandleHandshakeStatus:329(19587114ns) {DropIncomingHandshake:441(86ns) DropOutgoingHandshake:456(1086ns) SwitchToState:178(80ns) ProcessPendingSessionEvents:495(19531957ns) {ForwardSessionEventToSession:612(19522831ns) ForwardSessionEventToSession:612(1179ns) ForwardSessionEventToSession:612(626ns) ForwardSessionEventToSession:612(804ns) ForwardSessionEventToSession:612(523ns) ForwardSessionEventToSession:612(924ns) ForwardSessionEventToSession:612(641ns) ForwardSessionEventToSession:612(567ns) ForwardSessionEventToSession:612(729ns) ForwardSessionEventToSession:612(798ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.635952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435815477983379:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-11-19T13:59:26.637172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435815477983287:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.638394Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435815477983379:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.638429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435815477983329:2083] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:26.638723Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435818086603572:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31564) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435818086603652:2117] held: no
2025-11-19T13:59:26.637260Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435817345038587:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.637274Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435817345038587:2102] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.639836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435815477983315:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.639861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:26.639870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435815477983315:2070] [node 6] ICP22 created new session: [5:7574435815477983381:2070]
2025-11-19T13:59:26.639875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435815477983381:2070] [node 6] ICS09 handshake done sender: [5:7574435815477983317:2074] self: [5:7574435815477983316:0] peer: [6:7574435816736531019:0] socket: 425 qp: -1
2025-11-19T13:59:26.639883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435815477983381:2070] [node 6] ICS10 traffic start
2025-11-19T13:59:26.639900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.640316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435818086603669:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31528
2025-11-19T13:59:26.640569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435816059027313:2078] [node 6] ICP35 event processing took too much time PendingConnection:278(26760450ns) {HandleHandshakeStatus:329(26759810ns) {DropIncomingHandshake:441(3438ns) DropOutgoingHandshake:456(87ns) SwitchToState:178(65ns) ProcessPendingSessionEvents:495(9119998ns) {ForwardSessionEventToSession:612(9109292ns) ForwardSessionEventToSession:612(1392ns) ForwardSessionEventToSession:612(831ns) ForwardSessionEventToSession:612(592ns) ForwardSessionEventToSession:612(598ns) ForwardSessionEventToSession:612(944ns) ForwardSessionEventToSession:612(567ns) ForwardSessionEventToSession:612(654ns) ForwardSessionEventToSession:612(791ns) ForwardSessionEventToSession:612(870ns)}}}
2025-11-19T13:59:26.640717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435816736530935:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31564) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435816736531014:2117] held: no
2025-11-19T13:59:26.640740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435816736530935:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.640749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:26.640756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435816736530935:2049] [node 2] ICP22 created new session: [6:7574435816736531034:2049]
2025-11-19T13:59:26.640761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435816736531034:2049] [node 2] ICS09 handshake done sender: [6:7574435816736531014:2117] self: [6:7574435816736531016:0] peer: [2:7574435816059027325:0] socket: 464 qp: -1
2025-11-19T13:59:26.640766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435816736531034:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:26.640771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:26.640907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435816736530971:2076] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.640914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:26.640919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435816736530971:2076] [node 5] ICP22 created new session: [6:7574435816736531036:2076]
2025-11-19T13:59:26.640924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435816736531036:2076] [node 5] ICS09 handshake done sender: [6:7574435816736531011:2114] self: [6:7574435816736531019:0] peer: [5:7574435815477983316:0] socket: 454 qp: -1
2025-11-19T13:59:26.640928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435816736531036:2076] [node 5] ICS10 traffic start
2025-11-19T13:59:26.640935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.642405Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435815477983374:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.642526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435815477983315:2070] [node 6] ICP18 handshake (actor [5:7574435815477983374:2120]) is too old
2025-11-19T13:59:26.642725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435815477983383:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.643024Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435818086603652:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:26.642867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435816059027345:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19579
2025-11-19T13:59:26.643255Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435818689891674:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:26.643218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435818086603572:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.643232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:26.643238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435818086603572:2049] [node 2] ICP22 created new session: [4:7574435818086603670:2049]
2025-11-19T13:59:26.643242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435818086603670:2049] [node 2] ICS09 handshake done sender: [4:7574435818086603652:2117] self: [4:7574435818086603657:0] peer: [2:7574435816059027319:0] socket: 455 qp: -1
2025-11-19T13:59:26.643247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435818086603670:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:26.643263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:26.643298Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435818086603627:2093] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:26.643308Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435818086603627:2093] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:26.643488Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435817345038587:2102] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:26.643431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435818689891589:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.643453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:26.643459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435818689891589:2048] [node 2] ICP22 created new session: [1:7574435818689891691:2048]
2025-11-19T13:59:26.643465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435818689891691:2048] [node 2] ICS09 handshake done sender: [1:7574435818689891674:2118] self: [1:7574435818689891631:0] peer: [2:7574435816059027341:0] socket: 436 qp: -1
2025-11-19T13:59:26.643470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435818689891691:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:26.643481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:26.643804Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435816059027337:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:26.643915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435816059027242:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.643930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:26.643934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435816059027242:2048] [node 1] ICP22 created new session: [2:7574435816059027346:2048]
2025-11-19T13:59:26.643938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435816059027346:2048] [node 1] ICS09 handshake done sender: [2:7574435816059027337:2119] self: [2:7574435816059027341:0] peer: [1:7574435818689891631:0] socket: 471 qp: -1
2025-11-19T13:59:26.643943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435816059027346:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:26.643955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:26.643701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435817345038553:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.643720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:26.643724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435817345038553:2076] [node 5] ICP22 created new session: [3:7574435817345038623:2076]
2025-11-19T13:59:26.643729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435817345038623:2076] [node 5] ICS09 handshake done sender: [3:7574435817345038587:2102] self: [3:7574435817345038555:0] peer: [5:7574435815477983377:0] socket: 474 qp: -1
2025-11-19T13:59:26.643733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435817345038623:2076] [node 5] ICS10 traffic start
2025-11-19T13:59:26.643788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:26.644167Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435815477983373:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:26.644344Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435818086603627:2093] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:26.644229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435815477983287:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.644243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:26.644248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435815477983287:2050] [node 3] ICP22 created new session: [5:7574435815477983384:2050]
2025-11-19T13:59:26.644253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435815477983384:2050] [node 3] ICS09 handshake done sender: [5:7574435815477983373:2119] self: [5:7574435815477983377:0] peer: [3:7574435817345038555:0] socket: 447 qp: -1
2025-11-19T13:59:26.644257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435815477983384:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:26.644267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:26.644338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435815477983386:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-11-19T13:59:26.644802Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435815477983379:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:26.648559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435815477983329:2083] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:26.648577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:26.648583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435815477983329:2083] [node 4] ICP22 created new session: [5:7574435815477983387:2083]
2025-11-19T13:59:26.648588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435815477983387:2083] [node 4] ICS09 handshake done sender: [5:7574435815477983379:2121] self: [5:7574435815477983380:0] peer: [4:7574435818086603595:0] socket: 485 qp: -1
2025-11-19T13:59:26.648593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435815477983387:2083] [node 4] ICS10 traffic start
2025-11-19T13:59:26.648599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:26.651867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435818086603589:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:26.651897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:26.651904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435818086603589:2064] [node 5] ICP22 created new session: [4:7574435818086603673:2064]
2025-11-19T13:59:26.651910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435818086603673:2064] [node 5] ICS09 handshake done sender: [4:7574435818086603627:2093] self: [4:7574435818086603595:0] peer: [5:7574435815477983380:0] socket: 430 qp: -1
2025-11-19T13:59:26.651915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435818086603673:2064] [node 5] ICS10 traffic start
2025-11-19T13:59:26.651921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:26.662438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:26.662531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574435816059027346:2048] [node 1] ICS07 socket disconnect 471 reason# EndOfStream
2025-11-19T13:59:26.662537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435816059027346:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.662566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:26.662581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435816059027346:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:26.663164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:26.663232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435817345038614:2048] [node 1] ICS01 socket: 444 reason# EndOfStream
2025-11-19T13:59:26.663242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435817345038520:2048] [node 1] ICP30 unregister session Session# [3:7574435817345038614:2048] VirtualId# [3:7574435817345038602:0]
2025-11-19T13:59:26.663247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435817345038614:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.663269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:26.663287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:26.670828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435816059027242:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29963) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:26.670848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435816059027346:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:26.671358Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435816059027242:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29963) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:26.671373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435816059027346:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:26.671379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435816059027242:2048] [node 1] ICP30 unregister session Session# [2:7574435816059027346:2048] VirtualId# [2:7574435816059027341:0]
2025-11-19T13:59:26.671402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:26.671411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574435816059027242:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29963) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.671414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29963) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.679837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7574435816059027242:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:26.699858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:26.700049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435816736531028:2048] [node 1] ICS01 socket: 463 reason# EndOfStream
2025-11-19T13:59:26.700061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435816736530934:2048] [node 1] ICP30 unregister session Session# [6:7574435816736531028:2048] VirtualId# [6:7574435816736531015:0]
2025-11-19T13:59:26.700068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435816736531028:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.700116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:26.700141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:26.699909Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:26.700009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435818086603664:2048] [node 1] ICS07 socket disconnect 465 reason# EndOfStream
2025-11-19T13:59:26.700017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435818086603664:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.700067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:26.700087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435818086603664:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:26.700558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435818086603571:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29963) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:26.700567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435818086603664:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:26.700571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435818086603571:2048] [node 1] ICP30 unregister session Session# [4:7574435818086603664:2048] VirtualId# [4:7574435818086603656:0]
2025-11-19T13:59:26.700588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:26.700594Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435818086603571:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29963) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.700597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29963) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:26.703103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:26.703188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435815477983375:2048] [node 1] ICS01 socket: 443 reason# EndOfStream
2025-11-19T13:59:26.703196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435815477983285:2048] [node 1] ICP30 unregister session Session# [5:7574435815477983375:2048] VirtualId# [5:7574435815477983366:0]
2025-11-19T13:59:26.703202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435815477983375:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.703247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:26.703271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:26.710469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435818086603571:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:26.713105Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:26.713202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435817345038607:2049] [node 2] ICS01 socket: 422 reason# EndOfStream
2025-11-19T13:59:26.713209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435817345038521:2049] [node 2] ICP30 unregister session Session# [3:7574435817345038607:2049] VirtualId# [3:7574435817345038586:0]
2025-11-19T13:59:26.713216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435817345038607:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.713250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:26.713270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:26.719873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:26.719967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435816736531034:2049] [node 2] ICS01 socket: 464 reason# EndOfStream
2025-11-19T13:59:26.719974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435816736530935:2049] [node 2] ICP30 unregister session Session# [6:7574435816736531034:2049] VirtualId# [6:7574435816736531016:0]
2025-11-19T13:59:26.719982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435816736531034:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.720028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:26.720052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:26.721748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:26.721820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435815477983369:2049] [node 2] ICS01 socket: 442 reason# EndOfStream
2025-11-19T13:59:26.721828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435815477983286:2049] [node 2] ICP30 unregister session Session# [5:7574435815477983369:2049] VirtualId# [5:7574435815477983358:0]
2025-11-19T13:59:26.721833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435815477983369:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.721872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:26.721892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:26.721920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:26.721976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435818086603670:2049] [node 2] ICS01 socket: 455 reason# EndOfStream
2025-11-19T13:59:26.721980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435818086603572:2049] [node 2] ICP30 unregister session Session# [4:7574435818086603670:2049] VirtualId# [4:7574435818086603657:0]
2025-11-19T13:59:26.721984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435818086603670:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.722004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:26.722022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:26.731952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:26.732065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435816736531031:2050] [node 3] ICS01 socket: 476 reason# EndOfStream
2025-11-19T13:59:26.732073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435816736530936:2050] [node 3] ICP30 unregister session Session# [6:7574435816736531031:2050] VirtualId# [6:7574435816736531026:0]
2025-11-19T13:59:26.732080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435816736531031:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.732123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:26.732146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:26.734491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:26.734564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435815477983384:2050] [node 3] ICS01 socket: 447 reason# EndOfStream
2025-11-19T13:59:26.734570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435815477983287:2050] [node 3] ICP30 unregister session Session# [5:7574435815477983384:2050] VirtualId# [5:7574435815477983377:0]
2025-11-19T13:59:26.734574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435815477983384:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.734616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:26.734637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:26.747466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:26.747573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435816736531021:2067] [node 4] ICS01 socket: 435 reason# EndOfStream
2025-11-19T13:59:26.747585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435816736530959:2067] [node 4] ICP30 unregister session Session# [6:7574435816736531021:2067] VirtualId# [6:7574435816736531017:0]
2025-11-19T13:59:26.747592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435816736531021:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:26.747641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:26.747666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:26.873067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435817270635849:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:26.873081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435817270635850:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:26.943919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435815671247961:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:26.943945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435815671247962:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:26.987849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435815925727964:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:26.987865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435815925727965:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:27.031856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435819883705739:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:27.031882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435819883705740:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:27.031888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435819883705741:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:27.088500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435818948295656:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:27.088515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435818948295657:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:27.088520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435818948295658:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:27.151901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435822444232703:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:27.151917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435822444232704:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:27.151922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435822444232705:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:27.162764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435821565603192:2082] [node 4] ICP01 ready to work
2025-11-19T13:59:27.162866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435821565603193:2087] [node 6] ICP01 ready to work
2025-11-19T13:59:27.162926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435821565603194:2091] [node 5] ICP01 ready to work
2025-11-19T13:59:27.164005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435820220695281:2067] [node 6] ICP01 ready to work
2025-11-19T13:59:27.168019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435818948295673:2060] [node 6] ICP01 ready to work
2025-11-19T13:59:27.168913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435818948295683:2066] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.184028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435815671247961:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21408766ns) {Configure:104(21405871ns) {ProcessConfigured:135(20987745ns) {StartInitialHandshake:157(20986307ns) {DropHandshakes:469(1781ns) {DropIncomingHandshake:441(216ns) DropOutgoingHandshake:456(209ns)} GenerateSessionVirtualId:988(174ns) PrepareNewSessionHandshake:476(2557ns) {SwitchToState:178(1541ns)}}}}}
2025-11-19T13:59:27.184229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435819966215319:2074] [node 4] ICP01 ready to work
2025-11-19T13:59:27.184328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435819966215320:2080] [node 6] ICP01 ready to work
2025-11-19T13:59:27.184398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435819966215321:2085] [node 5] ICP01 ready to work
2025-11-19T13:59:27.185497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435819966215322:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.185802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435821565603192:2082] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(22835545ns) {Configure:104(22834912ns) {ProcessConfigured:135(22483451ns) {StartInitialHandshake:157(22483167ns) {DropHandshakes:469(692ns) {DropIncomingHandshake:441(117ns) DropOutgoingHandshake:456(97ns)} GenerateSessionVirtualId:988(138ns) PrepareNewSessionHandshake:476(1235ns) {SwitchToState:178(30ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.187152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435819883705739:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23236640ns) {Configure:104(23235379ns) {ProcessConfigured:135(22861174ns) {StartInitialHandshake:157(22860778ns) {DropHandshakes:469(438ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(42ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:476(1882ns) {SwitchToState:178(41ns)}}}}}
2025-11-19T13:59:27.187295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435819883705801:2071] [node 5] ICP01 ready to work
2025-11-19T13:59:27.187392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435819883705802:2085] [node 6] ICP01 ready to work
2025-11-19T13:59:27.188480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435819883705803:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45675
2025-11-19T13:59:27.188494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435819883705804:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45675
2025-11-19T13:59:27.188769Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435820220695281:2067] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(24712831ns) {Configure:104(24712226ns) {ProcessConfigured:135(24385230ns) {StartInitialHandshake:157(24384985ns) {DropHandshakes:469(469ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(44ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:476(707ns) {SwitchToState:178(310ns)}}}}}
2025-11-19T13:59:27.188890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435820220695321:2076] [node 4] ICP01 ready to work
2025-11-19T13:59:27.188959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435820220695322:2081] [node 5] ICP01 ready to work
2025-11-19T13:59:27.189945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435820220695323:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18941
2025-11-19T13:59:27.189957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435820220695324:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18941
2025-11-19T13:59:27.189966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435820220695325:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18941
2025-11-19T13:59:27.192863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435822444232762:2061] [node 4] ICP01 ready to work
2025-11-19T13:59:27.192964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435822444232763:2063] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.205317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435822444232768:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14883
2025-11-19T13:59:27.205345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435822444232769:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14883
2025-11-19T13:59:27.205358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435822444232770:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14883
2025-11-19T13:59:27.205799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435821565603224:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20752
2025-11-19T13:59:27.205820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435821565603225:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20752
2025-11-19T13:59:27.205831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435821565603226:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20752
2025-11-19T13:59:27.205844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435821565603227:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20752
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.213796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435818948295734:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31322
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.223680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435819966215337:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
2025-11-19T13:59:27.223708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435819966215338:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
2025-11-19T13:59:27.223725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435819966215339:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
2025-11-19T13:59:27.231950Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435820220695323:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.239701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435819883705819:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45675
2025-11-19T13:59:27.240727Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435821565603227:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.240950Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435821565603216:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.240967Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435821565603216:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.241718Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435821565603216:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:27.242005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435821565603193:2087] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.242025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:27.242032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435821565603193:2087] [node 6] ICP22 created new session: [1:7574435821565603230:2087]
2025-11-19T13:59:27.242040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435821565603230:2087] [node 6] ICS09 handshake done sender: [1:7574435821565603216:2112] self: [1:7574435821565603215:0] peer: [6:7574435822444232785:0] socket: 435 qp: -1
2025-11-19T13:59:27.242048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435821565603230:2087] [node 6] ICS10 traffic start
2025-11-19T13:59:27.242059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:27.243490Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435821565603225:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.240523Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435822444232768:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.240601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435822444232763:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:27.240859Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435822444232769:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.240875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435822444232703:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:27.241632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435822444232786:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14883
2025-11-19T13:59:27.241923Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435822444232769:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:27.242387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435822444232703:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.242395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:27.242400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435822444232703:2048] [node 1] ICP22 created new session: [6:7574435822444232787:2048]
2025-11-19T13:59:27.242405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435822444232787:2048] [node 1] ICS09 handshake done sender: [6:7574435822444232769:2111] self: [6:7574435822444232785:0] peer: [1:7574435821565603215:0] socket: 450 qp: -1
2025-11-19T13:59:27.242409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435822444232787:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:27.242414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:27.244163Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435819966215322:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.244523Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435819966215338:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.244511Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435819883705804:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.244542Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435819883705819:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.244560Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435819883705803:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.244622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435819883705739:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:27.244637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435819883705740:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:27.244590Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435820220695324:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.244614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435815925727965:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:27.244936Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435820220695325:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.243938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435818948295680:2068] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.243952Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435818948295680:2068] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.254854Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435820220695320:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.254865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435820220695320:2105] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:27.255528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435820220695345:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18941
2025-11-19T13:59:27.253085Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435819966215326:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.253109Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435819966215326:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:27.253510Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435819966215324:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.253513Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435819966215324:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:27.254298Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435822444232770:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.252522Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435818948295734:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.254447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435822444232705:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:27.254676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435818948295683:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:27.254709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435818948295735:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31322
2025-11-19T13:59:27.254732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435818948295736:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31322
2025-11-19T13:59:27.255812Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435821565603224:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.255874Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435821565603226:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.255894Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435821565603212:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.255896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435821565603212:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.256645Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435819883705806:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.256655Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435819883705806:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:27.257259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435819883705823:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45675
2025-11-19T13:59:27.257271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435819883705824:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45675
2025-11-19T13:59:27.257468Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435818948295680:2068] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.257629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435818948295673:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.257644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:27.257650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435818948295673:2060] [node 6] ICP22 created new session: [5:7574435818948295740:2060]
2025-11-19T13:59:27.257656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435818948295740:2060] [node 6] ICS09 handshake done sender: [5:7574435818948295680:2068] self: [5:7574435818948295679:0] peer: [6:7574435822444232784:0] socket: 427 qp: -1
2025-11-19T13:59:27.257662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435818948295740:2060] [node 6] ICS10 traffic start
2025-11-19T13:59:27.257671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:27.257821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435818948295739:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31322
2025-11-19T13:59:27.259097Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435820220695320:2105] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:27.259211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435820220695281:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.259224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:27.259229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435820220695281:2067] [node 6] ICP22 created new session: [3:7574435820220695347:2067]
2025-11-19T13:59:27.259234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435820220695347:2067] [node 6] ICS09 handshake done sender: [3:7574435820220695320:2105] self: [3:7574435820220695288:0] peer: [6:7574435822444232790:0] socket: 444 qp: -1
2025-11-19T13:59:27.259239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435820220695347:2067] [node 6] ICS10 traffic start
2025-11-19T13:59:27.259245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:27.259536Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435821565603212:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:27.259621Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435819966215326:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:27.259741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435821565603192:2082] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.259768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:27.259773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435821565603192:2082] [node 4] ICP22 created new session: [1:7574435821565603237:2082]
2025-11-19T13:59:27.259777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435821565603237:2082] [node 4] ICS09 handshake done sender: [1:7574435821565603212:2110] self: [1:7574435821565603195:0] peer: [4:7574435819883705820:0] socket: 433 qp: -1
2025-11-19T13:59:27.259781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435821565603237:2082] [node 4] ICS10 traffic start
2025-11-19T13:59:27.259788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.260920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435822444232791:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14883
2025-11-19T13:59:27.260222Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435819966215324:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:27.261556Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435820220695324:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:27.261792Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435818948295735:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.261846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435818948295656:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.267117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435822444232792:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14883
2025-11-19T13:59:27.267254Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435819966215339:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.267277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435819966215337:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.267288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435815671247962:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.267304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:27.267311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435815671247962:2049] [node 3] ICP22 created new session: [2:7574435819966215344:2049]
2025-11-19T13:59:27.267317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435819966215344:2049] [node 3] ICS09 handshake done sender: [2:7574435819966215326:2110] self: [2:7574435819966215325:0] peer: [3:7574435820220695343:0] socket: 432 qp: -1
2025-11-19T13:59:27.267323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435819966215344:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:27.267332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:27.267467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435819966215319:2074] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.267475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:27.267481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435819966215319:2074] [node 4] ICP22 created new session: [2:7574435819966215346:2074]
2025-11-19T13:59:27.267488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435819966215346:2074] [node 4] ICS09 handshake done sender: [2:7574435819966215324:2109] self: [2:7574435819966215323:0] peer: [4:7574435819883705822:0] socket: 431 qp: -1
2025-11-19T13:59:27.267492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435819966215346:2074] [node 4] ICS10 traffic start
2025-11-19T13:59:27.267496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:27.268524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435815925727965:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.268542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:27.268547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435815925727965:2049] [node 2] ICP22 created new session: [3:7574435820220695350:2049]
2025-11-19T13:59:27.268552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435820220695350:2049] [node 2] ICS09 handshake done sender: [3:7574435820220695324:2107] self: [3:7574435820220695343:0] peer: [2:7574435819966215325:0] socket: 442 qp: -1
2025-11-19T13:59:27.268557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435820220695350:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:27.268562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:27.269647Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435819883705806:2110] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:27.267900Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435821565603218:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.267912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435821565603218:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:27.272129Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435822444232768:2110] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:27.272443Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435818948295736:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.272473Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435818948295734:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:27.272615Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435822444232770:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.272686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435818948295744:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31322
2025-11-19T13:59:27.272711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435818948295683:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.272722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:27.272726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435818948295683:2066] [node 4] ICP22 created new session: [5:7574435818948295745:2066]
2025-11-19T13:59:27.272731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435818948295745:2066] [node 4] ICS09 handshake done sender: [5:7574435818948295734:2115] self: [5:7574435818948295738:0] peer: [4:7574435819883705805:0] socket: 460 qp: -1
2025-11-19T13:59:27.272735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435818948295745:2066] [node 4] ICS10 traffic start
2025-11-19T13:59:27.272741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:27.272828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435818948295673:2060] [node 6] ICP18 handshake (actor [5:7574435818948295736:2117]) is too old
2025-11-19T13:59:27.273391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435822444232763:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.273404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:27.273408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435822444232763:2063] [node 5] ICP22 created new session: [6:7574435822444232793:2063]
2025-11-19T13:59:27.273413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435822444232793:2063] [node 5] ICS09 handshake done sender: [6:7574435822444232768:2110] self: [6:7574435822444232784:0] peer: [5:7574435818948295679:0] socket: 447 qp: -1
2025-11-19T13:59:27.273418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435822444232793:2063] [node 5] ICS10 traffic start
2025-11-19T13:59:27.273423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:27.273545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435822444232705:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.273550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:27.273553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435822444232705:2050] [node 3] ICP22 created new session: [6:7574435822444232795:2050]
2025-11-19T13:59:27.273557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435822444232795:2050] [node 3] ICS09 handshake done sender: [6:7574435822444232770:2112] self: [6:7574435822444232790:0] peer: [3:7574435820220695288:0] socket: 451 qp: -1
2025-11-19T13:59:27.273560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435822444232795:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:27.273565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.276114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435815671247961:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:27.276333Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435821565603218:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:27.276614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435820220695352:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18941
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.276989Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435821565603214:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.276997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435821565603214:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:27.277509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435821565603194:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.277517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:27.277521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435821565603194:2091] [node 5] ICP22 created new session: [1:7574435821565603242:2091]
2025-11-19T13:59:27.277527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435821565603242:2091] [node 5] ICS09 handshake done sender: [1:7574435821565603218:2113] self: [1:7574435821565603217:0] peer: [5:7574435818948295743:0] socket: 457 qp: -1
2025-11-19T13:59:27.277532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435821565603242:2091] [node 5] ICS10 traffic start
2025-11-19T13:59:27.277539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:27.278706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435821565603241:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20752
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.282454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435819966215352:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
2025-11-19T13:59:27.282628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435819883705739:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4177) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435819883705804:2109] held: no
2025-11-19T13:59:27.282657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435819883705801:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.282665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:27.282671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435819883705801:2071] [node 5] ICP22 created new session: [4:7574435819883705828:2071]
2025-11-19T13:59:27.282677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435819883705828:2071] [node 5] ICS09 handshake done sender: [4:7574435819883705806:2110] self: [4:7574435819883705805:0] peer: [5:7574435818948295738:0] socket: 439 qp: -1
2025-11-19T13:59:27.282682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435819883705828:2071] [node 5] ICS10 traffic start
2025-11-19T13:59:27.282688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:27.282857Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435819883705804:2109] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.283010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435822444232797:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14883
2025-11-19T13:59:27.283146Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435819883705803:2108] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.283201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435819883705826:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45675
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.283373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435819883705827:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45675
2025-11-19T13:59:27.283412Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435819883705801:2071] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:27.283420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435819883705739:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.283427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:27.283430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435819883705739:2048] [node 1] ICP22 created new session: [4:7574435819883705830:2048]
2025-11-19T13:59:27.283434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435819883705830:2048] [node 1] ICS09 handshake done sender: [4:7574435819883705804:2109] self: [4:7574435819883705820:0] peer: [1:7574435821565603195:0] socket: 437 qp: -1
2025-11-19T13:59:27.283438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435819883705830:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:27.283442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:27.283505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435819883705740:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.283535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:27.283538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435819883705740:2049] [node 2] ICP22 created new session: [4:7574435819883705834:2049]
2025-11-19T13:59:27.283541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435819883705834:2049] [node 2] ICS09 handshake done sender: [4:7574435819883705803:2108] self: [4:7574435819883705822:0] peer: [2:7574435819966215323:0] socket: 436 qp: -1
2025-11-19T13:59:27.283544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435819883705834:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:27.283549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.284043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435822444232798:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14883
2025-11-19T13:59:27.284205Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435819883705827:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.284228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435819883705741:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:27.284313Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435820220695331:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.284322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435820220695331:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.285717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435819883705837:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45675
2025-11-19T13:59:27.286192Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435820220695331:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:27.286269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435820220695321:2076] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.286285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:27.286289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435820220695321:2076] [node 4] ICP22 created new session: [3:7574435820220695355:2076]
2025-11-19T13:59:27.286295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435820220695355:2076] [node 4] ICS09 handshake done sender: [3:7574435820220695331:2111] self: [3:7574435820220695330:0] peer: [4:7574435819883705836:0] socket: 478 qp: -1
2025-11-19T13:59:27.286299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435820220695355:2076] [node 4] ICS10 traffic start
2025-11-19T13:59:27.286305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:27.286786Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435821565603214:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:27.286963Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435821565603241:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.286974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435817270635849:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.286986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:27.286990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435817270635849:2048] [node 2] ICP22 created new session: [1:7574435821565603245:2048]
2025-11-19T13:59:27.286994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435821565603245:2048] [node 2] ICS09 handshake done sender: [1:7574435821565603214:2111] self: [1:7574435821565603213:0] peer: [2:7574435819966215351:0] socket: 434 qp: -1
2025-11-19T13:59:27.286998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435821565603245:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:27.287003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.288023Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435820220695352:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.288091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435815925727964:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:27.288330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435821565603194:2091] [node 5] ICP18 handshake (actor [1:7574435821565603241:2119]) is too old
2025-11-19T13:59:27.288386Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435821565603220:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.288390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435821565603220:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:27.289284Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435822444232797:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.289310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435822444232704:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:27.290386Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435820220695333:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.290393Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435820220695333:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:27.291167Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435822444232798:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.291191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435822444232762:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:27.286697Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435818948295735:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:27.291846Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435820220695333:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:27.287276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435818948295748:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31322
2025-11-19T13:59:27.287317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435818948295749:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31322
2025-11-19T13:59:27.287340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435818948295656:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.287349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:27.287354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435818948295656:2048] [node 1] ICP22 created new session: [5:7574435818948295750:2048]
2025-11-19T13:59:27.287359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435818948295750:2048] [node 1] ICS09 handshake done sender: [5:7574435818948295735:2116] self: [5:7574435818948295743:0] peer: [1:7574435821565603217:0] socket: 464 qp: -1
2025-11-19T13:59:27.287363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435818948295750:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:27.287369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:27.290201Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435818948295749:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.290219Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435818948295748:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.290259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435818948295658:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:27.290268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435818948295657:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.292266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435818948295754:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31322
2025-11-19T13:59:27.292393Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435818948295749:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:27.292441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435818948295658:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.292453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:27.292457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435818948295658:2050] [node 3] ICP22 created new session: [5:7574435818948295755:2050]
2025-11-19T13:59:27.292462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435818948295755:2050] [node 3] ICS09 handshake done sender: [5:7574435818948295749:2121] self: [5:7574435818948295752:0] peer: [3:7574435820220695332:0] socket: 491 qp: -1
2025-11-19T13:59:27.292466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435818948295755:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:27.292471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:27.293312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435820220695322:2081] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.293328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:27.293332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435820220695322:2081] [node 5] ICP22 created new session: [3:7574435820220695359:2081]
2025-11-19T13:59:27.293336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435820220695359:2081] [node 5] ICS09 handshake done sender: [3:7574435820220695333:2112] self: [3:7574435820220695332:0] peer: [5:7574435818948295752:0] socket: 477 qp: -1
2025-11-19T13:59:27.293341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435820220695359:2081] [node 5] ICS10 traffic start
2025-11-19T13:59:27.293345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:27.293660Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435819966215328:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.293670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435819966215328:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:27.294134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435815671247961:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4177) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435819966215337:2116] held: no
2025-11-19T13:59:27.294182Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435819966215330:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.294185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435819966215330:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:27.294573Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435819966215337:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.294759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435820220695361:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18941
2025-11-19T13:59:27.294846Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435821565603220:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:27.294986Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435820220695352:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:27.295054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435817270635850:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.295067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:27.295071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435817270635850:2049] [node 3] ICP22 created new session: [1:7574435821565603248:2049]
2025-11-19T13:59:27.295075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435821565603248:2049] [node 3] ICS09 handshake done sender: [1:7574435821565603220:2114] self: [1:7574435821565603219:0] peer: [3:7574435820220695357:0] socket: 481 qp: -1
2025-11-19T13:59:27.295080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435821565603248:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:27.295086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:27.295324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435815925727964:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.295334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:27.295339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435815925727964:2048] [node 1] ICP22 created new session: [3:7574435820220695362:2048]
2025-11-19T13:59:27.295342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435820220695362:2048] [node 1] ICS09 handshake done sender: [3:7574435820220695352:2119] self: [3:7574435820220695357:0] peer: [1:7574435821565603219:0] socket: 446 qp: -1
2025-11-19T13:59:27.295346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435820220695362:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:27.295350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:27.296152Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435819883705812:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:27.296161Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435819883705812:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:27.296565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435819883705741:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64841) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435819883705827:2121] held: no
2025-11-19T13:59:27.296590Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435819883705827:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:27.297054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435819883705741:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.297065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:27.297070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435819883705741:2050] [node 3] ICP22 created new session: [4:7574435819883705839:2050]
2025-11-19T13:59:27.297074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435819883705839:2050] [node 3] ICS09 handshake done sender: [4:7574435819883705827:2121] self: [4:7574435819883705836:0] peer: [3:7574435820220695330:0] socket: 455 qp: -1
2025-11-19T13:59:27.297078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435819883705839:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:27.297082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.297309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435822444232801:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14883
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.297563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435819966215353:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42283
2025-11-19T13:59:27.297711Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435819883705812:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:27.297850Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435822444232798:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:27.297917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435822444232762:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.297929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:27.297933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435822444232762:2061] [node 4] ICP22 created new session: [6:7574435822444232802:2061]
2025-11-19T13:59:27.297938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435822444232802:2061] [node 4] ICS09 handshake done sender: [6:7574435822444232798:2122] self: [6:7574435822444232800:0] peer: [4:7574435819883705811:0] socket: 488 qp: -1
2025-11-19T13:59:27.297942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435822444232802:2061] [node 4] ICS10 traffic start
2025-11-19T13:59:27.297947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:27.298907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435819883705802:2085] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.299095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435815671247961:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.299106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:27.299110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435815671247961:2048] [node 1] ICP22 created new session: [2:7574435819966215356:2048]
2025-11-19T13:59:27.299115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435819966215356:2048] [node 1] ICS09 handshake done sender: [2:7574435819966215337:2116] self: [2:7574435819966215351:0] peer: [1:7574435821565603213:0] socket: 449 qp: -1
2025-11-19T13:59:27.299119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435819966215356:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:27.299125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.299449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435818948295757:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31322
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.299534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435822444232804:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14883
2025-11-19T13:59:27.299578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:27.299586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435819883705802:2085] [node 6] ICP22 created new session: [4:7574435819883705841:2085]
2025-11-19T13:59:27.299590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435819883705841:2085] [node 6] ICS09 handshake done sender: [4:7574435819883705812:2113] self: [4:7574435819883705811:0] peer: [6:7574435822444232800:0] socket: 487 qp: -1
2025-11-19T13:59:27.299593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435819883705841:2085] [node 6] ICS10 traffic start
2025-11-19T13:59:27.299598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:27.300041Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435819966215328:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:27.300182Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435822444232797:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:27.300243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435822444232704:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.300254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:27.300258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435822444232704:2049] [node 2] ICP22 created new session: [6:7574435822444232805:2049]
2025-11-19T13:59:27.300262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435822444232805:2049] [node 2] ICS09 handshake done sender: [6:7574435822444232797:2121] self: [6:7574435822444232799:0] peer: [2:7574435819966215327:0] socket: 438 qp: -1
2025-11-19T13:59:27.300267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435822444232805:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:27.300273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:27.301080Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435819966215330:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:27.301220Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435818948295748:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:27.301290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435818948295657:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:27.301301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:27.301305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435818948295657:2049] [node 2] ICP22 created new session: [5:7574435818948295758:2049]
2025-11-19T13:59:27.301309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435818948295758:2049] [node 2] ICS09 handshake done sender: [5:7574435818948295748:2120] self: [5:7574435818948295753:0] peer: [2:7574435819966215329:0] socket: 490 qp: -1
2025-11-19T13:59:27.301313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435818948295758:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:27.301318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:27.302072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435819966215320:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.302088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:27.302092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435819966215320:2080] [node 6] ICP22 created new session: [2:7574435819966215358:2080]
2025-11-19T13:59:27.302097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435819966215358:2080] [node 6] ICS09 handshake done sender: [2:7574435819966215328:2111] self: [2:7574435819966215327:0] peer: [6:7574435822444232799:0] socket: 452 qp: -1
2025-11-19T13:59:27.302101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435819966215358:2080] [node 6] ICS10 traffic start
2025-11-19T13:59:27.302105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:27.302199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435819966215321:2085] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:27.302205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:27.302208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435819966215321:2085] [node 5] ICP22 created new session: [2:7574435819966215360:2085]
2025-11-19T13:59:27.302211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435819966215360:2085] [node 5] ICS09 handshake done sender: [2:7574435819966215330:2112] self: [2:7574435819966215329:0] peer: [5:7574435818948295753:0] socket: 458 qp: -1
2025-11-19T13:59:27.302214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435819966215360:2085] [node 5] ICS10 traffic start
2025-11-19T13:59:27.302219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:27.308167Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:27.308251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435819966215356:2048] [node 1] ICS01 socket: 449 reason# EndOfStream
2025-11-19T13:59:27.308269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435815671247961:2048] [node 1] ICP30 unregister session Session# [2:7574435819966215356:2048] VirtualId# [2:7574435819966215351:0]
2025-11-19T13:59:27.308278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435819966215356:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:27.308331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:27.308349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:27.308679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:27.308753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435820220695362:2048] [node 1] ICS01 socket: 446 reason# EndOfStream
2025-11-19T13:59:27.308780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435815925727964:2048] [node 1] ICP30 unregister session Session# [3:7574435820220695362:2048] VirtualId# [3:7574435820220695357:0]
2025-11-19T13:59:27.308806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435820220695362:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:27.308829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:27.308865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:27.347911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:27.348025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435819883705830:2048] [node 1] ICS01 socket: 437 reason# EndOfStream
2025-11-19T13:59:27.348035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435819883705739:2048] [node 1] ICP30 unregister session Session# [4:7574435819883705830:2048] VirtualId# [4:7574435819883705820:0]
2025-11-19T13:59:27.348044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435819883705830:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:27.348094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:27.348118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:27.348486Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:27.348575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435822444232787:2048] [node 1] ICS01 socket: 450 reason# EndOfStream
2025-11-19T13:59:27.348605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435822444232703:2048] [node 1] ICP30 unregister session Session# [6:7574435822444232787:2048] VirtualId# [6:7574435822444232785:0]
2025-11-19T13:59:27.348633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435822444232787:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:27.348720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:27.348763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:27.348796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:27.348877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435818948295750:2048] [node 1] ICS01 socket: 464 reason# EndOfStream
2025-11-19T13:59:27.348902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435818948295656:2048] [node 1] ICP30 unregister session Session# [5:7574435818948295750:2048] VirtualId# [5:7574435818948295743:0]
2025-11-19T13:59:27.348920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435818948295750:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:27.348964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:27.349027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:27.352295Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:27.352379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435820220695350:2049] [node 2] ICS01 socket: 442 reason# EndOfStream
2025-11-19T13:59:27.352388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435815925727965:2049] [node 2] ICP30 unregister session Session# [3:7574435820220695350:2049] VirtualId# [3:7574435820220695343:0]
2025-11-19T13:59:27.352394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435820220695350:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:27.352423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:27.352441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:27.360643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:27.360733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435818948295758:2049] [node 2] ICS01 socket: 490 reason# EndOfStream
2025-11-19T13:59:27.360742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435818948295657:2049] [node 2] ICP30 unregister session Session# [5:7574435818948295758:2049] VirtualId# [5:7574435818948295753:0]
2025-11-19T13:59:27.360769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435818948295758:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:27.360831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:27.360874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:27.361134Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:27.361220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435822444232805:2049] [node 2] ICS01 socket: 438 reason# EndOfStream
2025-11-19T13:59:27.361246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435822444232704:2049] [node 2] ICP30 unregister session Session# [6:7574435822444232805:2049] VirtualId# [6:7574435822444232799:0]
2025-11-19T13:59:27.361272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435822444232805:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:27.361315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:27.361332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:27.362815Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:27.362889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435819883705834:2049] [node 2] ICS01 socket: 436 reason# EndOfStream
2025-11-19T13:59:27.362895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435819883705740:2049] [node 2] ICP30 unregister session Session# [4:7574435819883705834:2049] VirtualId# [4:7574435819883705822:0]
2025-11-19T13:59:27.362900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435819883705834:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:27.362938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:27.362958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:27.379903Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:27.380013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435822444232795:2050] [node 3] ICS01 socket: 451 reason# EndOfStream
2025-11-19T13:59:27.380024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435822444232705:2050] [node 3] ICP30 unregister session Session# [6:7574435822444232795:2050] VirtualId# [6:7574435822444232790:0]
2025-11-19T13:59:27.380034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435822444232795:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:27.380089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:27.380117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:27.380061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:27.380145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435818948295755:2050] [node 3] ICS01 socket: 491 reason# EndOfStream
2025-11-19T13:59:27.380176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435818948295658:2050] [node 3] ICP30 unregister session Session# [5:7574435818948295755:2050] VirtualId# [5:7574435818948295752:0]
2025-11-19T13:59:27.380189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435818948295755:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:27.380280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:27.380330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:27.392586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:27.392695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435822444232802:2061] [node 4] ICS01 socket: 488 reason# EndOfStream
2025-11-19T13:59:27.392708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435822444232762:2061] [node 4] ICP30 unregister session Session# [6:7574435822444232802:2061] VirtualId# [6:7574435822444232800:0]
2025-11-19T13:59:27.392718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435822444232802:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:27.392779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:27.392806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:27.627842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435818813137301:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:27.627858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435818813137302:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:27.695877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435820606759902:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:27.695897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435820606759903:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:27.765236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435822852325337:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:27.765261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435822852325338:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:27.835906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435822106060031:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:27.835924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435822106060032:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:27.835928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435822106060033:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:27.916949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435821225385087:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:27.916972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435821225385088:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:27.916976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435821225385089:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:27.952247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435822472956601:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:27.952267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435822472956602:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:27.952271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435822472956603:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:27.989884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435818813137362:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.992208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435822106060087:2101] [node 5] ICP01 ready to work
2025-11-19T13:59:27.992672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435822106060086:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:27.992120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435821225385111:2069] [node 6] ICP01 ready to work
2025-11-19T13:59:27.995339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435821225385115:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42350
2025-11-19T13:59:27.996371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435822472956634:2074] [node 4] ICP01 ready to work
2025-11-19T13:59:27.996459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435822472956635:2077] [node 5] ICP01 ready to work
2025-11-19T13:59:27.999963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435821225385124:2075] [node 4] ICP01 ready to work
2025-11-19T13:59:28.000348Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435821225385115:2072] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.000788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435826401027389:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.008290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435823108104668:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:28.008475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435823108104669:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:28.008516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435818813137366:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13330
2025-11-19T13:59:28.008458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435827147292694:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:28.009213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435827147292699:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18527
2025-11-19T13:59:28.009681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435827147292703:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:28.010076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435827147292707:2112] [node 6] ICP01 ready to work
2025-11-19T13:59:28.008863Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435822106060086:2102] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.016214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435824901727264:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:28.016305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435824901727265:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:28.016377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435824901727266:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:28.017251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435824901727267:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31831
2025-11-19T13:59:28.017270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435824901727268:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31831
2025-11-19T13:59:28.017604Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435822472956602:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(21108707ns) {Configure:104(21108077ns) {ProcessConfigured:135(20782154ns) {StartInitialHandshake:157(20781557ns) {DropHandshakes:469(440ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(38ns)} GenerateSessionVirtualId:988(77ns) PrepareNewSessionHandshake:476(676ns) {SwitchToState:178(488ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.032505Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818813137366:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.032582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435823108104677:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13330
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.032715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435822106060031:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.032765Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435822106060031:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(22446764ns) {Configure:104(22446288ns) {ProcessConfigured:135(22192748ns) {StartInitialHandshake:157(22167442ns) {DropHandshakes:469(373ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(339ns) PrepareNewSessionHandshake:476(565ns) {SwitchToState:178(252ns)}} IncomingHandshake:261(18386ns) {DropIncomingHandshake:441(17ns) PrepareNewSessionHandshake:476(139ns) {SwitchToState:178(105ns)} GenerateSessionVirtualId:988(153ns) IssueIncomingHandshakeReply:193(14940ns) {SwitchToState:178(36ns)}}}}}
2025-11-19T13:59:28.033329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826401027405:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
2025-11-19T13:59:28.033340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826401027406:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
2025-11-19T13:59:28.033351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826401027407:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
2025-11-19T13:59:28.033533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435820606759903:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(30147047ns) {Configure:104(23082070ns) {ProcessConfigured:135(22799171ns) {StartInitialHandshake:157(22798793ns) {DropHandshakes:469(463ns) {DropIncomingHandshake:441(82ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(332ns) PrepareNewSessionHandshake:476(569ns) {SwitchToState:178(361ns)}}}}}
2025-11-19T13:59:28.033650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435824901727279:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31831
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.034204Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435821225385089:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(24377435ns) {Configure:104(24377119ns) {ProcessConfigured:135(24162308ns) {StartInitialHandshake:157(24162046ns) {DropHandshakes:469(199ns) {DropIncomingHandshake:441(29ns) DropOutgoingHandshake:456(29ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(1034ns) {SwitchToState:178(86ns)}}}}}
2025-11-19T13:59:28.034422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435825520352459:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42350
2025-11-19T13:59:28.034433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435825520352460:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42350
2025-11-19T13:59:28.034443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435825520352461:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42350
2025-11-19T13:59:28.033801Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435818813137364:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.033808Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435818813137364:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.036349Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435818813137364:2108] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.039430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435826767923975:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15437
2025-11-19T13:59:28.039547Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435826401027389:2105] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(38705229ns) {Configure:104(38704381ns) {ProcessConfigured:135(38435668ns) {StartInitialHandshake:157(38435175ns) {DropHandshakes:469(350ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(78ns) PrepareNewSessionHandshake:476(794ns) {SwitchToState:178(586ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.055370Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435824901727267:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.055493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435824901727280:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31831
2025-11-19T13:59:28.055539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435820606759902:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.056081Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435826767923975:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.056184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435822472956635:2077] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.056164Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435821225385124:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(38736923ns) {Configure:104(38736248ns) {ProcessConfigured:135(38515071ns) {StartInitialHandshake:157(38514529ns) {DropHandshakes:469(284ns) {DropIncomingHandshake:441(30ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:476(1202ns) {SwitchToState:178(709ns)}}}}}
2025-11-19T13:59:28.056280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435821225385124:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.055692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435818813137362:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.055715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:28.055723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435818813137362:2107] [node 4] ICP22 created new session: [1:7574435823108104680:2107]
2025-11-19T13:59:28.055732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435823108104680:2107] [node 4] ICS09 handshake done sender: [1:7574435818813137364:2108] self: [1:7574435818813137363:0] peer: [4:7574435826401027402:0] socket: 432 qp: -1
2025-11-19T13:59:28.055741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435823108104680:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:28.056744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826401027410:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
2025-11-19T13:59:28.056967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:28.055461Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435827147292699:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.055917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435827147292707:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(45792579ns) {Configure:104(45791754ns) {ProcessConfigured:135(45588885ns) {StartInitialHandshake:157(45588005ns) {DropHandshakes:469(374ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(40ns)} GenerateSessionVirtualId:988(50ns) PrepareNewSessionHandshake:476(892ns) {SwitchToState:178(26ns)}}}}}
2025-11-19T13:59:28.056522Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435824901727279:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.056550Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435824901727280:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.060028Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435821225385113:2071] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.060045Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435821225385113:2071] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.063037Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435822106060089:2103] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.063050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435822106060089:2103] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.063208Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435822106060086:2102] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:28.065737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435822106060031:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.065762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:28.065770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435822106060031:2048] [node 1] ICP22 created new session: [4:7574435826401027411:2048]
2025-11-19T13:59:28.065777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435826401027411:2048] [node 1] ICS09 handshake done sender: [4:7574435822106060086:2102] self: [4:7574435826401027402:0] peer: [1:7574435818813137363:0] socket: 433 qp: -1
2025-11-19T13:59:28.065786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435826401027411:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:28.065796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.070610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435826767923977:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15437
2025-11-19T13:59:28.069586Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435823108104667:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.069601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435823108104667:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.070252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435823108104682:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13330
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.072357Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435825520352459:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.072391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435821225385087:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.072522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435823108104683:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13330
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.073075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435824901727282:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31831
2025-11-19T13:59:28.073113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435824901727283:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31831
2025-11-19T13:59:28.073526Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435826401027406:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.073602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435822106060032:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.073711Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435824901727282:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.076166Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435821225385113:2071] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:28.077341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435818813137362:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(21620366ns) {HandleHandshakeStatus:329(21619245ns) {DropIncomingHandshake:441(165ns) DropOutgoingHandshake:456(398ns) SwitchToState:178(59ns) ProcessPendingSessionEvents:495(15356720ns) {ForwardSessionEventToSession:612(15344642ns) ForwardSessionEventToSession:612(1876ns) ForwardSessionEventToSession:612(1030ns) ForwardSessionEventToSession:612(582ns) ForwardSessionEventToSession:612(632ns) ForwardSessionEventToSession:612(868ns) ForwardSessionEventToSession:612(590ns) ForwardSessionEventToSession:612(645ns) ForwardSessionEventToSession:612(568ns) ForwardSessionEventToSession:612(945ns)}}}
2025-11-19T13:59:28.077414Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435823108104667:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:28.078103Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435826401027405:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.078130Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435826401027407:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.078223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435822106060033:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.078876Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435823108104671:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.078886Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435823108104671:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.079220Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435822106060089:2103] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:28.077498Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435825520352460:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.077526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435821225385089:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.080180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435822106060087:2101] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.080198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:28.080205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435822106060087:2101] [node 5] ICP22 created new session: [4:7574435826401027418:2101]
2025-11-19T13:59:28.080211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435826401027418:2101] [node 5] ICS09 handshake done sender: [4:7574435822106060089:2103] self: [4:7574435822106060088:0] peer: [5:7574435825520352465:0] socket: 436 qp: -1
2025-11-19T13:59:28.080219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435826401027418:2101] [node 5] ICS10 traffic start
2025-11-19T13:59:28.080227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.080296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435824901727270:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.080306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435824901727270:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.080910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435826767923978:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15437
2025-11-19T13:59:28.081199Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435826767923975:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:28.081347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435822472956635:2077] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.081355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:28.081360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435822472956635:2077] [node 5] ICP22 created new session: [6:7574435826767923979:2077]
2025-11-19T13:59:28.081364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435826767923979:2077] [node 5] ICS09 handshake done sender: [6:7574435826767923975:2115] self: [6:7574435826767923976:0] peer: [5:7574435821225385112:0] socket: 439 qp: -1
2025-11-19T13:59:28.081371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435826767923979:2077] [node 5] ICS10 traffic start
2025-11-19T13:59:28.081376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:28.081058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435825520352469:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42350
2025-11-19T13:59:28.081300Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435821225385115:2072] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:28.081731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435821225385124:2075] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.081741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:28.081746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435821225385124:2075] [node 4] ICP22 created new session: [5:7574435825520352470:2075]
2025-11-19T13:59:28.081750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435825520352470:2075] [node 4] ICS09 handshake done sender: [5:7574435821225385115:2072] self: [5:7574435825520352465:0] peer: [4:7574435822106060088:0] socket: 437 qp: -1
2025-11-19T13:59:28.081755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435825520352470:2075] [node 4] ICS10 traffic start
2025-11-19T13:59:28.081762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:28.084539Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435827147292706:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.084557Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435827147292706:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.084630Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435827147292698:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.084637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435827147292698:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.085375Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435823108104677:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.085396Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435823108104683:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.085457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435818813137362:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:28.085533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435818813137301:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.085541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:28.085545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435818813137301:2048] [node 2] ICP22 created new session: [1:7574435823108104686:2048]
2025-11-19T13:59:28.085549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435823108104686:2048] [node 2] ICS09 handshake done sender: [1:7574435823108104667:2112] self: [1:7574435818813137369:0] peer: [2:7574435824901727281:0] socket: 441 qp: -1
2025-11-19T13:59:28.085553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435823108104686:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:28.085558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:28.086710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435818813137301:2048] [node 2] ICP18 handshake (actor [1:7574435823108104677:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.087905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435826767923982:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15437
2025-11-19T13:59:28.088329Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435823108104671:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:28.088593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435821225385111:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.088608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:28.088614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435821225385111:2069] [node 6] ICP22 created new session: [5:7574435825520352473:2069]
2025-11-19T13:59:28.088619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435825520352473:2069] [node 6] ICS09 handshake done sender: [5:7574435821225385113:2071] self: [5:7574435821225385112:0] peer: [6:7574435826767923976:0] socket: 435 qp: -1
2025-11-19T13:59:28.088625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435825520352473:2069] [node 6] ICS10 traffic start
2025-11-19T13:59:28.088631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:28.088570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435823108104668:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.088586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:28.088591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435823108104668:2110] [node 5] ICP22 created new session: [1:7574435823108104691:2110]
2025-11-19T13:59:28.088596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435823108104691:2110] [node 5] ICS09 handshake done sender: [1:7574435823108104671:2113] self: [1:7574435823108104670:0] peer: [5:7574435825520352466:0] socket: 443 qp: -1
2025-11-19T13:59:28.088601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435823108104691:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:28.088607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.088898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435825520352475:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42350
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.089575Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435824901727270:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:28.089531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435827147292714:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18527
2025-11-19T13:59:28.089548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435827147292715:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18527
2025-11-19T13:59:28.089575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435827147292716:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18527
2025-11-19T13:59:28.090609Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435827147292706:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:28.091457Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435827147292715:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.091691Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435827147292698:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:28.088754Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435826767923977:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.088863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435822472956601:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.091779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435827147292703:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.091795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:28.091802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435827147292703:2111] [node 4] ICP22 created new session: [3:7574435827147292720:2111]
2025-11-19T13:59:28.091807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435827147292720:2111] [node 4] ICS09 handshake done sender: [3:7574435827147292706:2113] self: [3:7574435827147292705:0] peer: [4:7574435826401027416:0] socket: 446 qp: -1
2025-11-19T13:59:28.091811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435827147292720:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:28.091845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:28.091951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435827147292694:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.091958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:28.091963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435827147292694:2107] [node 5] ICP22 created new session: [3:7574435827147292722:2107]
2025-11-19T13:59:28.091967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435827147292722:2107] [node 5] ICS09 handshake done sender: [3:7574435827147292698:2109] self: [3:7574435827147292697:0] peer: [5:7574435825520352468:0] socket: 445 qp: -1
2025-11-19T13:59:28.091970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435827147292722:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:28.091974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.092476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826401027421:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
2025-11-19T13:59:28.090311Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435825520352459:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:28.090684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435821225385087:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.090693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:28.090699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435821225385087:2048] [node 1] ICP22 created new session: [5:7574435825520352476:2048]
2025-11-19T13:59:28.090704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435825520352476:2048] [node 1] ICS09 handshake done sender: [5:7574435825520352459:2115] self: [5:7574435825520352466:0] peer: [1:7574435823108104670:0] socket: 461 qp: -1
2025-11-19T13:59:28.090708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435825520352476:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:28.090714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.093567Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435824901727268:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.093631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435824901727264:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.093642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:28.093647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435824901727264:2107] [node 4] ICP22 created new session: [2:7574435824901727288:2107]
2025-11-19T13:59:28.093652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435824901727288:2107] [node 4] ICS09 handshake done sender: [2:7574435824901727270:2113] self: [2:7574435824901727269:0] peer: [4:7574435826401027414:0] socket: 450 qp: -1
2025-11-19T13:59:28.093656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435824901727288:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:28.093663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:28.092572Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435826767923982:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.092610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435822472956634:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.092834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435826767923985:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15437
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.095897Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435823108104673:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.095905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435823108104673:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.097570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435823108104693:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13330
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.097763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435821225385089:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24392) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435825520352460:2116] held: no
2025-11-19T13:59:28.097808Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435825520352461:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.097847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435825520352478:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42350
2025-11-19T13:59:28.097862Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435823108104673:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:28.097988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435821225385088:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.097999Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435825520352460:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:28.098082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435821225385089:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.098093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:28.098097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435821225385089:2050] [node 3] ICP22 created new session: [5:7574435825520352480:2050]
2025-11-19T13:59:28.098101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435825520352480:2050] [node 3] ICS09 handshake done sender: [5:7574435825520352460:2116] self: [5:7574435825520352468:0] peer: [3:7574435827147292697:0] socket: 462 qp: -1
2025-11-19T13:59:28.098106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435825520352480:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:28.098112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:28.098003Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435823108104693:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.098013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435823108104669:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.098025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:28.098029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435823108104669:2111] [node 6] ICP22 created new session: [1:7574435823108104695:2111]
2025-11-19T13:59:28.098034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435823108104695:2111] [node 6] ICS09 handshake done sender: [1:7574435823108104673:2114] self: [1:7574435823108104672:0] peer: [6:7574435826767923983:0] socket: 471 qp: -1
2025-11-19T13:59:28.098054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435823108104695:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:28.098061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:28.098527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826401027422:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57889
2025-11-19T13:59:28.098559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435826767923986:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15437
2025-11-19T13:59:28.098643Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435824901727267:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:28.098713Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435827147292716:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.098784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435822852325337:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.099099Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435826767923985:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.099119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435822472956603:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.100579Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435826401027409:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.100587Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435826401027409:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.101710Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435824901727272:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.101717Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435824901727272:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.102163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435820606759902:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.102172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:28.102176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435820606759902:2048] [node 1] ICP22 created new session: [2:7574435824901727292:2048]
2025-11-19T13:59:28.102181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435824901727292:2048] [node 1] ICS09 handshake done sender: [2:7574435824901727267:2111] self: [2:7574435824901727281:0] peer: [1:7574435818813137369:0] socket: 447 qp: -1
2025-11-19T13:59:28.102186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435824901727292:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:28.102191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:28.102251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435820606759902:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:28.103368Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435827147292710:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.103376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435827147292710:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.104388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435822472956601:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4660) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435826767923977:2116] held: no
2025-11-19T13:59:28.104072Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435823108104676:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.104082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435823108104676:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.104526Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435826767923977:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.104617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435825520352482:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42350
2025-11-19T13:59:28.104723Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435823108104676:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.104774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435822472956601:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.104788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:28.104791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435822472956601:2048] [node 1] ICP22 created new session: [6:7574435826767923990:2048]
2025-11-19T13:59:28.104795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435826767923990:2048] [node 1] ICS09 handshake done sender: [6:7574435826767923977:2116] self: [6:7574435826767923983:0] peer: [1:7574435823108104672:0] socket: 472 qp: -1
2025-11-19T13:59:28.104803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435826767923990:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:28.104809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:28.104916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435826767923988:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15437
2025-11-19T13:59:28.104927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435826767923989:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15437
2025-11-19T13:59:28.104808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435818813137302:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.104822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:28.104826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435818813137302:2049] [node 3] ICP22 created new session: [1:7574435823108104699:2049]
2025-11-19T13:59:28.104831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435823108104699:2049] [node 3] ICS09 handshake done sender: [1:7574435823108104676:2115] self: [1:7574435818813137367:0] peer: [3:7574435827147292726:0] socket: 483 qp: -1
2025-11-19T13:59:28.104835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435823108104699:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:28.104842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:28.105257Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435824901727272:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:28.108397Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435825520352461:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:28.108490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435824901727265:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.108511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:28.108518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435824901727265:2108] [node 5] ICP22 created new session: [2:7574435824901727296:2108]
2025-11-19T13:59:28.108523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435824901727296:2108] [node 5] ICS09 handshake done sender: [2:7574435824901727272:2114] self: [2:7574435824901727271:0] peer: [5:7574435825520352479:0] socket: 451 qp: -1
2025-11-19T13:59:28.108528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435824901727296:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:28.108542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:28.108798Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435826767923988:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.108825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435822472956602:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.108965Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435827147292710:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:28.109114Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435826767923985:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:28.109440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435822472956603:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.109454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:28.109460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435822472956603:2050] [node 3] ICP22 created new session: [6:7574435826767923993:2050]
2025-11-19T13:59:28.109465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435826767923993:2050] [node 3] ICS09 handshake done sender: [6:7574435826767923985:2119] self: [6:7574435826767923987:0] peer: [3:7574435827147292708:0] socket: 494 qp: -1
2025-11-19T13:59:28.109470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435826767923993:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:28.109476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:28.108512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435821225385088:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.108527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:28.108532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435821225385088:2049] [node 2] ICP22 created new session: [5:7574435825520352483:2049]
2025-11-19T13:59:28.108537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435825520352483:2049] [node 2] ICS09 handshake done sender: [5:7574435825520352461:2117] self: [5:7574435825520352479:0] peer: [2:7574435824901727271:0] socket: 463 qp: -1
2025-11-19T13:59:28.108544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435825520352483:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:28.108551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:28.110273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435822106060033:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24392) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435826401027405:2115] held: no
2025-11-19T13:59:28.110298Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435826401027406:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:28.110372Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435827147292714:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.111986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435822106060032:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22396) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435826401027406:2116] held: no
2025-11-19T13:59:28.112009Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435826401027405:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:28.112351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435827147292707:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.112362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:28.112367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435827147292707:2112] [node 6] ICP22 created new session: [3:7574435827147292730:2112]
2025-11-19T13:59:28.112372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435827147292730:2112] [node 6] ICS09 handshake done sender: [3:7574435827147292710:2114] self: [3:7574435827147292708:0] peer: [6:7574435826767923987:0] socket: 487 qp: -1
2025-11-19T13:59:28.112388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435827147292730:2112] [node 6] ICS10 traffic start
2025-11-19T13:59:28.112395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:28.112386Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435824901727274:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.112397Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435824901727274:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.113177Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435824901727274:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:28.113250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435824901727266:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.113264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:28.113271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435824901727266:2109] [node 6] ICP22 created new session: [2:7574435824901727299:2109]
2025-11-19T13:59:28.113276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435824901727299:2109] [node 6] ICS09 handshake done sender: [2:7574435824901727274:2115] self: [2:7574435824901727273:0] peer: [6:7574435826767923992:0] socket: 438 qp: -1
2025-11-19T13:59:28.113280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435824901727299:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:28.113285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:28.112888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435822852325337:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4660) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435827147292716:2118] held: no
2025-11-19T13:59:28.114483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435822472956602:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22396) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435826767923988:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.123887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435827147292707:2112] [node 6] ICP18 handshake (actor [3:7574435827147292714:2116]) is too old
2025-11-19T13:59:28.123936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435827147292728:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18527
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.123997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435826767923995:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15437
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.124301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435826767923996:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15437
2025-11-19T13:59:28.124065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435827147292729:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18527
2025-11-19T13:59:28.124555Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435826767923988:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:28.124633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435822472956602:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.124646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:28.124650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435822472956602:2049] [node 2] ICP22 created new session: [6:7574435826767923997:2049]
2025-11-19T13:59:28.124656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435826767923997:2049] [node 2] ICS09 handshake done sender: [6:7574435826767923988:2121] self: [6:7574435826767923992:0] peer: [2:7574435824901727273:0] socket: 496 qp: -1
2025-11-19T13:59:28.124662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435826767923997:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:28.124667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:28.123319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435822106060033:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.123344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:28.123352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435822106060033:2050] [node 3] ICP22 created new session: [4:7574435826401027424:2050]
2025-11-19T13:59:28.123360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435826401027424:2050] [node 3] ICS09 handshake done sender: [4:7574435826401027405:2115] self: [4:7574435826401027416:0] peer: [3:7574435827147292705:0] socket: 455 qp: -1
2025-11-19T13:59:28.123367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435826401027424:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:28.123373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:28.123475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435822106060032:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.123481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:28.123486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435822106060032:2049] [node 2] ICP22 created new session: [4:7574435826401027426:2049]
2025-11-19T13:59:28.123490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435826401027426:2049] [node 2] ICS09 handshake done sender: [4:7574435826401027406:2116] self: [4:7574435826401027414:0] peer: [2:7574435824901727269:0] socket: 456 qp: -1
2025-11-19T13:59:28.123495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435826401027426:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:28.123501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:28.125373Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435827147292729:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.125394Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435827147292716:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:28.125477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435822852325338:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.125579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435822852325337:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.125588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:28.125592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435822852325337:2048] [node 1] ICP22 created new session: [3:7574435827147292733:2048]
2025-11-19T13:59:28.125596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435827147292733:2048] [node 1] ICS09 handshake done sender: [3:7574435827147292716:2118] self: [3:7574435827147292726:0] peer: [1:7574435818813137367:0] socket: 488 qp: -1
2025-11-19T13:59:28.125601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435827147292733:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:28.125606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:28.131846Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435826401027409:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:28.132111Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435824901727278:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.132121Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435824901727278:2116] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.133337Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435826767923982:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:28.133154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435827147292735:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18527
2025-11-19T13:59:28.133469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435826401027389:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.133489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:28.133497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435826401027389:2105] [node 6] ICP22 created new session: [4:7574435826401027428:2105]
2025-11-19T13:59:28.133502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435826401027428:2105] [node 6] ICS09 handshake done sender: [4:7574435826401027409:2118] self: [4:7574435826401027390:0] peer: [6:7574435826767923984:0] socket: 480 qp: -1
2025-11-19T13:59:28.133508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435826401027428:2105] [node 6] ICS10 traffic start
2025-11-19T13:59:28.133514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:28.133843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435822472956634:2074] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.133857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:28.133861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435822472956634:2074] [node 4] ICP22 created new session: [6:7574435826767923999:2074]
2025-11-19T13:59:28.133866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435826767923999:2074] [node 4] ICS09 handshake done sender: [6:7574435826767923982:2118] self: [6:7574435826767923984:0] peer: [4:7574435826401027390:0] socket: 482 qp: -1
2025-11-19T13:59:28.133871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435826767923999:2074] [node 4] ICS10 traffic start
2025-11-19T13:59:28.133876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:28.134996Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435824901727278:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:28.135162Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435827147292729:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:28.135262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435822852325338:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.135276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:28.135280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435822852325338:2049] [node 2] ICP22 created new session: [3:7574435827147292736:2049]
2025-11-19T13:59:28.135285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435827147292736:2049] [node 2] ICS09 handshake done sender: [3:7574435827147292729:2120] self: [3:7574435827147292732:0] peer: [2:7574435824901727262:0] socket: 442 qp: -1
2025-11-19T13:59:28.135290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435827147292736:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:28.135295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:28.136328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435820606759903:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.136361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:28.136367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435820606759903:2049] [node 3] ICP22 created new session: [2:7574435824901727302:2049]
2025-11-19T13:59:28.136372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435824901727302:2049] [node 3] ICS09 handshake done sender: [2:7574435824901727278:2116] self: [2:7574435824901727262:0] peer: [3:7574435827147292732:0] socket: 434 qp: -1
2025-11-19T13:59:28.136376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435824901727302:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:28.136381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:28.142461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:28.142549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435824901727292:2048] [node 1] ICS01 socket: 447 reason# EndOfStream
2025-11-19T13:59:28.142557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435820606759902:2048] [node 1] ICP30 unregister session Session# [2:7574435824901727292:2048] VirtualId# [2:7574435824901727281:0]
2025-11-19T13:59:28.142562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435824901727292:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.142591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:28.142610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:28.142915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:28.142973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435827147292733:2048] [node 1] ICS01 socket: 488 reason# EndOfStream
2025-11-19T13:59:28.142987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435822852325337:2048] [node 1] ICP30 unregister session Session# [3:7574435827147292733:2048] VirtualId# [3:7574435827147292726:0]
2025-11-19T13:59:28.142991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435827147292733:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.143014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:28.143031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:28.156927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:28.157030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435826401027411:2048] [node 1] ICS01 socket: 433 reason# EndOfStream
2025-11-19T13:59:28.157040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435822106060031:2048] [node 1] ICP30 unregister session Session# [4:7574435826401027411:2048] VirtualId# [4:7574435826401027402:0]
2025-11-19T13:59:28.157052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435826401027411:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.157102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:28.157125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:28.157264Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:28.157322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435825520352476:2048] [node 1] ICS01 socket: 461 reason# EndOfStream
2025-11-19T13:59:28.157326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435821225385087:2048] [node 1] ICP30 unregister session Session# [5:7574435825520352476:2048] VirtualId# [5:7574435825520352466:0]
2025-11-19T13:59:28.157330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435825520352476:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.157354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:28.157371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:28.157393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:28.157448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435826767923990:2048] [node 1] ICS01 socket: 472 reason# EndOfStream
2025-11-19T13:59:28.157452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435822472956601:2048] [node 1] ICP30 unregister session Session# [6:7574435826767923990:2048] VirtualId# [6:7574435826767923983:0]
2025-11-19T13:59:28.157456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435826767923990:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.157478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:28.157496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:28.168485Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:28.168592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435827147292736:2049] [node 2] ICS01 socket: 442 reason# EndOfStream
2025-11-19T13:59:28.168601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435822852325338:2049] [node 2] ICP30 unregister session Session# [3:7574435827147292736:2049] VirtualId# [3:7574435827147292732:0]
2025-11-19T13:59:28.168608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435827147292736:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.168639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:28.168660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:28.180011Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:28.180124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435826767923997:2049] [node 2] ICS01 socket: 496 reason# EndOfStream
2025-11-19T13:59:28.180139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435822472956602:2049] [node 2] ICP30 unregister session Session# [6:7574435826767923997:2049] VirtualId# [6:7574435826767923992:0]
2025-11-19T13:59:28.180148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435826767923997:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.180267Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:28.180333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435825520352483:2049] [node 2] ICS07 socket disconnect 463 reason# EndOfStream
2025-11-19T13:59:28.180337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435825520352483:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.180365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:28.180382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435825520352483:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:28.180630Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:28.180692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435826401027426:2049] [node 2] ICS01 socket: 456 reason# EndOfStream
2025-11-19T13:59:28.180699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435822106060032:2049] [node 2] ICP30 unregister session Session# [4:7574435826401027426:2049] VirtualId# [4:7574435826401027414:0]
2025-11-19T13:59:28.180704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435826401027426:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.180730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:28.180750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:28.181062Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435821225385088:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22396) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:28.181075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435825520352483:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:28.181079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435821225385088:2049] [node 2] ICP30 unregister session Session# [5:7574435825520352483:2049] VirtualId# [5:7574435825520352479:0]
2025-11-19T13:59:28.181094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:28.181100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435821225385088:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:28.181103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:28.180192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:28.180220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:28.191811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435821225385088:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:28.221966Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:28.222083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435825520352480:2050] [node 3] ICS01 socket: 462 reason# EndOfStream
2025-11-19T13:59:28.222095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435821225385089:2050] [node 3] ICP30 unregister session Session# [5:7574435825520352480:2050] VirtualId# [5:7574435825520352468:0]
2025-11-19T13:59:28.222104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435825520352480:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.222162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:28.222189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:28.222143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:28.222209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435826767923993:2050] [node 3] ICS01 socket: 494 reason# EndOfStream
2025-11-19T13:59:28.222214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435822472956603:2050] [node 3] ICP30 unregister session Session# [6:7574435826767923993:2050] VirtualId# [6:7574435826767923987:0]
2025-11-19T13:59:28.222220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435826767923993:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.222260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:28.222280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:28.244740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:28.244852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435826767923999:2074] [node 4] ICS01 socket: 482 reason# EndOfStream
2025-11-19T13:59:28.244863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435822472956634:2074] [node 4] ICP30 unregister session Session# [6:7574435826767923999:2074] VirtualId# [6:7574435826767923984:0]
2025-11-19T13:59:28.244871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435826767923999:2074] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.244921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:28.244949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:28.267854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:59:28.267961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435826767923979:2077] [node 5] ICS01 socket: 439 reason# EndOfStream
2025-11-19T13:59:28.267973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435822472956635:2077] [node 5] ICP30 unregister session Session# [6:7574435826767923979:2077] VirtualId# [6:7574435826767923976:0]
2025-11-19T13:59:28.267982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435826767923979:2077] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.268014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:59:28.268038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:28.379865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435824606234963:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:28.379890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435824606234964:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:28.455849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435826972422042:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:28.455865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435826972422043:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:28.520090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435823277348370:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:28.520106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435823277348371:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:28.583922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435826022341887:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:28.583939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435826022341888:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:28.583944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435826022341889:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:28.667849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435826104163490:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:28.667867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435826104163491:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:28.667872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435826104163492:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:28.725298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435825182440895:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:28.725313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435825182440896:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:28.725317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435825182440897:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:28.801871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435824606235024:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.803083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826022341948:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51266
2025-11-19T13:59:28.803378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435824606235032:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:28.816144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435826104163550:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.816873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435826972422106:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:28.816961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435826972422107:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:28.817627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435826972422105:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18694
2025-11-19T13:59:28.820214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435824606235039:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:28.824391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435825182440923:2061] [node 4] ICP01 ready to work
2025-11-19T13:59:28.824459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435825182440922:2059] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.829198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435826972422119:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18694
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.832628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435824606235041:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10296
2025-11-19T13:59:28.835878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435826972422115:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:28.840428Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435824606235041:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.840230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435823277348437:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:28.845869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435825182440897:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(38028449ns) {Configure:104(38026845ns) {ProcessConfigured:135(37659606ns) {StartInitialHandshake:157(37658789ns) {DropHandshakes:469(661ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:476(1434ns) {SwitchToState:178(1103ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.847262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435823277348446:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:28.847353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435823277348447:2112] [node 6] ICP01 ready to work
2025-11-19T13:59:28.847905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435825182440923:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(23404463ns) {Configure:104(23404071ns) {ProcessConfigured:135(23114809ns) {StartInitialHandshake:157(23114621ns) {DropHandshakes:469(526ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:476(752ns) {SwitchToState:178(365ns)}}}}}
2025-11-19T13:59:28.848181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435825182440972:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58955
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.852074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435826022341948:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.860674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435826104163550:2105] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(44443262ns) {Configure:104(44442317ns) {ProcessConfigured:135(44091626ns) {StartInitialHandshake:157(44090752ns) {DropHandshakes:469(585ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(52ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:476(1265ns) {SwitchToState:178(784ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.863867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435826022341889:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(27618319ns) {Configure:104(27617166ns) {ProcessConfigured:135(27223158ns) {StartInitialHandshake:157(27222594ns) {DropHandshakes:469(969ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(172ns) PrepareNewSessionHandshake:476(1357ns) {SwitchToState:178(261ns)}}}}}
2025-11-19T13:59:28.864059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435826022341954:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:28.864800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435826022341887:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.866520Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435826972422105:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.866653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435826972422042:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.866678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435826972422122:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18694
2025-11-19T13:59:28.866692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435826972422123:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18694
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.869064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435823277348443:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34031
2025-11-19T13:59:28.869164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435823277348444:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34031
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.872135Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435823277348447:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(24440070ns) {Configure:104(24439594ns) {ProcessConfigured:135(24239320ns) {StartInitialHandshake:157(24238677ns) {DropHandshakes:469(160ns) {DropIncomingHandshake:441(22ns) DropOutgoingHandshake:456(14ns)} GenerateSessionVirtualId:988(137ns) PrepareNewSessionHandshake:476(1173ns) {SwitchToState:178(674ns)}}}}}
2025-11-19T13:59:28.872280Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435823277348444:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.872314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435823277348371:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.872462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826022341965:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51266
2025-11-19T13:59:28.878625Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435826022341965:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.878007Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435826972422119:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.878047Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435826972422116:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.878050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435826972422116:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.883416Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435826972422116:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:28.883987Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435825182440972:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.884021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435825182440896:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.883669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435826972422043:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.883693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:28.883700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435826972422043:2049] [node 3] ICP22 created new session: [2:7574435826972422127:2049]
2025-11-19T13:59:28.883707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435826972422127:2049] [node 3] ICS09 handshake done sender: [2:7574435826972422116:2113] self: [2:7574435826972422114:0] peer: [3:7574435823277348452:0] socket: 451 qp: -1
2025-11-19T13:59:28.883714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435826972422127:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:28.883723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:28.887162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435823277348445:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34031
2025-11-19T13:59:28.887646Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435823277348439:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.887653Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435823277348439:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.887257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435826022341952:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.887847Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435824606235028:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.887865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435824606235028:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.887909Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435824606235026:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.887915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435824606235026:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.890577Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435826972422109:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.890591Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435826972422109:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.890853Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435823277348445:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.891780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435826104163555:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-19T13:59:28.892894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435824606235044:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10296
2025-11-19T13:59:28.892515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435826972422115:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(55470166ns) {Configure:104(55468457ns) {ProcessConfigured:135(55195494ns) {StartInitialHandshake:157(55195164ns) {DropHandshakes:469(527ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(184ns) PrepareNewSessionHandshake:476(688ns) {SwitchToState:178(341ns)}}}}}
2025-11-19T13:59:28.892597Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435826972422123:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.893169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435824606235045:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10296
2025-11-19T13:59:28.893242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435824606235046:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10296
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.892338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435823277348454:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34031
2025-11-19T13:59:28.892399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435823277348455:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34031
2025-11-19T13:59:28.892431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435823277348456:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34031
2025-11-19T13:59:28.895969Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435823277348454:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.895993Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435823277348455:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.896013Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435823277348443:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.896786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435826022341889:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.896817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435826022341889:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(18141838ns) {IncomingHandshake:261(18141184ns) {DropIncomingHandshake:441(244ns) PrepareNewSessionHandshake:476(893ns) {SwitchToState:178(102ns)} GenerateSessionVirtualId:988(486ns) IssueIncomingHandshakeReply:193(18136748ns) {SwitchToState:178(247ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.897066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435823277348371:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1609) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435823277348444:2114] held: no
2025-11-19T13:59:28.897117Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435823277348444:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:28.897524Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435824606235044:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.897547Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435824606235045:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.897774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435823277348370:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.898673Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435823277348439:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:28.898847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435823277348371:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.898863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:28.898869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435823277348371:2049] [node 2] ICP22 created new session: [3:7574435823277348461:2049]
2025-11-19T13:59:28.898875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435823277348461:2049] [node 2] ICS09 handshake done sender: [3:7574435823277348444:2114] self: [3:7574435823277348452:0] peer: [2:7574435826972422114:0] socket: 457 qp: -1
2025-11-19T13:59:28.898881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435823277348461:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:28.898889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:28.899003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435823277348437:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.899011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:28.899015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435823277348437:2109] [node 4] ICP22 created new session: [3:7574435823277348463:2109]
2025-11-19T13:59:28.899018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435823277348463:2109] [node 4] ICS09 handshake done sender: [3:7574435823277348439:2110] self: [3:7574435823277348438:0] peer: [4:7574435826022341966:0] socket: 453 qp: -1
2025-11-19T13:59:28.899022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435823277348463:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:28.899026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:28.896100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435826104163566:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5131
2025-11-19T13:59:28.897255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435826104163569:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5131
2025-11-19T13:59:28.895862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435825182440975:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58955
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.900083Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435826104163569:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.900117Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435826104163566:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.900158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435826104163491:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.900170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435826104163570:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5131
2025-11-19T13:59:28.900240Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435826972422111:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.900248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435826972422111:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.900809Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435826972422111:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:28.900370Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435825182440975:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.900407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435825182440922:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.900889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435826972422107:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.900901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:28.900907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435826972422107:2109] [node 5] ICP22 created new session: [2:7574435826972422133:2109]
2025-11-19T13:59:28.900912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435826972422133:2109] [node 5] ICS09 handshake done sender: [2:7574435826972422111:2112] self: [2:7574435826972422110:0] peer: [5:7574435826104163572:0] socket: 445 qp: -1
2025-11-19T13:59:28.900917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435826972422133:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:28.900923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.901701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435826104163573:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5131
2025-11-19T13:59:28.901857Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435826104163566:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:28.901998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435826104163491:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.902014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:28.902019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435826104163491:2049] [node 2] ICP22 created new session: [5:7574435826104163574:2049]
2025-11-19T13:59:28.902025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435826104163574:2049] [node 2] ICS09 handshake done sender: [5:7574435826104163566:2114] self: [5:7574435826104163572:0] peer: [2:7574435826972422110:0] socket: 446 qp: -1
2025-11-19T13:59:28.902031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435826104163574:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:28.902046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:28.899458Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435824606235034:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.899468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435824606235034:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.900000Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435824606235028:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:28.904347Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435824606235046:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.905284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435826022341889:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61316) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435826022341965:2115] held: no
2025-11-19T13:59:28.905353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826022341969:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51266
2025-11-19T13:59:28.905823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826022341970:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51266
2025-11-19T13:59:28.905988Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435824606235034:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:28.906121Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435824606235026:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:28.906325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435826104163556:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.906334Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435826104163556:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.906806Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435826104163570:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.906883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435826104163492:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.907354Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435826972422109:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:28.907426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435824606234964:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.907444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:28.907449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435824606234964:2049] [node 3] ICP22 created new session: [1:7574435824606235053:2049]
2025-11-19T13:59:28.907454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435824606235053:2049] [node 3] ICS09 handshake done sender: [1:7574435824606235034:2111] self: [1:7574435824606235033:0] peer: [3:7574435823277348459:0] socket: 441 qp: -1
2025-11-19T13:59:28.907459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435824606235053:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:28.907488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:28.907614Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435826104163556:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:28.907849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435826104163550:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.907867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:28.907872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435826104163550:2105] [node 6] ICP22 created new session: [5:7574435826104163579:2105]
2025-11-19T13:59:28.907877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435826104163579:2105] [node 6] ICS09 handshake done sender: [5:7574435826104163556:2110] self: [5:7574435826104163551:0] peer: [6:7574435825182440976:0] socket: 460 qp: -1
2025-11-19T13:59:28.907882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435826104163579:2105] [node 6] ICS10 traffic start
2025-11-19T13:59:28.907887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:28.911896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435826972422106:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.911913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:28.911919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435826972422106:2108] [node 6] ICP22 created new session: [2:7574435826972422137:2108]
2025-11-19T13:59:28.911924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435826972422137:2108] [node 6] ICS09 handshake done sender: [2:7574435826972422109:2111] self: [2:7574435826972422108:0] peer: [6:7574435825182440974:0] socket: 444 qp: -1
2025-11-19T13:59:28.911930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435826972422137:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:28.911936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:28.912346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435824606234963:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.912366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:28.912372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435824606234963:2048] [node 2] ICP22 created new session: [1:7574435824606235055:2048]
2025-11-19T13:59:28.912379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435824606235055:2048] [node 2] ICS09 handshake done sender: [1:7574435824606235026:2108] self: [1:7574435824606235025:0] peer: [2:7574435826972422124:0] socket: 438 qp: -1
2025-11-19T13:59:28.912384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435824606235055:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:28.912390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:28.912499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435824606235024:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.912510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:28.912514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435824606235024:2107] [node 4] ICP22 created new session: [1:7574435824606235057:2107]
2025-11-19T13:59:28.912518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435824606235057:2107] [node 4] ICS09 handshake done sender: [1:7574435824606235028:2109] self: [1:7574435824606235027:0] peer: [4:7574435826022341961:0] socket: 439 qp: -1
2025-11-19T13:59:28.912522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435824606235057:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:28.912526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.915186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826022341971:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51266
2025-11-19T13:59:28.915259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826022341972:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51266
2025-11-19T13:59:28.914399Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435826972422122:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.916293Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435826972422042:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14376) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435826972422105:2110] held: no
2025-11-19T13:59:28.916557Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435826022341948:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:28.916616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435826972422106:2108] [node 6] ICP18 handshake (actor [2:7574435826972422122:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.917050Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435826022341965:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:28.918044Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435826022341887:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14376) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435826022341948:2108] held: no
2025-11-19T13:59:28.918063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435826022341887:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.918073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:28.918078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435826022341887:2048] [node 1] ICP22 created new session: [4:7574435826022341973:2048]
2025-11-19T13:59:28.918085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435826022341973:2048] [node 1] ICS09 handshake done sender: [4:7574435826022341948:2108] self: [4:7574435826022341961:0] peer: [1:7574435824606235027:0] socket: 440 qp: -1
2025-11-19T13:59:28.918092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435826022341973:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:28.918099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:28.918239Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435826022341971:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.918258Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435826022341972:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.918262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435826022341889:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.918284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:28.918288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435826022341889:2050] [node 3] ICP22 created new session: [4:7574435826022341975:2050]
2025-11-19T13:59:28.918293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435826022341975:2050] [node 3] ICS09 handshake done sender: [4:7574435826022341965:2115] self: [4:7574435826022341966:0] peer: [3:7574435823277348438:0] socket: 459 qp: -1
2025-11-19T13:59:28.918298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435826022341975:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:28.918304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:28.918518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435826022341888:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.916993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435826972422139:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18694
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.918998Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435826972422129:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.919007Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435826972422129:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.919491Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435826972422105:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.916704Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435823277348449:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.916721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435823277348449:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.919655Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435823277348370:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14376) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435823277348443:2113] held: no
2025-11-19T13:59:28.919792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435823277348466:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34031
2025-11-19T13:59:28.917880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435826104163581:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5131
2025-11-19T13:59:28.918836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435826104163582:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5131
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.920626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435825182440922:2059] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2836) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435825182440975:2116] held: no
2025-11-19T13:59:28.921149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435826104163582:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.921176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435826104163555:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.921367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435825182440979:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58955
2025-11-19T13:59:28.921447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435825182440980:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58955
2025-11-19T13:59:28.921759Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435826972422129:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:28.921693Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435823277348443:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:28.921828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435823277348370:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.921836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:28.921841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435823277348370:2048] [node 1] ICP22 created new session: [3:7574435823277348468:2048]
2025-11-19T13:59:28.921846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435823277348468:2048] [node 1] ICS09 handshake done sender: [3:7574435823277348443:2113] self: [3:7574435823277348459:0] peer: [1:7574435824606235033:0] socket: 442 qp: -1
2025-11-19T13:59:28.921850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435823277348468:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:28.921856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:28.923074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435826972422115:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.923093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:28.923098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435826972422115:2107] [node 4] ICP22 created new session: [2:7574435826972422141:2107]
2025-11-19T13:59:28.923103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435826972422141:2107] [node 4] ICS09 handshake done sender: [2:7574435826972422129:2118] self: [2:7574435826972422121:0] peer: [4:7574435826022341978:0] socket: 482 qp: -1
2025-11-19T13:59:28.923108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435826972422141:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:28.923114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:28.923377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435826972422042:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.923400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:28.923404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435826972422042:2048] [node 1] ICP22 created new session: [2:7574435826972422143:2048]
2025-11-19T13:59:28.923408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435826972422143:2048] [node 1] ICS09 handshake done sender: [2:7574435826972422105:2110] self: [2:7574435826972422124:0] peer: [1:7574435824606235025:0] socket: 443 qp: -1
2025-11-19T13:59:28.923411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435826972422143:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:28.923415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:28.924355Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435826022341960:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.924367Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435826022341960:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.927802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435825182440977:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58955
2025-11-19T13:59:28.927839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435825182440978:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58955
2025-11-19T13:59:28.928447Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435824606234964:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(20987425ns) {HandleHandshakeStatus:329(20986801ns) {DropIncomingHandshake:441(4367ns) DropOutgoingHandshake:456(100ns) SwitchToState:178(111ns) ProcessPendingSessionEvents:495(11731191ns) {ForwardSessionEventToSession:612(11721576ns) ForwardSessionEventToSession:612(1095ns) ForwardSessionEventToSession:612(1097ns) ForwardSessionEventToSession:612(538ns) ForwardSessionEventToSession:612(514ns) ForwardSessionEventToSession:612(963ns) ForwardSessionEventToSession:612(607ns) ForwardSessionEventToSession:612(582ns) ForwardSessionEventToSession:612(595ns) ForwardSessionEventToSession:612(888ns)}}}
2025-11-19T13:59:28.928637Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435826104163581:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.928665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435826104163490:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.929621Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435823277348449:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:28.929456Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435826104163550:2105] [node 6] ICP35 event processing took too much time PendingConnection:278(21583723ns) {HandleHandshakeStatus:329(21583119ns) {DropIncomingHandshake:441(3488ns) DropOutgoingHandshake:456(84ns) SwitchToState:178(65ns) ProcessPendingSessionEvents:495(12654521ns) {ForwardSessionEventToSession:612(12643540ns) ForwardSessionEventToSession:612(1230ns) ForwardSessionEventToSession:612(846ns) ForwardSessionEventToSession:612(496ns) ForwardSessionEventToSession:612(520ns) ForwardSessionEventToSession:612(1377ns) ForwardSessionEventToSession:612(687ns) ForwardSessionEventToSession:612(637ns) ForwardSessionEventToSession:612(1164ns) ForwardSessionEventToSession:612(618ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.930701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435823277348446:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.930717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:28.930723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435823277348446:2111] [node 5] ICP22 created new session: [3:7574435823277348471:2111]
2025-11-19T13:59:28.930728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435823277348471:2111] [node 5] ICS09 handshake done sender: [3:7574435823277348449:2116] self: [3:7574435823277348448:0] peer: [5:7574435826104163578:0] socket: 479 qp: -1
2025-11-19T13:59:28.930733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435823277348471:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:28.930738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:28.930424Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435824606235036:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.930435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435824606235036:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.931048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435824606235061:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10296
2025-11-19T13:59:28.931114Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435824606235036:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:28.931189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435824606235032:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.931201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:28.931206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435824606235032:2110] [node 5] ICP22 created new session: [1:7574435824606235063:2110]
2025-11-19T13:59:28.931211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435824606235063:2110] [node 5] ICS09 handshake done sender: [1:7574435824606235036:2112] self: [1:7574435824606235035:0] peer: [5:7574435826104163584:0] socket: 476 qp: -1
2025-11-19T13:59:28.931215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435824606235063:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:28.931220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.932698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435826104163490:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14376) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435826104163581:2119] held: no
2025-11-19T13:59:28.932722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435826104163492:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61316) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435826104163570:2117] held: no
2025-11-19T13:59:28.935285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435825182440896:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1609) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435825182440972:2114] held: no
2025-11-19T13:59:28.935363Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435825182440979:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.935616Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435824606235061:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.935024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826022341981:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51266
2025-11-19T13:59:28.935303Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435826022341888:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1609) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435826022341971:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.948126Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435826022341971:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:28.950859Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435825182440972:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.951102Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435825182440975:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:28.951324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435825182440982:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58955
2025-11-19T13:59:28.951463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435825182440895:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.947951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435826104163585:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5131
2025-11-19T13:59:28.948043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435826104163586:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5131
2025-11-19T13:59:28.948296Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435826104163581:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:28.948340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435826104163490:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.948358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:28.948365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435826104163490:2048] [node 1] ICP22 created new session: [5:7574435826104163587:2048]
2025-11-19T13:59:28.948372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435826104163587:2048] [node 1] ICS09 handshake done sender: [5:7574435826104163581:2119] self: [5:7574435826104163584:0] peer: [1:7574435824606235035:0] socket: 449 qp: -1
2025-11-19T13:59:28.948381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435826104163587:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:28.948389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:28.948669Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435826104163570:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:28.948710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435826104163492:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.948718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:28.948722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435826104163492:2050] [node 3] ICP22 created new session: [5:7574435826104163589:2050]
2025-11-19T13:59:28.948727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435826104163589:2050] [node 3] ICS09 handshake done sender: [5:7574435826104163570:2117] self: [5:7574435826104163578:0] peer: [3:7574435823277348448:0] socket: 480 qp: -1
2025-11-19T13:59:28.948731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435826104163589:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:28.948736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:28.952092Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435826022341960:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:28.953133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435826022341954:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.953152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:28.953159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435826022341954:2110] [node 5] ICP22 created new session: [4:7574435826022341984:2110]
2025-11-19T13:59:28.953164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435826022341984:2110] [node 5] ICS09 handshake done sender: [4:7574435826022341960:2114] self: [4:7574435826022341959:0] peer: [5:7574435826104163583:0] socket: 452 qp: -1
2025-11-19T13:59:28.953170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435826022341984:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:28.953176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.953533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435826022341983:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51266
2025-11-19T13:59:28.955899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435826022341888:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.955915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:28.955921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435826022341888:2049] [node 2] ICP22 created new session: [4:7574435826022341986:2049]
2025-11-19T13:59:28.955926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435826022341986:2049] [node 2] ICS09 handshake done sender: [4:7574435826022341971:2119] self: [4:7574435826022341978:0] peer: [2:7574435826972422121:0] socket: 488 qp: -1
2025-11-19T13:59:28.955932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435826022341986:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:28.955940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:28.956128Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435825182440982:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.956156Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435825182440980:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.957238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435825182440897:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.957254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435825182440923:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:28.957397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435825182440896:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.957408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:28.957414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435825182440896:2049] [node 2] ICP22 created new session: [6:7574435825182440987:2049]
2025-11-19T13:59:28.957419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435825182440987:2049] [node 2] ICS09 handshake done sender: [6:7574435825182440972:2114] self: [6:7574435825182440974:0] peer: [2:7574435826972422108:0] socket: 447 qp: -1
2025-11-19T13:59:28.957424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435825182440987:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:28.957431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:28.957998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435825182440922:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.958008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:28.958012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435825182440922:2059] [node 5] ICP22 created new session: [6:7574435825182440989:2059]
2025-11-19T13:59:28.958017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435825182440989:2059] [node 5] ICS09 handshake done sender: [6:7574435825182440975:2116] self: [6:7574435825182440976:0] peer: [5:7574435826104163551:0] socket: 463 qp: -1
2025-11-19T13:59:28.958021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435825182440989:2059] [node 5] ICS10 traffic start
2025-11-19T13:59:28.958025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:28.957520Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435823277348451:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.957537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435823277348451:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.957928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435826104163555:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17096) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435826104163582:2120] held: no
2025-11-19T13:59:28.958300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435825182440991:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58955
2025-11-19T13:59:28.958442Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435826022341968:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.958450Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435826022341968:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.959265Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435826022341968:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:28.959526Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435826022341983:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.959540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435826022341952:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.959553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:28.959564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435826022341952:2109] [node 6] ICP22 created new session: [4:7574435826022341989:2109]
2025-11-19T13:59:28.959570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435826022341989:2109] [node 6] ICS09 handshake done sender: [4:7574435826022341968:2116] self: [4:7574435826022341967:0] peer: [6:7574435825182440986:0] socket: 467 qp: -1
2025-11-19T13:59:28.959576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435826022341989:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:28.959582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:28.960112Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435823277348451:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:28.959981Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435824606235042:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:28.960221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435823277348447:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.960237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:28.960241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435823277348447:2112] [node 6] ICP22 created new session: [3:7574435823277348474:2112]
2025-11-19T13:59:28.959997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435824606235042:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:28.960267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435823277348474:2112] [node 6] ICS09 handshake done sender: [3:7574435823277348451:2117] self: [3:7574435823277348450:0] peer: [6:7574435825182440985:0] socket: 496 qp: -1
2025-11-19T13:59:28.960274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435823277348474:2112] [node 6] ICS10 traffic start
2025-11-19T13:59:28.960291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.960667Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435825182440982:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:28.960603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435826104163591:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5131
2025-11-19T13:59:28.960756Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435826104163582:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.960870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435825182440897:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.960882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:28.960886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435825182440897:2050] [node 3] ICP22 created new session: [6:7574435825182440993:2050]
2025-11-19T13:59:28.960890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435825182440993:2050] [node 3] ICS09 handshake done sender: [6:7574435825182440982:2121] self: [6:7574435825182440985:0] peer: [3:7574435823277348450:0] socket: 497 qp: -1
2025-11-19T13:59:28.960894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435825182440993:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:28.960899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:28.960934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435826104163555:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.960948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:28.960953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435826104163555:2109] [node 4] ICP22 created new session: [5:7574435826104163592:2109]
2025-11-19T13:59:28.960959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435826104163592:2109] [node 4] ICS09 handshake done sender: [5:7574435826104163582:2120] self: [5:7574435826104163583:0] peer: [4:7574435826022341959:0] socket: 472 qp: -1
2025-11-19T13:59:28.960964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435826104163592:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:28.960969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:28.961973Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435824606235042:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:28.962073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435824606235039:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:28.962089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:28.962095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435824606235039:2113] [node 6] ICP22 created new session: [1:7574435824606235067:2113]
2025-11-19T13:59:28.962099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435824606235067:2113] [node 6] ICS09 handshake done sender: [1:7574435824606235042:2115] self: [1:7574435824606235040:0] peer: [6:7574435825182440984:0] socket: 477 qp: -1
2025-11-19T13:59:28.962105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435824606235067:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:28.962111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:28.963334Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435825182440895:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14376) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435825182440979:2119] held: no
2025-11-19T13:59:28.967849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435826022341952:2109] [node 6] ICP18 handshake (actor [4:7574435826022341983:2122]) is too old
2025-11-19T13:59:28.972669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435825182440992:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58955
2025-11-19T13:59:28.973885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435825182440923:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17096) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435825182440980:2120] held: no
2025-11-19T13:59:28.973942Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435825182440980:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:28.974178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435825182440995:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58955
2025-11-19T13:59:28.974310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435825182440923:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.974330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:28.974335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435825182440923:2061] [node 4] ICP22 created new session: [6:7574435825182440996:2061]
2025-11-19T13:59:28.974342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435825182440996:2061] [node 4] ICS09 handshake done sender: [6:7574435825182440980:2120] self: [6:7574435825182440986:0] peer: [4:7574435826022341967:0] socket: 475 qp: -1
2025-11-19T13:59:28.974349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435825182440996:2061] [node 4] ICS10 traffic start
2025-11-19T13:59:28.974358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:28.974496Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435825182440979:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:28.974716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435825182440895:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:28.974759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:28.974765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435825182440895:2048] [node 1] ICP22 created new session: [6:7574435825182440998:2048]
2025-11-19T13:59:28.974770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435825182440998:2048] [node 1] ICS09 handshake done sender: [6:7574435825182440979:2119] self: [6:7574435825182440984:0] peer: [1:7574435824606235040:0] socket: 494 qp: -1
2025-11-19T13:59:28.974774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435825182440998:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:28.974781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:28.985205Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:28.985294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435826972422143:2048] [node 1] ICS01 socket: 443 reason# EndOfStream
2025-11-19T13:59:28.985309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435826972422042:2048] [node 1] ICP30 unregister session Session# [2:7574435826972422143:2048] VirtualId# [2:7574435826972422124:0]
2025-11-19T13:59:28.985316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435826972422143:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.985349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:28.985367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:28.985726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:28.985787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435823277348468:2048] [node 1] ICS01 socket: 442 reason# EndOfStream
2025-11-19T13:59:28.985792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435823277348370:2048] [node 1] ICP30 unregister session Session# [3:7574435823277348468:2048] VirtualId# [3:7574435823277348459:0]
2025-11-19T13:59:28.985796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435823277348468:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:28.985820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:28.985834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:29.003525Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:29.003645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435826022341973:2048] [node 1] ICS01 socket: 440 reason# EndOfStream
2025-11-19T13:59:29.003656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435826022341887:2048] [node 1] ICP30 unregister session Session# [4:7574435826022341973:2048] VirtualId# [4:7574435826022341961:0]
2025-11-19T13:59:29.003665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435826022341973:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.003732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:29.008174Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:29.008279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435826104163587:2048] [node 1] ICS01 socket: 449 reason# EndOfStream
2025-11-19T13:59:29.008289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435826104163490:2048] [node 1] ICP30 unregister session Session# [5:7574435826104163587:2048] VirtualId# [5:7574435826104163584:0]
2025-11-19T13:59:29.008297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435826104163587:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.008213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:29.008349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:29.008376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:29.008410Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:29.008481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435825182440998:2048] [node 1] ICS01 socket: 494 reason# EndOfStream
2025-11-19T13:59:29.008486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435825182440895:2048] [node 1] ICP30 unregister session Session# [6:7574435825182440998:2048] VirtualId# [6:7574435825182440984:0]
2025-11-19T13:59:29.008490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435825182440998:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.008518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:29.008535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:29.011852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:29.011929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435823277348461:2049] [node 2] ICS01 socket: 457 reason# EndOfStream
2025-11-19T13:59:29.011937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435823277348371:2049] [node 2] ICP30 unregister session Session# [3:7574435823277348461:2049] VirtualId# [3:7574435823277348452:0]
2025-11-19T13:59:29.011943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435823277348461:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.011972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:29.011988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:29.022881Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:29.022510Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:29.022611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435826022341986:2049] [node 2] ICS01 socket: 488 reason# EndOfStream
2025-11-19T13:59:29.022620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435826022341888:2049] [node 2] ICP30 unregister session Session# [4:7574435826022341986:2049] VirtualId# [4:7574435826022341978:0]
2025-11-19T13:59:29.022628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435826022341986:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.022678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:29.022704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:29.022735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:29.022806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435825182440987:2049] [node 2] ICS01 socket: 447 reason# EndOfStream
2025-11-19T13:59:29.022810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435825182440896:2049] [node 2] ICP30 unregister session Session# [6:7574435825182440987:2049] VirtualId# [6:7574435825182440974:0]
2025-11-19T13:59:29.022815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435825182440987:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.022839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:29.022858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:29.022941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435826104163574:2049] [node 2] ICS01 socket: 446 reason# EndOfStream
2025-11-19T13:59:29.022948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435826104163491:2049] [node 2] ICP30 unregister session Session# [5:7574435826104163574:2049] VirtualId# [5:7574435826104163572:0]
2025-11-19T13:59:29.022952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435826104163574:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.022974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:29.022994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:29.056800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:29.056916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435826104163589:2050] [node 3] ICS01 socket: 480 reason# EndOfStream
2025-11-19T13:59:29.056927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435826104163492:2050] [node 3] ICP30 unregister session Session# [5:7574435826104163589:2050] VirtualId# [5:7574435826104163578:0]
2025-11-19T13:59:29.056935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435826104163589:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.056986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:29.057012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:29.057057Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:29.057122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435825182440993:2050] [node 3] ICS01 socket: 497 reason# EndOfStream
2025-11-19T13:59:29.057127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435825182440897:2050] [node 3] ICP30 unregister session Session# [6:7574435825182440993:2050] VirtualId# [6:7574435825182440985:0]
2025-11-19T13:59:29.057132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435825182440993:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.057164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:29.057187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:29.068559Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:29.068652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435825182440996:2061] [node 4] ICS01 socket: 475 reason# EndOfStream
2025-11-19T13:59:29.068661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435825182440923:2061] [node 4] ICP30 unregister session Session# [6:7574435825182440996:2061] VirtualId# [6:7574435825182440986:0]
2025-11-19T13:59:29.068668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435825182440996:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.068714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:29.068734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:29.156248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435827792122408:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:29.156261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435827792122409:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:29.198973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435830375944616:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:29.198991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435830375944617:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:29.212066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435828791313552:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:29.212081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435828791313553:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:29.275918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435831002635959:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:29.275933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435831002635960:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:29.275938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435831002635961:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:29.351861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435830105538065:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:29.351882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435830105538066:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:29.351887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435830105538067:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:29.431999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435830703792942:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:29.432023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435830703792943:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:29.432027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435830703792944:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:29.480123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435830703792960:2060] [node 5] ICP01 ready to work
2025-11-19T13:59:29.480205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435830703792961:2062] [node 4] ICP01 ready to work
2025-11-19T13:59:29.480512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435830105538096:2074] [node 6] ICP01 ready to work
2025-11-19T13:59:29.480124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435827792122469:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:29.488110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435827792122475:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:29.493685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435828791313614:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:29.493799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435828791313615:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:29.491996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435830105538120:2082] [node 4] ICP01 ready to work
2025-11-19T13:59:29.495270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435831002636024:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.496498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435831002636033:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:29.496879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435831002636031:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9042
2025-11-19T13:59:29.496897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435831002636032:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9042
2025-11-19T13:59:29.496189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435828791313622:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19583
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.511933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435827792122409:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(27988737ns) {Configure:104(27987897ns) {ProcessConfigured:135(27723915ns) {StartInitialHandshake:157(27722142ns) {DropHandshakes:469(625ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(91ns)} GenerateSessionVirtualId:988(279ns) PrepareNewSessionHandshake:476(1761ns) {SwitchToState:178(484ns)}}}}}
2025-11-19T13:59:29.513187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435827792122481:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3417
2025-11-19T13:59:29.516332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435828791313628:2112] [node 6] ICP01 ready to work
2025-11-19T13:59:29.515919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435827792122477:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:29.516816Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435828791313622:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.517783Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435831002636032:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.517813Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435831002636031:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.524215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435830105538143:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49507
2025-11-19T13:59:29.524284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435830105538144:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49507
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.524405Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435830703792961:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(40596486ns) {Configure:104(37367317ns) {ProcessConfigured:135(37084458ns) {StartInitialHandshake:157(37082946ns) {DropHandshakes:469(515ns) {DropIncomingHandshake:441(113ns) DropOutgoingHandshake:456(49ns)} GenerateSessionVirtualId:988(275ns) PrepareNewSessionHandshake:476(1615ns) {SwitchToState:178(902ns)}}}}}
2025-11-19T13:59:29.524622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435830703793014:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47120
2025-11-19T13:59:29.524657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435830703793015:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47120
2025-11-19T13:59:29.524856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435830703792942:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(37489479ns) {Configure:104(37489011ns) {ProcessConfigured:135(37280267ns) {StartInitialHandshake:157(37279950ns) {DropHandshakes:469(171ns) {DropIncomingHandshake:441(27ns) DropOutgoingHandshake:456(29ns)} GenerateSessionVirtualId:988(138ns) PrepareNewSessionHandshake:476(576ns) {SwitchToState:178(213ns)}}}}}
2025-11-19T13:59:29.525046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435830703792943:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(37423723ns) {Configure:104(37423353ns) {ProcessConfigured:135(37211164ns) {StartInitialHandshake:157(37210927ns) {DropHandshakes:469(160ns) {DropIncomingHandshake:441(24ns) DropOutgoingHandshake:456(21ns)} GenerateSessionVirtualId:988(128ns) PrepareNewSessionHandshake:476(1117ns) {SwitchToState:178(341ns)}}}}}
2025-11-19T13:59:29.525272Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435830375944616:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(36663039ns) {Configure:104(36662107ns) {ProcessConfigured:135(36370179ns) {StartInitialHandshake:157(36369598ns) {DropHandshakes:469(209ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(45ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(2278ns) {SwitchToState:178(959ns)}}}}}
2025-11-19T13:59:29.525493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435830375944683:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:29.525586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435830375944684:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:29.525163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435827792122408:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(36967924ns) {Configure:104(36967100ns) {ProcessConfigured:135(36659972ns) {StartInitialHandshake:157(36659469ns) {DropHandshakes:469(1199ns) {DropIncomingHandshake:441(119ns) DropOutgoingHandshake:456(140ns)} GenerateSessionVirtualId:988(380ns) PrepareNewSessionHandshake:476(468ns) {SwitchToState:178(212ns)}}}}}
2025-11-19T13:59:29.525320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435827792122485:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3417
2025-11-19T13:59:29.525330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435827792122486:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3417
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.528277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435831002636040:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9042
2025-11-19T13:59:29.528523Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435830105538143:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.528568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435830105538065:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:29.529592Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435830105538144:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.529634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435830105538067:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:29.529898Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435830703793014:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.529148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435830375944678:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:29.536527Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435830703793015:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.536580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435830703792961:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.539855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435830703792960:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(58420833ns) {Configure:104(58420056ns) {ProcessConfigured:135(58153855ns) {StartInitialHandshake:157(58152905ns) {DropHandshakes:469(481ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(899ns) {SwitchToState:178(382ns)}}}}}
2025-11-19T13:59:29.539905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435830703792960:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.540834Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435830105538116:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.540845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435830105538116:2094] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.541527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435830105538147:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49507
2025-11-19T13:59:29.541651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435827792122475:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(43469352ns) {Configure:104(43468551ns) {ProcessConfigured:135(43163908ns) {StartInitialHandshake:157(43163171ns) {DropHandshakes:469(505ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(43ns)} GenerateSessionVirtualId:988(159ns) PrepareNewSessionHandshake:476(769ns) {SwitchToState:178(383ns)}}}}}
2025-11-19T13:59:29.541779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435827792122489:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3417
2025-11-19T13:59:29.540263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435830375944690:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56147
2025-11-19T13:59:29.540310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435830375944691:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56147
2025-11-19T13:59:29.541943Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435830375944690:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.543102Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435830375944691:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.544135Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435831002635960:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(48244211ns) {Configure:104(48242821ns) {ProcessConfigured:135(48048975ns) {StartInitialHandshake:157(48048450ns) {DropHandshakes:469(155ns) {DropIncomingHandshake:441(23ns) DropOutgoingHandshake:456(19ns)} GenerateSessionVirtualId:988(281ns) PrepareNewSessionHandshake:476(1215ns) {SwitchToState:178(331ns)}}}}}
2025-11-19T13:59:29.544341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435828791313628:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27954759ns) {Configure:104(27954130ns) {ProcessConfigured:135(27661509ns) {StartInitialHandshake:157(27661153ns) {DropHandshakes:469(763ns) {DropIncomingHandshake:441(113ns) DropOutgoingHandshake:456(105ns)} GenerateSessionVirtualId:988(170ns) PrepareNewSessionHandshake:476(796ns) {SwitchToState:178(221ns)}}}}}
2025-11-19T13:59:29.544381Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435828791313619:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.544385Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435828791313619:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.544236Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435827792122477:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(28252860ns) {Configure:104(28252006ns) {ProcessConfigured:135(27836584ns) {StartInitialHandshake:157(27836340ns) {DropHandshakes:469(1519ns) {DropIncomingHandshake:441(385ns) DropOutgoingHandshake:456(425ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:476(533ns) {SwitchToState:178(175ns)}}}}}
2025-11-19T13:59:29.543849Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435830105538116:2094] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.545764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435830105538096:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.545782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:29.545789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435830105538096:2074] [node 6] ICP22 created new session: [5:7574435830105538150:2074]
2025-11-19T13:59:29.545795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435830105538150:2074] [node 6] ICS09 handshake done sender: [5:7574435830105538116:2094] self: [5:7574435830105538099:0] peer: [6:7574435830703793023:0] socket: 446 qp: -1
2025-11-19T13:59:29.545802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435830105538150:2074] [node 6] ICS10 traffic start
2025-11-19T13:59:29.545820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.545578Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435827792122484:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.545584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435827792122484:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.546132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435827792122492:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3417
2025-11-19T13:59:29.546221Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435827792122484:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:29.545421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435830375944616:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:29.548697Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435831002636026:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.548708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435831002636026:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.548861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435830105538149:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49507
2025-11-19T13:59:29.547516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435827792122408:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.547535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:29.547540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435827792122408:2048] [node 2] ICP22 created new session: [1:7574435827792122493:2048]
2025-11-19T13:59:29.547546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435827792122493:2048] [node 2] ICS09 handshake done sender: [1:7574435827792122484:2113] self: [1:7574435827792122478:0] peer: [2:7574435830375944694:0] socket: 470 qp: -1
2025-11-19T13:59:29.547552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435827792122493:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:29.547561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:29.548771Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435827792122492:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.549041Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435827792122481:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.548589Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435828791313617:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.551834Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435827792122471:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.551852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435827792122471:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.551993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435831002635961:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:29.552027Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435831002635961:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(34169068ns) {IncomingHandshake:261(34168486ns) {DropIncomingHandshake:441(185ns) PrepareNewSessionHandshake:476(1205ns) {SwitchToState:178(675ns)} GenerateSessionVirtualId:988(196ns) IssueIncomingHandshakeReply:193(34161962ns) {SwitchToState:178(1010ns)}}}
2025-11-19T13:59:29.548604Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435828791313617:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.551904Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435828791313619:2110] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:29.555837Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435830105538147:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.555861Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435830105538144:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:29.556074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435830105538067:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.556090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:29.556096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435830105538067:2050] [node 3] ICP22 created new session: [5:7574435830105538153:2050]
2025-11-19T13:59:29.556103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435830105538153:2050] [node 3] ICS09 handshake done sender: [5:7574435830105538144:2116] self: [5:7574435830105538146:0] peer: [3:7574435828791313618:0] socket: 462 qp: -1
2025-11-19T13:59:29.556109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435830105538153:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:29.556128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:29.555896Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435827792122489:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.555921Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435827792122485:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.555941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435827792122408:2048] [node 2] ICP18 handshake (actor [1:7574435827792122489:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.556316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435831002636042:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9042
2025-11-19T13:59:29.556970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435828791313615:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.556989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:29.556996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435828791313615:2108] [node 5] ICP22 created new session: [3:7574435828791313634:2108]
2025-11-19T13:59:29.557002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435828791313634:2108] [node 5] ICS09 handshake done sender: [3:7574435828791313619:2110] self: [3:7574435828791313618:0] peer: [5:7574435830105538146:0] socket: 450 qp: -1
2025-11-19T13:59:29.557008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435828791313634:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:29.557016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:29.557783Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435831002636042:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.557812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435831002635960:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:29.555902Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435830375944617:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(71402460ns) {Configure:104(71401578ns) {ProcessConfigured:135(71075164ns) {StartInitialHandshake:157(71074298ns) {DropHandshakes:469(369ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(44ns)} GenerateSessionVirtualId:988(137ns) PrepareNewSessionHandshake:476(799ns) {SwitchToState:178(585ns)}}}}}
2025-11-19T13:59:29.555930Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435830375944678:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(26644474ns) {Configure:104(26643430ns) {ProcessConfigured:135(26352300ns) {StartInitialHandshake:157(26351477ns) {DropHandshakes:469(959ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(153ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(14017ns) {SwitchToState:178(930ns)}}}}}
2025-11-19T13:59:29.556412Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435830375944616:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22797) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435830375944691:2113] held: no
2025-11-19T13:59:29.557670Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435830375944684:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(31586181ns) {Configure:104(31585839ns) {ProcessConfigured:135(31428037ns) {StartInitialHandshake:157(31427554ns) {DropHandshakes:469(96ns) {DropIncomingHandshake:441(13ns) DropOutgoingHandshake:456(18ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:476(726ns) {SwitchToState:178(370ns)}}}}}
2025-11-19T13:59:29.557910Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435830375944695:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.557913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435830375944695:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.558494Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435830375944695:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:29.559956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435830105538066:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.562237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435830703793024:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47120
2025-11-19T13:59:29.562282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435830703793025:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47120
2025-11-19T13:59:29.562313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435830703793026:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47120
2025-11-19T13:59:29.562003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435830105538096:2074] [node 6] ICP35 event processing took too much time PendingConnection:278(16213971ns) {HandleHandshakeStatus:329(16213153ns) {DropIncomingHandshake:441(28ns) DropOutgoingHandshake:456(414ns) SwitchToState:178(216ns) ProcessPendingSessionEvents:495(13031024ns) {ForwardSessionEventToSession:612(13018537ns) ForwardSessionEventToSession:612(1683ns) ForwardSessionEventToSession:612(788ns) ForwardSessionEventToSession:612(497ns) ForwardSessionEventToSession:612(994ns) ForwardSessionEventToSession:612(1050ns) ForwardSessionEventToSession:612(848ns) ForwardSessionEventToSession:612(860ns) ForwardSessionEventToSession:612(493ns) ForwardSessionEventToSession:612(1390ns)}}}
2025-11-19T13:59:29.562576Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435830703793014:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.562704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435830105538156:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49507
2025-11-19T13:59:29.562848Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435830703793025:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.562873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435830703792942:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:29.562984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435830703792960:2060] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.563000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:29.563006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435830703792960:2060] [node 5] ICP22 created new session: [6:7574435830703793028:2060]
2025-11-19T13:59:29.563011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435830703793028:2060] [node 5] ICS09 handshake done sender: [6:7574435830703793014:2112] self: [6:7574435830703793023:0] peer: [5:7574435830105538099:0] socket: 463 qp: -1
2025-11-19T13:59:29.563016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435830703793028:2060] [node 5] ICS10 traffic start
2025-11-19T13:59:29.563032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:29.563706Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435831002636026:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:29.563610Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435827792122488:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.563620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435827792122488:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.560924Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435830375944686:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.560934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435830375944686:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.564007Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435828791313617:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:29.563890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435831002636024:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.563907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:29.563913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435831002636024:2109] [node 6] ICP22 created new session: [4:7574435831002636045:2109]
2025-11-19T13:59:29.563919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435831002636045:2109] [node 6] ICS09 handshake done sender: [4:7574435831002636026:2110] self: [4:7574435831002636025:0] peer: [6:7574435830703793021:0] socket: 455 qp: -1
2025-11-19T13:59:29.563926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435831002636045:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:29.563942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:29.564131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435828791313614:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.564145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:29.564149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435828791313614:2107] [node 4] ICP22 created new session: [3:7574435828791313638:2107]
2025-11-19T13:59:29.564153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435828791313638:2107] [node 4] ICS09 handshake done sender: [3:7574435828791313617:2109] self: [3:7574435828791313616:0] peer: [4:7574435831002636038:0] socket: 449 qp: -1
2025-11-19T13:59:29.564157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435828791313638:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:29.564161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:29.564690Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435827792122471:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:29.564880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435827792122469:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.564896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:29.564902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435827792122469:2107] [node 5] ICP22 created new session: [1:7574435827792122500:2107]
2025-11-19T13:59:29.564907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435827792122500:2107] [node 5] ICS09 handshake done sender: [1:7574435827792122471:2108] self: [1:7574435827792122470:0] peer: [5:7574435830105538145:0] socket: 444 qp: -1
2025-11-19T13:59:29.564911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435827792122500:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:29.564926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:29.565148Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435830703793015:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.567093Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435830105538065:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22797) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435830105538143:2115] held: no
2025-11-19T13:59:29.567131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435830105538157:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49507
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.567458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435830105538158:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49507
2025-11-19T13:59:29.567489Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435830105538147:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:29.564587Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435830375944686:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:29.567183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435830375944678:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.567192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:29.567197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435830375944678:2107] [node 4] ICP22 created new session: [2:7574435830375944705:2107]
2025-11-19T13:59:29.567202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435830375944705:2107] [node 4] ICS09 handshake done sender: [2:7574435830375944695:2114] self: [2:7574435830375944689:0] peer: [4:7574435831002636043:0] socket: 486 qp: -1
2025-11-19T13:59:29.567206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435830375944705:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:29.567211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:29.567354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435830375944702:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56147
2025-11-19T13:59:29.567612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435830105538066:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.567625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:29.567629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435830105538066:2049] [node 2] ICP22 created new session: [5:7574435830105538159:2049]
2025-11-19T13:59:29.567634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435830105538159:2049] [node 2] ICS09 handshake done sender: [5:7574435830105538147:2117] self: [5:7574435830105538155:0] peer: [2:7574435830375944685:0] socket: 475 qp: -1
2025-11-19T13:59:29.567639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435830105538159:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:29.567652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:29.567727Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435830105538143:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.570043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435830375944703:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56147
2025-11-19T13:59:29.569076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435831002636048:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9042
2025-11-19T13:59:29.569911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435831002636049:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9042
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.572149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435830375944683:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.572169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:29.572176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435830375944683:2108] [node 5] ICP22 created new session: [2:7574435830375944708:2108]
2025-11-19T13:59:29.572181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435830375944708:2108] [node 5] ICS09 handshake done sender: [2:7574435830375944686:2111] self: [2:7574435830375944685:0] peer: [5:7574435830105538155:0] socket: 474 qp: -1
2025-11-19T13:59:29.572186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435830375944708:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:29.572218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:29.572497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435831002636050:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9042
2025-11-19T13:59:29.572540Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435831002636042:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:29.571537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435830703792961:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.571556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:29.571566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435830703792961:2062] [node 4] ICP22 created new session: [6:7574435830703793031:2062]
2025-11-19T13:59:29.571572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435830703793031:2062] [node 4] ICS09 handshake done sender: [6:7574435830703793015:2113] self: [6:7574435830703793021:0] peer: [4:7574435831002636025:0] socket: 464 qp: -1
2025-11-19T13:59:29.571578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435830703793031:2062] [node 4] ICS10 traffic start
2025-11-19T13:59:29.571584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:29.573501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435830105538065:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.573518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:29.573522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435830105538065:2048] [node 1] ICP22 created new session: [5:7574435830105538161:2048]
2025-11-19T13:59:29.573528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435830105538161:2048] [node 1] ICS09 handshake done sender: [5:7574435830105538143:2115] self: [5:7574435830105538145:0] peer: [1:7574435827792122470:0] socket: 445 qp: -1
2025-11-19T13:59:29.573533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435830105538161:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:29.573538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:29.576501Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435830105538158:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.576536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435830105538120:2082] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:29.576239Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435831002636050:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.576261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435831002635960:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.576273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:29.576278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435831002635960:2049] [node 2] ICP22 created new session: [4:7574435831002636051:2049]
2025-11-19T13:59:29.576283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435831002636051:2049] [node 2] ICS09 handshake done sender: [4:7574435831002636042:2118] self: [4:7574435831002636043:0] peer: [2:7574435830375944689:0] socket: 477 qp: -1
2025-11-19T13:59:29.576288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435831002636051:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:29.576294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:29.577175Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435827792122488:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:29.577283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435827792122475:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.577297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:29.577303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435827792122475:2109] [node 6] ICP22 created new session: [1:7574435827792122504:2109]
2025-11-19T13:59:29.577308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435827792122504:2109] [node 6] ICS09 handshake done sender: [1:7574435827792122488:2116] self: [1:7574435827792122479:0] peer: [6:7574435830703793027:0] socket: 483 qp: -1
2025-11-19T13:59:29.577313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435827792122504:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:29.577321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.579902Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435830375944691:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:29.580103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435830375944616:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.580113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:29.580119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435830375944616:2048] [node 1] ICP22 created new session: [2:7574435830375944710:2048]
2025-11-19T13:59:29.580125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435830375944710:2048] [node 1] ICS09 handshake done sender: [2:7574435830375944691:2113] self: [2:7574435830375944694:0] peer: [1:7574435827792122478:0] socket: 472 qp: -1
2025-11-19T13:59:29.580129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435830375944710:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:29.580134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:29.584146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435828791313640:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19583
2025-11-19T13:59:29.585416Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435830375944703:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.585621Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435828791313640:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.587791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435830375944704:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56147
2025-11-19T13:59:29.591859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435831002636024:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(27934345ns) {HandleHandshakeStatus:329(27933442ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(172ns) SwitchToState:178(216ns) ProcessPendingSessionEvents:495(23710946ns) {ForwardSessionEventToSession:612(23697963ns) ForwardSessionEventToSession:612(1764ns) ForwardSessionEventToSession:612(1455ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(601ns) ForwardSessionEventToSession:612(1051ns) ForwardSessionEventToSession:612(586ns) ForwardSessionEventToSession:612(487ns) ForwardSessionEventToSession:612(474ns) ForwardSessionEventToSession:612(838ns)}}}
2025-11-19T13:59:29.591972Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435831002636035:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.591977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435831002636035:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.592474Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435831002636032:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:29.592653Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435828791313614:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(28495305ns) {HandleHandshakeStatus:329(28494394ns) {DropIncomingHandshake:441(3514ns) DropOutgoingHandshake:456(70ns) SwitchToState:178(235ns) ProcessPendingSessionEvents:495(24489983ns) {ForwardSessionEventToSession:612(24477645ns) ForwardSessionEventToSession:612(1866ns) ForwardSessionEventToSession:612(680ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(713ns) ForwardSessionEventToSession:612(1114ns) ForwardSessionEventToSession:612(759ns) ForwardSessionEventToSession:612(919ns) ForwardSessionEventToSession:612(1071ns) ForwardSessionEventToSession:612(1039ns)}}}
2025-11-19T13:59:29.592741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435828791313552:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:29.593159Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435827792122480:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.593172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435827792122480:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.596196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435831002635959:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:29.596428Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435830375944704:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.596611Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435827792122480:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:29.596654Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435827792122486:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.596686Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435827792122490:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.596689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435827792122490:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.597009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435827792122475:2109] [node 6] ICP18 handshake (actor [1:7574435827792122486:2115]) is too old
2025-11-19T13:59:29.597749Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435831002636035:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:29.598596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435831002636033:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.598619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:29.598626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435831002636033:2112] [node 5] ICP22 created new session: [4:7574435831002636055:2112]
2025-11-19T13:59:29.598632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435831002636055:2112] [node 5] ICS09 handshake done sender: [4:7574435831002636035:2115] self: [4:7574435831002636034:0] peer: [5:7574435830105538163:0] socket: 500 qp: -1
2025-11-19T13:59:29.598638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435831002636055:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:29.598647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:29.598760Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435831002635961:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32588) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435831002636032:2114] held: no
2025-11-19T13:59:29.600004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435827792122409:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.600027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:29.600035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435827792122409:2049] [node 3] ICP22 created new session: [1:7574435827792122507:2049]
2025-11-19T13:59:29.600041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435827792122507:2049] [node 3] ICS09 handshake done sender: [1:7574435827792122480:2111] self: [1:7574435827792122473:0] peer: [3:7574435828791313642:0] socket: 459 qp: -1
2025-11-19T13:59:29.600047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435827792122507:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:29.600055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:29.600143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435831002635961:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.600155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:29.600161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435831002635961:2050] [node 3] ICP22 created new session: [4:7574435831002636057:2050]
2025-11-19T13:59:29.600165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435831002636057:2050] [node 3] ICS09 handshake done sender: [4:7574435831002636032:2114] self: [4:7574435831002636038:0] peer: [3:7574435828791313616:0] socket: 454 qp: -1
2025-11-19T13:59:29.600169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435831002636057:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:29.600174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:29.600703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435830105538120:2082] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21027) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435830105538158:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.604082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435828791313643:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19583
2025-11-19T13:59:29.604144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435828791313644:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19583
2025-11-19T13:59:29.604173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435828791313645:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19583
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.604756Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435828791313643:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.604777Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435828791313644:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.604813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435828791313553:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:29.605014Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435828791313640:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:29.605075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435828791313552:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.605089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:29.605096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435828791313552:2048] [node 1] ICP22 created new session: [3:7574435828791313648:2048]
2025-11-19T13:59:29.605102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435828791313648:2048] [node 1] ICS09 handshake done sender: [3:7574435828791313640:2116] self: [3:7574435828791313642:0] peer: [1:7574435827792122473:0] socket: 460 qp: -1
2025-11-19T13:59:29.605107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435828791313648:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:29.605113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.606317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435830105538164:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49507
2025-11-19T13:59:29.607142Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435830105538158:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:29.607255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435830105538120:2082] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.607268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:29.607273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435830105538120:2082] [node 4] ICP22 created new session: [5:7574435830105538165:2082]
2025-11-19T13:59:29.607279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435830105538165:2082] [node 4] ICS09 handshake done sender: [5:7574435830105538158:2121] self: [5:7574435830105538163:0] peer: [4:7574435831002636034:0] socket: 503 qp: -1
2025-11-19T13:59:29.607284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435830105538165:2082] [node 4] ICS10 traffic start
2025-11-19T13:59:29.607293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:29.612497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435830703792942:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22797) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435830703793025:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.613473Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435830375944696:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.613487Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435830375944696:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.613323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435830703793034:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47120
2025-11-19T13:59:29.613420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435830703793033:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47120
2025-11-19T13:59:29.614645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435830703793035:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47120
2025-11-19T13:59:29.615579Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435830375944696:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.615681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435828791313650:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19583
2025-11-19T13:59:29.613023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435831002636024:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:29.613060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435831002636059:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9042
2025-11-19T13:59:29.615825Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435827792122490:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:29.615774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435830375944617:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.615796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:29.615803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435830375944617:2049] [node 3] ICP22 created new session: [2:7574435830375944715:2049]
2025-11-19T13:59:29.616337Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435830703793033:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.615809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435830375944715:2049] [node 3] ICS09 handshake done sender: [2:7574435830375944696:2115] self: [2:7574435830375944677:0] peer: [3:7574435828791313647:0] socket: 473 qp: -1
2025-11-19T13:59:29.615815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435830375944715:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:29.615821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:29.616364Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435830703793035:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.616379Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435830703793025:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:29.616508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435827792122477:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.616527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:29.616533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435827792122477:2110] [node 4] ICP22 created new session: [1:7574435827792122510:2110]
2025-11-19T13:59:29.616539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435827792122510:2110] [node 4] ICS09 handshake done sender: [1:7574435827792122490:2118] self: [1:7574435827792122483:0] peer: [4:7574435831002636053:0] socket: 501 qp: -1
2025-11-19T13:59:29.616543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435827792122510:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:29.616549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:29.617397Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435828791313632:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.617407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435828791313632:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.616802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435830703792943:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:29.616819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435830703792944:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:29.617142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435830703792942:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.617149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:29.617152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435830703792942:2048] [node 1] ICP22 created new session: [6:7574435830703793038:2048]
2025-11-19T13:59:29.617157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435830703793038:2048] [node 1] ICS09 handshake done sender: [6:7574435830703793025:2118] self: [6:7574435830703793027:0] peer: [1:7574435827792122479:0] socket: 493 qp: -1
2025-11-19T13:59:29.617161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435830703793038:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:29.617170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:29.618060Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435830375944698:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:29.618066Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435830375944698:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:29.618858Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435830375944698:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:29.618920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435828791313553:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21467) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435828791313644:2118] held: no
2025-11-19T13:59:29.618991Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435828791313632:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:29.619101Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435828791313644:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:29.618916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435830375944684:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.618930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:29.618936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435830375944684:2109] [node 6] ICP22 created new session: [2:7574435830375944718:2109]
2025-11-19T13:59:29.618942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435830375944718:2109] [node 6] ICS09 handshake done sender: [2:7574435830375944698:2116] self: [2:7574435830375944687:0] peer: [6:7574435830703793036:0] socket: 505 qp: -1
2025-11-19T13:59:29.618946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435830375944718:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:29.618951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.619270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435830703793040:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47120
2025-11-19T13:59:29.619385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435828791313553:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.619398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:29.619402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435828791313553:2049] [node 2] ICP22 created new session: [3:7574435828791313652:2049]
2025-11-19T13:59:29.619407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435828791313652:2049] [node 2] ICS09 handshake done sender: [3:7574435828791313644:2118] self: [3:7574435828791313647:0] peer: [2:7574435830375944677:0] socket: 447 qp: -1
2025-11-19T13:59:29.619411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435828791313652:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:29.619417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:29.616115Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435831002636050:2121] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:29.617064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435831002635959:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.617077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:29.617081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435831002635959:2048] [node 1] ICP22 created new session: [4:7574435831002636060:2048]
2025-11-19T13:59:29.617085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435831002636060:2048] [node 1] ICS09 handshake done sender: [4:7574435831002636050:2121] self: [4:7574435831002636053:0] peer: [1:7574435827792122483:0] socket: 504 qp: -1
2025-11-19T13:59:29.617090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435831002636060:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:29.617095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:29.620117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435828791313628:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:29.620131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:29.620136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435828791313628:2112] [node 6] ICP22 created new session: [3:7574435828791313654:2112]
2025-11-19T13:59:29.620141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435828791313654:2112] [node 6] ICS09 handshake done sender: [3:7574435828791313632:2115] self: [3:7574435828791313630:0] peer: [6:7574435830703793037:0] socket: 507 qp: -1
2025-11-19T13:59:29.620146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435828791313654:2112] [node 6] ICS10 traffic start
2025-11-19T13:59:29.620151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:29.620050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435830703792943:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21467) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435830703793033:2120] held: no
2025-11-19T13:59:29.620074Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435830703793035:2122] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:29.620627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435830703792944:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.620637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:29.620642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435830703792944:2050] [node 3] ICP22 created new session: [6:7574435830703793041:2050]
2025-11-19T13:59:29.620647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435830703793041:2050] [node 3] ICS09 handshake done sender: [6:7574435830703793035:2122] self: [6:7574435830703793037:0] peer: [3:7574435828791313630:0] socket: 461 qp: -1
2025-11-19T13:59:29.620652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435830703793041:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:29.620656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:29.668022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435830703793043:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47120
2025-11-19T13:59:29.675933Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435830703793033:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:29.676078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435830703792943:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:29.676101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:29.676111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435830703792943:2049] [node 2] ICP22 created new session: [6:7574435830703793044:2049]
2025-11-19T13:59:29.676118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435830703793044:2049] [node 2] ICS09 handshake done sender: [6:7574435830703793033:2120] self: [6:7574435830703793036:0] peer: [2:7574435830375944687:0] socket: 508 qp: -1
2025-11-19T13:59:29.676127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435830703793044:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:29.676137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:29.699935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:29.700057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435830375944710:2048] [node 1] ICS01 socket: 472 reason# EndOfStream
2025-11-19T13:59:29.700069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435830375944616:2048] [node 1] ICP30 unregister session Session# [2:7574435830375944710:2048] VirtualId# [2:7574435830375944694:0]
2025-11-19T13:59:29.700078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435830375944710:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.700132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:29.700161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:29.700301Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:29.700375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435828791313648:2048] [node 1] ICS07 socket disconnect 460 reason# EndOfStream
2025-11-19T13:59:29.700379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435828791313648:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.700410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:29.700428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435828791313648:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:29.702721Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:29.702821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435830703793038:2048] [node 1] ICS01 socket: 493 reason# EndOfStream
2025-11-19T13:59:29.702831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435830703792942:2048] [node 1] ICP30 unregister session Session# [6:7574435830703793038:2048] VirtualId# [6:7574435830703793027:0]
2025-11-19T13:59:29.702837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435830703793038:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.702880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:29.702902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:29.703629Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435828791313552:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22797) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:29.703645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435828791313648:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:29.703649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435828791313552:2048] [node 1] ICP30 unregister session Session# [3:7574435828791313648:2048] VirtualId# [3:7574435828791313642:0]
2025-11-19T13:59:29.703664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:29.703671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435828791313552:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22797) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:29.703686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22797) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:29.703807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:29.703887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435831002636060:2048] [node 1] ICS07 socket disconnect 504 reason# EndOfStream
2025-11-19T13:59:29.703894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435831002636060:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.703946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:29.703966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435831002636060:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:29.703974Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:29.704048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435830105538161:2048] [node 1] ICS01 socket: 445 reason# EndOfStream
2025-11-19T13:59:29.704054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435830105538065:2048] [node 1] ICP30 unregister session Session# [5:7574435830105538161:2048] VirtualId# [5:7574435830105538145:0]
2025-11-19T13:59:29.704059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435830105538161:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.704110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:29.704134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:29.704335Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435831002635959:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22797) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:29.704345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435831002636060:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:29.704349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435831002635959:2048] [node 1] ICP30 unregister session Session# [4:7574435831002636060:2048] VirtualId# [4:7574435831002636053:0]
2025-11-19T13:59:29.704363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:29.704368Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435831002635959:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22797) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:29.704369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22797) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:29.709925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435831002635959:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:29.712029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574435828791313552:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:29.714641Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:29.714716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435828791313652:2049] [node 2] ICS01 socket: 447 reason# EndOfStream
2025-11-19T13:59:29.714723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435828791313553:2049] [node 2] ICP30 unregister session Session# [3:7574435828791313652:2049] VirtualId# [3:7574435828791313647:0]
2025-11-19T13:59:29.714728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435828791313652:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.714768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:29.714785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:29.733756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:29.733871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435831002636051:2049] [node 2] ICS01 socket: 477 reason# EndOfStream
2025-11-19T13:59:29.733884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435831002635960:2049] [node 2] ICP30 unregister session Session# [4:7574435831002636051:2049] VirtualId# [4:7574435831002636043:0]
2025-11-19T13:59:29.733893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435831002636051:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.733950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:29.733973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:29.738877Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:29.738975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435830703793044:2049] [node 2] ICS01 socket: 508 reason# EndOfStream
2025-11-19T13:59:29.738984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435830703792943:2049] [node 2] ICP30 unregister session Session# [6:7574435830703793044:2049] VirtualId# [6:7574435830703793036:0]
2025-11-19T13:59:29.738991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435830703793044:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.739040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:29.739063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:29.739944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:29.740018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435830105538159:2049] [node 2] ICS01 socket: 475 reason# EndOfStream
2025-11-19T13:59:29.740025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435830105538066:2049] [node 2] ICP30 unregister session Session# [5:7574435830105538159:2049] VirtualId# [5:7574435830105538155:0]
2025-11-19T13:59:29.740030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435830105538159:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.740071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:29.740093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:29.760339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:29.760449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435830105538153:2050] [node 3] ICS07 socket disconnect 462 reason# EndOfStream
2025-11-19T13:59:29.760457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435830105538153:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.760514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:29.760535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435830105538153:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:29.761295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435830105538067:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32588) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:29.761307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435830105538153:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:29.761312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435830105538067:2050] [node 3] ICP30 unregister session Session# [5:7574435830105538153:2050] VirtualId# [5:7574435830105538146:0]
2025-11-19T13:59:29.761330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:29.761336Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435830105538067:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32588) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:29.761339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32588) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:29.760768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:29.760837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435830703793041:2050] [node 3] ICS01 socket: 461 reason# EndOfStream
2025-11-19T13:59:29.760843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435830703792944:2050] [node 3] ICP30 unregister session Session# [6:7574435830703793041:2050] VirtualId# [6:7574435830703793037:0]
2025-11-19T13:59:29.760849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435830703793041:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.760884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:29.760907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:29.773142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435830105538067:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:59:29.796542Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:29.796650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435830703793031:2062] [node 4] ICS01 socket: 464 reason# EndOfStream
2025-11-19T13:59:29.796661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435830703792961:2062] [node 4] ICP30 unregister session Session# [6:7574435830703793031:2062] VirtualId# [6:7574435830703793021:0]
2025-11-19T13:59:29.796669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435830703793031:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:29.796714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:29.796737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:29.912014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435831148361187:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:29.912035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435831148361188:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:29.995873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435830869678715:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:29.995890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435830869678716:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:30.056034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435832087335871:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:30.056047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435832087335872:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:30.171369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435834889922249:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:30.171387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435834889922250:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:30.171393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435834889922251:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:30.272700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435833668799633:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:30.272715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435833668799634:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:30.272720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435833668799635:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:30.364175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435834019815322:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:30.364197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435834019815323:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:30.364203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435834019815324:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:30.436168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435834889922305:2099] [node 6] ICP01 ready to work
2025-11-19T13:59:30.436446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435833668799653:2066] [node 4] ICP01 ready to work
2025-11-19T13:59:30.437382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435835443328544:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.437811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435835443328546:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:30.437941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435835164646071:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21603
2025-11-19T13:59:30.438216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435833668799658:2068] [node 6] ICP01 ready to work
2025-11-19T13:59:30.439375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435832087335932:2104] [node 6] ICP01 ready to work
2025-11-19T13:59:30.439276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435835443328552:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.446951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435834889922304:2097] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.447411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435835443328550:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11872
2025-11-19T13:59:30.448161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435835164646078:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:30.448242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435835164646079:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:30.448309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435835164646080:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:30.455845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435835443328546:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17969454ns) {Configure:104(17968865ns) {ProcessConfigured:135(16167605ns) {StartInitialHandshake:157(16166686ns) {DropHandshakes:469(417ns) {DropIncomingHandshake:441(34ns) DropOutgoingHandshake:456(15ns)} GenerateSessionVirtualId:988(63ns) PrepareNewSessionHandshake:476(835ns) {SwitchToState:178(265ns)}}}}}
2025-11-19T13:59:30.456828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435832087335871:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17469137ns) {Configure:104(17468384ns) {ProcessConfigured:135(17162856ns) {StartInitialHandshake:157(17162481ns) {DropHandshakes:469(476ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(289ns) PrepareNewSessionHandshake:476(1038ns) {SwitchToState:178(436ns)}}}}}
2025-11-19T13:59:30.456988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435832087335941:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:30.457551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435832087335932:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(18129643ns) {Configure:104(18128781ns) {ProcessConfigured:135(17864052ns) {StartInitialHandshake:157(17863881ns) {DropHandshakes:469(395ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(94ns) PrepareNewSessionHandshake:476(658ns) {SwitchToState:178(23ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.458948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435835443328560:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11872
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.463500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435833668799695:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13926
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.464862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435834019815350:2069] [node 4] ICP01 ready to work
2025-11-19T13:59:30.463624Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435835443328544:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(26181889ns) {Configure:104(26181300ns) {ProcessConfigured:135(23829812ns) {StartInitialHandshake:157(23829239ns) {DropHandshakes:469(153ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(23ns)} GenerateSessionVirtualId:988(183ns) PrepareNewSessionHandshake:476(784ns) {SwitchToState:178(489ns)}}}}}
2025-11-19T13:59:30.464778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435835443328551:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11872
2025-11-19T13:59:30.465359Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435835443328551:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.463977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435834889922305:2099] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27737233ns) {Configure:104(27736486ns) {ProcessConfigured:135(27369820ns) {StartInitialHandshake:157(27369109ns) {DropHandshakes:469(563ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(327ns) PrepareNewSessionHandshake:476(837ns) {SwitchToState:178(430ns)}}}}}
2025-11-19T13:59:30.464095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435834889922321:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17209
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.473990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435832087335948:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22581
2025-11-19T13:59:30.474034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435832087335949:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22581
2025-11-19T13:59:30.474061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435832087335950:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22581
2025-11-19T13:59:30.473741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435834019815392:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59435
2025-11-19T13:59:30.473802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435834019815393:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59435
2025-11-19T13:59:30.473833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435834019815394:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59435
2025-11-19T13:59:30.475483Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435834019815392:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.475511Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435834019815393:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.475331Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435834889922304:2097] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(28283478ns) {Configure:104(28282219ns) {ProcessConfigured:135(27952454ns) {StartInitialHandshake:157(27952042ns) {DropHandshakes:469(1145ns) {DropIncomingHandshake:441(323ns) DropOutgoingHandshake:456(208ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(928ns) {SwitchToState:178(351ns)}}}}}
2025-11-19T13:59:30.475932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435833668799709:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13926
2025-11-19T13:59:30.475720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435834019815322:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(35475987ns) {Configure:104(28046716ns) {ProcessConfigured:135(27735438ns) {StartInitialHandshake:157(27734895ns) {DropHandshakes:469(476ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(258ns) PrepareNewSessionHandshake:476(595ns) {SwitchToState:178(331ns)}}}}}
2025-11-19T13:59:30.475772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435834019815322:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.476068Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435835443328552:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(36189232ns) {Configure:104(27916978ns) {ProcessConfigured:135(27671861ns) {StartInitialHandshake:157(27671116ns) {DropHandshakes:469(313ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:476(436ns) {SwitchToState:178(100ns)}}}}}
2025-11-19T13:59:30.476865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435834019815364:2073] [node 5] ICP01 ready to work
2025-11-19T13:59:30.479881Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435832087335949:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.479953Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435832087335950:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.479979Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435832087335948:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.480027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435832087335871:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.482280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435834889922251:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21344888ns) {Configure:104(21344227ns) {ProcessConfigured:135(21016702ns) {StartInitialHandshake:157(21016134ns) {DropHandshakes:469(558ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(415ns) PrepareNewSessionHandshake:476(721ns) {SwitchToState:178(238ns)}}}}}
2025-11-19T13:59:30.482423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435835164646079:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(33792901ns) {Configure:104(17608716ns) {ProcessConfigured:135(17299250ns) {StartInitialHandshake:157(17299091ns) {DropHandshakes:469(874ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(158ns)} GenerateSessionVirtualId:988(373ns) PrepareNewSessionHandshake:476(1206ns) {SwitchToState:178(18ns)}}}}}
2025-11-19T13:59:30.482462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435835164646084:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21603
2025-11-19T13:59:30.482477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435835164646085:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21603
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.482982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435833668799711:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13926
2025-11-19T13:59:30.485294Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435835443328549:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.485311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435835443328549:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.486191Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435835443328549:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.487025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435834019815402:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59435
2025-11-19T13:59:30.487229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435831148361188:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.487244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:30.487250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435831148361188:2049] [node 3] ICP22 created new session: [1:7574435835443328566:2049]
2025-11-19T13:59:30.487255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435835443328566:2049] [node 3] ICS09 handshake done sender: [1:7574435835443328549:2110] self: [1:7574435835443328548:0] peer: [3:7574435832087335952:0] socket: 463 qp: -1
2025-11-19T13:59:30.487260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435835443328566:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:30.487266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.489001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435833668799712:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13926
2025-11-19T13:59:30.489098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435835164646071:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.489172Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435834889922321:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.486032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435832087335954:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22581
2025-11-19T13:59:30.486386Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435832087335950:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:30.486481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435832087335871:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.486501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:30.486508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435832087335871:2048] [node 1] ICP22 created new session: [3:7574435832087335955:2048]
2025-11-19T13:59:30.486514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435832087335955:2048] [node 1] ICS09 handshake done sender: [3:7574435832087335950:2116] self: [3:7574435832087335952:0] peer: [1:7574435835443328548:0] socket: 476 qp: -1
2025-11-19T13:59:30.488157Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435831148361187:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(48804332ns) {Configure:104(48803682ns) {ProcessConfigured:135(48569473ns) {StartInitialHandshake:157(48568909ns) {DropHandshakes:469(298ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(114ns) PrepareNewSessionHandshake:476(811ns) {SwitchToState:178(612ns)}}}}}
2025-11-19T13:59:30.486522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435832087335955:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:30.486539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:30.492186Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435833668799634:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23938056ns) {Configure:104(23937101ns) {ProcessConfigured:135(23604740ns) {StartInitialHandshake:157(23603892ns) {DropHandshakes:469(435ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(269ns) PrepareNewSessionHandshake:476(991ns) {SwitchToState:178(751ns)}}}}}
2025-11-19T13:59:30.492091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435834889922249:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23993830ns) {Configure:104(23993157ns) {ProcessConfigured:135(23639680ns) {StartInitialHandshake:157(23639338ns) {DropHandshakes:469(448ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(81ns)} GenerateSessionVirtualId:988(255ns) PrepareNewSessionHandshake:476(1329ns) {SwitchToState:178(27ns)}}}}}
2025-11-19T13:59:30.492127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435834889922249:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.495782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435832087335947:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:30.496400Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435833668799712:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.496444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435833668799633:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.495939Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435835443328557:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.495963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435835443328557:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.504607Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435830869678716:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(29576784ns) {Configure:104(29576113ns) {ProcessConfigured:135(29329764ns) {StartInitialHandshake:157(29328836ns) {DropHandshakes:469(1040ns) {DropIncomingHandshake:441(101ns) DropOutgoingHandshake:456(182ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(455ns) {SwitchToState:178(212ns)}}}}}
2025-11-19T13:59:30.504873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435835164646080:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(43143308ns) {Configure:104(24083761ns) {ProcessConfigured:135(23722682ns) {StartInitialHandshake:157(23722283ns) {DropHandshakes:469(1199ns) {DropIncomingHandshake:441(100ns) DropOutgoingHandshake:456(197ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:476(682ns) {SwitchToState:178(312ns)}}}}}
2025-11-19T13:59:30.503861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435833668799653:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(67341108ns) {Configure:104(67340253ns) {ProcessConfigured:135(63787988ns) {StartInitialHandshake:157(63787020ns) {DropHandshakes:469(552ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(39ns)} GenerateSessionVirtualId:988(113ns) PrepareNewSessionHandshake:476(1242ns) {SwitchToState:178(260ns)}}}}}
2025-11-19T13:59:30.504037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435834019815364:2073] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.504091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435834019815364:2073] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(27160051ns) {Configure:104(27159705ns) {ProcessConfigured:135(26863008ns) {StartInitialHandshake:157(26821430ns) {DropHandshakes:469(703ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(220ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:476(962ns) {SwitchToState:178(461ns)}} IncomingHandshake:261(31152ns) {DropIncomingHandshake:441(73ns) PrepareNewSessionHandshake:476(188ns) {SwitchToState:178(141ns)} GenerateSessionVirtualId:988(441ns) IssueIncomingHandshakeReply:193(24651ns) {SwitchToState:178(31ns)}}}}}
2025-11-19T13:59:30.504704Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435834019815350:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(39773050ns) {Configure:104(39772532ns) {ProcessConfigured:135(39487781ns) {StartInitialHandshake:157(39487448ns) {DropHandshakes:469(540ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(297ns) PrepareNewSessionHandshake:476(549ns) {SwitchToState:178(358ns)}}}}}
2025-11-19T13:59:30.504424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435835443328570:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11872
2025-11-19T13:59:30.504806Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435835443328561:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.504814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435835443328561:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:30.504934Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435835443328563:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.504938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435835443328563:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:30.508305Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435835443328570:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.511862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435835164646078:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(63500162ns) {Configure:104(63498932ns) {ProcessConfigured:135(63239635ns) {StartInitialHandshake:157(63238954ns) {DropHandshakes:469(554ns) {DropIncomingHandshake:441(129ns) DropOutgoingHandshake:456(83ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:476(2376ns) {SwitchToState:178(1694ns)}}}}}
2025-11-19T13:59:30.512858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435832087335947:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16995776ns) {Configure:104(16995189ns) {ProcessConfigured:135(16660288ns) {StartInitialHandshake:157(16659276ns) {DropHandshakes:469(740ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(192ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:476(1334ns) {SwitchToState:178(791ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.514739Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435835443328557:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:30.515147Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435833668799660:2070] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.515161Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435833668799660:2070] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.516200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435834889922329:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17209
2025-11-19T13:59:30.515914Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435833668799660:2070] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.516874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435833668799658:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.516892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:30.516899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435833668799658:2068] [node 6] ICP22 created new session: [5:7574435833668799717:2068]
2025-11-19T13:59:30.516907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435833668799717:2068] [node 6] ICS09 handshake done sender: [5:7574435833668799660:2070] self: [5:7574435833668799659:0] peer: [6:7574435834019815404:0] socket: 454 qp: -1
2025-11-19T13:59:30.516915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435833668799717:2068] [node 6] ICS10 traffic start
2025-11-19T13:59:30.516942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:30.518524Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435835443328561:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.519125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435834889922330:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17209
2025-11-19T13:59:30.514282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435834019815406:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59435
2025-11-19T13:59:30.514301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435834019815407:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59435
2025-11-19T13:59:30.516554Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435834019815393:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:30.516586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435834019815408:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59435
2025-11-19T13:59:30.516747Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435834019815392:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:30.518656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435834019815364:2073] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.518672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:30.518677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435834019815364:2073] [node 5] ICP22 created new session: [6:7574435834019815409:2073]
2025-11-19T13:59:30.518682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435834019815409:2073] [node 5] ICS09 handshake done sender: [6:7574435834019815392:2109] self: [6:7574435834019815404:0] peer: [5:7574435833668799659:0] socket: 457 qp: -1
2025-11-19T13:59:30.518687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435834019815409:2073] [node 5] ICS10 traffic start
2025-11-19T13:59:30.518716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:30.520520Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435834889922321:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:30.520662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435834889922249:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.520662Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435834889922330:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.520678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:30.520685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435834889922249:2048] [node 1] ICP22 created new session: [4:7574435834889922331:2048]
2025-11-19T13:59:30.520692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435834889922331:2048] [node 1] ICS09 handshake done sender: [4:7574435834889922321:2112] self: [4:7574435834889922328:0] peer: [1:7574435835443328555:0] socket: 471 qp: -1
2025-11-19T13:59:30.520698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435834889922331:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:30.520712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:30.521388Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435835443328563:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.521607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435833668799720:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13926
2025-11-19T13:59:30.521733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435835443328552:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.521750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:30.521755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435835443328552:2109] [node 5] ICP22 created new session: [1:7574435835443328575:2109]
2025-11-19T13:59:30.521760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435835443328575:2109] [node 5] ICS09 handshake done sender: [1:7574435835443328563:2116] self: [1:7574435835443328556:0] peer: [5:7574435833668799714:0] socket: 488 qp: -1
2025-11-19T13:59:30.521764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435835443328575:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:30.521779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:30.522144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435835443328544:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.522155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:30.522159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435835443328544:2107] [node 4] ICP22 created new session: [1:7574435835443328577:2107]
2025-11-19T13:59:30.522164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435835443328577:2107] [node 4] ICS09 handshake done sender: [1:7574435835443328561:2115] self: [1:7574435835443328555:0] peer: [4:7574435834889922328:0] socket: 470 qp: -1
2025-11-19T13:59:30.522168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435835443328577:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:30.522172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:30.522485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435834019815322:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.522502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:30.522506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435834019815322:2048] [node 1] ICP22 created new session: [6:7574435834019815412:2048]
2025-11-19T13:59:30.522510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435834019815412:2048] [node 1] ICS09 handshake done sender: [6:7574435834019815393:2110] self: [6:7574435834019815396:0] peer: [1:7574435835443328554:0] socket: 473 qp: -1
2025-11-19T13:59:30.522515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435834019815412:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:30.522523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:30.523387Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435833668799712:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:30.524569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435833668799633:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.524586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:30.524593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435833668799633:2048] [node 1] ICP22 created new session: [5:7574435833668799721:2048]
2025-11-19T13:59:30.524598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435833668799721:2048] [node 1] ICS09 handshake done sender: [5:7574435833668799712:2116] self: [5:7574435833668799714:0] peer: [1:7574435835443328556:0] socket: 466 qp: -1
2025-11-19T13:59:30.524604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435833668799721:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:30.524610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.527690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435832087335960:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22581
2025-11-19T13:59:30.528748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435835164646095:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21603
2025-11-19T13:59:30.528938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435835164646084:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.529286Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435834019815406:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.529344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435834019815323:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.529943Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435834019815394:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.529984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435834019815350:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.530312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435835443328546:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.530333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:30.530339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435835443328546:2108] [node 6] ICP22 created new session: [1:7574435835443328580:2108]
2025-11-19T13:59:30.530344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435835443328580:2108] [node 6] ICS09 handshake done sender: [1:7574435835443328557:2113] self: [1:7574435835443328554:0] peer: [6:7574435834019815396:0] socket: 464 qp: -1
2025-11-19T13:59:30.530350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435835443328580:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:30.530356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:30.535052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435835164646095:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.535109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435830869678715:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.535300Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435835443328550:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.535350Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435833668799711:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.535388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435833668799634:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.539874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435835164646094:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21603
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.541255Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435834019815402:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.541300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435834019815324:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.541496Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435832087335946:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.541507Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435832087335946:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:30.542348Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435832087335946:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:30.542479Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435832087335943:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.542486Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435832087335943:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:30.542910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435832087335932:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.542926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:30.542933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435832087335932:2104] [node 6] ICP22 created new session: [3:7574435832087335962:2104]
2025-11-19T13:59:30.542938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435832087335962:2104] [node 6] ICS09 handshake done sender: [3:7574435832087335946:2113] self: [3:7574435832087335935:0] peer: [6:7574435834019815416:0] socket: 486 qp: -1
2025-11-19T13:59:30.542944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435832087335962:2104] [node 6] ICS10 traffic start
2025-11-19T13:59:30.542965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:30.540579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435834889922335:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17209
2025-11-19T13:59:30.543452Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435834889922335:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.543835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435834019815417:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59435
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.543990Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435832087335943:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:30.542202Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435833668799695:2100] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.542230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435833668799635:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.543818Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435833668799709:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.543934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435833668799653:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.543955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435833668799726:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13926
2025-11-19T13:59:30.544142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435832087335941:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.544162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:30.544169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435832087335941:2110] [node 5] ICP22 created new session: [3:7574435832087335966:2110]
2025-11-19T13:59:30.544175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435832087335966:2110] [node 5] ICS09 handshake done sender: [3:7574435832087335943:2112] self: [3:7574435832087335942:0] peer: [5:7574435833668799725:0] socket: 467 qp: -1
2025-11-19T13:59:30.544180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435832087335966:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:30.544186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:30.545025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435833668799635:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13656) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435833668799695:2100] held: no
2025-11-19T13:59:30.545053Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435833668799695:2100] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:30.545260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435833668799635:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.545274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:30.545279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435833668799635:2050] [node 3] ICP22 created new session: [5:7574435833668799728:2050]
2025-11-19T13:59:30.545284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435833668799728:2050] [node 3] ICS09 handshake done sender: [5:7574435833668799695:2100] self: [5:7574435833668799725:0] peer: [3:7574435832087335942:0] socket: 469 qp: -1
2025-11-19T13:59:30.545289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435833668799728:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:30.545298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:30.545938Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435834019815402:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:30.546080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435834019815324:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.546095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:30.546100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435834019815324:2050] [node 3] ICP22 created new session: [6:7574435834019815418:2050]
2025-11-19T13:59:30.546105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435834019815418:2050] [node 3] ICS09 handshake done sender: [6:7574435834019815402:2116] self: [6:7574435834019815416:0] peer: [3:7574435832087335935:0] socket: 487 qp: -1
2025-11-19T13:59:30.546111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435834019815418:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:30.546121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:30.548277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435835164646094:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.545397Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435834889922324:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.545407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435834889922324:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:30.546975Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435834889922324:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:30.547520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435834889922304:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.547532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:30.547537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435834889922304:2097] [node 5] ICP22 created new session: [4:7574435834889922338:2097]
2025-11-19T13:59:30.547541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435834889922338:2097] [node 5] ICS09 handshake done sender: [4:7574435834889922324:2113] self: [4:7574435834889922310:0] peer: [5:7574435833668799727:0] socket: 478 qp: -1
2025-11-19T13:59:30.547545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435834889922338:2097] [node 5] ICS10 traffic start
2025-11-19T13:59:30.547551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:30.552349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435834889922320:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.552363Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435834889922320:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:30.555583Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435835443328569:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.555601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435835443328569:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:30.555636Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435835164646087:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.555655Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435835164646087:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:30.555844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435833668799653:2066] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14659) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435833668799709:2113] held: no
2025-11-19T13:59:30.556573Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435835443328546:2108] [node 6] ICP35 event processing took too much time PendingConnection:278(26231681ns) {HandleHandshakeStatus:329(26230858ns) {DropIncomingHandshake:441(2249ns) DropOutgoingHandshake:456(153ns) SwitchToState:178(56ns) ProcessPendingSessionEvents:495(26162476ns) {ForwardSessionEventToSession:612(26152490ns) ForwardSessionEventToSession:612(1835ns) ForwardSessionEventToSession:612(534ns) ForwardSessionEventToSession:612(993ns) ForwardSessionEventToSession:612(545ns) ForwardSessionEventToSession:612(602ns) ForwardSessionEventToSession:612(638ns) ForwardSessionEventToSession:612(879ns) ForwardSessionEventToSession:612(531ns) ForwardSessionEventToSession:612(900ns)}}}
2025-11-19T13:59:30.556738Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435835164646089:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.556748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435835164646089:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:30.557504Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435835164646089:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:30.557602Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435835164646085:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.557610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435835164646080:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.557624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:30.557629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435835164646080:2111] [node 6] ICP22 created new session: [2:7574435835164646100:2111]
2025-11-19T13:59:30.557633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435835164646100:2111] [node 6] ICS09 handshake done sender: [2:7574435835164646089:2115] self: [2:7574435835164646083:0] peer: [6:7574435834019815414:0] socket: 468 qp: -1
2025-11-19T13:59:30.557638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435835164646100:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:30.557658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:30.559977Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435832087335960:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.561076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435834019815323:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25428) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435834019815406:2119] held: no
2025-11-19T13:59:30.561136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435834019815420:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59435
No RDMA context found for GID: 2025-11-19T13:59:30.562228Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435835443328569:2117] [node 2] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-19T13:59:30.562272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435834019815421:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59435
2025-11-19T13:59:30.562372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435831148361187:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.562394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:30.562400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435831148361187:2048] [node 2] ICP22 created new session: [1:7574435835443328584:2048]
2025-11-19T13:59:30.562406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435835443328584:2048] [node 2] ICS09 handshake done sender: [1:7574435835443328569:2117] self: [1:7574435835443328553:0] peer: [2:7574435835164646097:0] socket: 502 qp: -1
2025-11-19T13:59:30.562412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435835443328584:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:30.562418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.562568Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435835164646087:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:30.562613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435835164646105:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21603
2025-11-19T13:59:30.562696Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435834889922320:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:30.563212Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435835164646095:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:30.563333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435834889922305:2099] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.563356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:30.563362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435834889922305:2099] [node 6] ICP22 created new session: [4:7574435834889922341:2099]
2025-11-19T13:59:30.563367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435834889922341:2099] [node 6] ICS09 handshake done sender: [4:7574435834889922320:2111] self: [4:7574435834889922306:0] peer: [6:7574435834019815415:0] socket: 472 qp: -1
2025-11-19T13:59:30.563373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435834889922341:2099] [node 6] ICS10 traffic start
2025-11-19T13:59:30.563378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:30.564058Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435830869678715:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24622) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435835164646095:2119] held: no
2025-11-19T13:59:30.564285Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435834019815394:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:30.564364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435834019815350:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.564375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:30.564380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435834019815350:2069] [node 4] ICP22 created new session: [6:7574435834019815422:2069]
2025-11-19T13:59:30.564385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435834019815422:2069] [node 4] ICS09 handshake done sender: [6:7574435834019815394:2111] self: [6:7574435834019815415:0] peer: [4:7574435834889922306:0] socket: 474 qp: -1
2025-11-19T13:59:30.564390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435834019815422:2069] [node 4] ICS10 traffic start
2025-11-19T13:59:30.564395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:30.564810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435835164646079:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.564826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:30.564832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435835164646079:2110] [node 5] ICP22 created new session: [2:7574435835164646106:2110]
2025-11-19T13:59:30.564839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435835164646106:2110] [node 5] ICS09 handshake done sender: [2:7574435835164646087:2114] self: [2:7574435835164646082:0] peer: [5:7574435833668799724:0] socket: 482 qp: -1
2025-11-19T13:59:30.564845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435835164646106:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:30.564851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.567854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435834889922343:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17209
2025-11-19T13:59:30.568474Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435834019815406:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:30.568868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435833668799634:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25428) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435833668799711:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.569075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435832087335969:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22581
2025-11-19T13:59:30.573656Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435834889922343:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.573750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435834889922251:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.573941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435834019815323:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.573959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:30.573964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435834019815323:2049] [node 2] ICP22 created new session: [6:7574435834019815424:2049]
2025-11-19T13:59:30.573970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435834019815424:2049] [node 2] ICS09 handshake done sender: [6:7574435834019815406:2119] self: [6:7574435834019815414:0] peer: [2:7574435835164646083:0] socket: 489 qp: -1
2025-11-19T13:59:30.573976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435834019815424:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:30.573983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.577269Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435832087335959:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.577282Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435832087335959:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:30.577661Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435832087335969:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.577762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435832087335872:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.577993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435834889922345:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17209
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.578654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435833668799730:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13926
2025-11-19T13:59:30.578830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435833668799731:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13926
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.579232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435834889922346:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17209
2025-11-19T13:59:30.579312Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435833668799711:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:30.579403Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435835164646091:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.579413Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435835164646091:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:30.580128Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435834889922345:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.580170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435834889922250:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:30.580385Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435832087335959:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:30.580554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435832087335947:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.580576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:30.580582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435832087335947:2109] [node 4] ICP22 created new session: [3:7574435832087335972:2109]
2025-11-19T13:59:30.580588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435832087335972:2109] [node 4] ICS09 handshake done sender: [3:7574435832087335959:2118] self: [3:7574435832087335958:0] peer: [4:7574435834889922344:0] socket: 507 qp: -1
2025-11-19T13:59:30.580593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435832087335972:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:30.580614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:30.580727Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435833668799709:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:30.580817Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435835164646091:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:30.580872Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435834889922343:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:30.580946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435834889922251:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.580959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:30.580963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435834889922251:2050] [node 3] ICP22 created new session: [4:7574435834889922348:2050]
2025-11-19T13:59:30.580967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435834889922348:2050] [node 3] ICS09 handshake done sender: [4:7574435834889922343:2119] self: [4:7574435834889922344:0] peer: [3:7574435832087335958:0] socket: 510 qp: -1
2025-11-19T13:59:30.580972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435834889922348:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:30.580980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:30.581205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435830869678716:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.581218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:30.581222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435830869678716:2049] [node 3] ICP22 created new session: [2:7574435835164646110:2049]
2025-11-19T13:59:30.581226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435835164646110:2049] [node 3] ICS09 handshake done sender: [2:7574435835164646091:2116] self: [2:7574435835164646086:0] peer: [3:7574435832087335971:0] socket: 451 qp: -1
2025-11-19T13:59:30.581230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435835164646110:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:30.581234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:30.586922Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435835164646093:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:30.586934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435835164646093:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.587848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435832087335974:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22581
2025-11-19T13:59:30.587914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435833668799653:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.587934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:30.587941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435833668799653:2066] [node 4] ICP22 created new session: [5:7574435833668799732:2066]
2025-11-19T13:59:30.587949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435833668799732:2066] [node 4] ICS09 handshake done sender: [5:7574435833668799709:2113] self: [5:7574435833668799727:0] peer: [4:7574435834889922310:0] socket: 479 qp: -1
2025-11-19T13:59:30.587956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435833668799732:2066] [node 4] ICS10 traffic start
2025-11-19T13:59:30.587965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:30.588122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435833668799634:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.588131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:30.588137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435833668799634:2049] [node 2] ICP22 created new session: [5:7574435833668799734:2049]
2025-11-19T13:59:30.588142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435833668799734:2049] [node 2] ICS09 handshake done sender: [5:7574435833668799711:2115] self: [5:7574435833668799724:0] peer: [2:7574435835164646082:0] socket: 483 qp: -1
2025-11-19T13:59:30.588145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435833668799734:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:30.588150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:30.588903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435830869678715:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.588921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:30.588927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435830869678715:2048] [node 1] ICP22 created new session: [2:7574435835164646112:2048]
2025-11-19T13:59:30.588933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435835164646112:2048] [node 1] ICS09 handshake done sender: [2:7574435835164646095:2119] self: [2:7574435835164646097:0] peer: [1:7574435835443328553:0] socket: 504 qp: -1
2025-11-19T13:59:30.588938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435835164646112:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:30.588955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:30.589310Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435832087335872:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25428) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435832087335969:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:30.590986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435834889922350:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17209
2025-11-19T13:59:30.591148Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435835164646093:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:30.591232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435830869678715:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22778424ns) {HandleHandshakeStatus:329(22777983ns) {DropIncomingHandshake:441(504ns) DropOutgoingHandshake:456(78ns) SwitchToState:178(55ns) ProcessPendingSessionEvents:495(208ns)}}
2025-11-19T13:59:30.591995Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435834889922345:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:30.592139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435834889922250:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.592151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:30.592156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435834889922250:2049] [node 2] ICP22 created new session: [4:7574435834889922351:2049]
2025-11-19T13:59:30.592162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435834889922351:2049] [node 2] ICS09 handshake done sender: [4:7574435834889922345:2120] self: [4:7574435834889922347:0] peer: [2:7574435835164646081:0] socket: 455 qp: -1
2025-11-19T13:59:30.592167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435834889922351:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:30.592176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:30.592251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435835164646078:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:30.592265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:30.592268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435835164646078:2109] [node 4] ICP22 created new session: [2:7574435835164646115:2109]
2025-11-19T13:59:30.592272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435835164646115:2109] [node 4] ICS09 handshake done sender: [2:7574435835164646093:2117] self: [2:7574435835164646081:0] peer: [4:7574435834889922347:0] socket: 450 qp: -1
2025-11-19T13:59:30.592276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435835164646115:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:30.592280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:30.628129Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435832087335969:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:30.628596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435832087335872:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:30.628620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:30.628631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435832087335872:2049] [node 2] ICP22 created new session: [3:7574435832087335975:2049]
2025-11-19T13:59:30.628638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435832087335975:2049] [node 2] ICS09 handshake done sender: [3:7574435832087335969:2120] self: [3:7574435832087335971:0] peer: [2:7574435835164646086:0] socket: 459 qp: -1
2025-11-19T13:59:30.628656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435832087335975:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:30.628668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:30.640316Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:30.640423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435835164646112:2048] [node 1] ICS01 socket: 504 reason# EndOfStream
2025-11-19T13:59:30.640434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435830869678715:2048] [node 1] ICP30 unregister session Session# [2:7574435835164646112:2048] VirtualId# [2:7574435835164646097:0]
2025-11-19T13:59:30.640444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435835164646112:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:30.640482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:30.640503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:30.641729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:30.641810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435832087335955:2048] [node 1] ICS01 socket: 476 reason# EndOfStream
2025-11-19T13:59:30.641818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435832087335871:2048] [node 1] ICP30 unregister session Session# [3:7574435832087335955:2048] VirtualId# [3:7574435832087335952:0]
2025-11-19T13:59:30.641824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435832087335955:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:30.641887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:30.641911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:30.651828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:30.651931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435834019815412:2048] [node 1] ICS01 socket: 473 reason# EndOfStream
2025-11-19T13:59:30.651940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435834019815322:2048] [node 1] ICP30 unregister session Session# [6:7574435834019815412:2048] VirtualId# [6:7574435834019815396:0]
2025-11-19T13:59:30.651948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435834019815412:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:30.652003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:30.652031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:30.652065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:30.652141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435833668799721:2048] [node 1] ICS01 socket: 466 reason# EndOfStream
2025-11-19T13:59:30.652146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435833668799633:2048] [node 1] ICP30 unregister session Session# [5:7574435833668799721:2048] VirtualId# [5:7574435833668799714:0]
2025-11-19T13:59:30.652151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435833668799721:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:30.652175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:30.652194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:30.652606Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:30.652678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435834889922331:2048] [node 1] ICS01 socket: 471 reason# EndOfStream
2025-11-19T13:59:30.652685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435834889922249:2048] [node 1] ICP30 unregister session Session# [4:7574435834889922331:2048] VirtualId# [4:7574435834889922328:0]
2025-11-19T13:59:30.652690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435834889922331:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:30.652733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:30.652756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:30.662101Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:30.662200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435832087335975:2049] [node 2] ICS01 socket: 459 reason# EndOfStream
2025-11-19T13:59:30.662209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435832087335872:2049] [node 2] ICP30 unregister session Session# [3:7574435832087335975:2049] VirtualId# [3:7574435832087335971:0]
2025-11-19T13:59:30.662216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435832087335975:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:30.662262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:30.662282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:30.672043Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:30.672145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435834889922351:2049] [node 2] ICS01 socket: 455 reason# EndOfStream
2025-11-19T13:59:30.672154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435834889922250:2049] [node 2] ICP30 unregister session Session# [4:7574435834889922351:2049] VirtualId# [4:7574435834889922347:0]
2025-11-19T13:59:30.672162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435834889922351:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:30.672202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:30.672224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:30.675593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:30.675666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435834019815424:2049] [node 2] ICS01 socket: 489 reason# EndOfStream
2025-11-19T13:59:30.675675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435834019815323:2049] [node 2] ICP30 unregister session Session# [6:7574435834019815424:2049] VirtualId# [6:7574435834019815414:0]
2025-11-19T13:59:30.675679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435834019815424:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:30.675369Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:30.675466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435833668799734:2049] [node 2] ICS01 socket: 483 reason# EndOfStream
2025-11-19T13:59:30.675473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435833668799634:2049] [node 2] ICP30 unregister session Session# [5:7574435833668799734:2049] VirtualId# [5:7574435833668799724:0]
2025-11-19T13:59:30.675480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435833668799734:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:30.675534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:30.675557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:30.675707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:30.675729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:30.686912Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:30.687015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435833668799728:2050] [node 3] ICS01 socket: 469 reason# EndOfStream
2025-11-19T13:59:30.687025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435833668799635:2050] [node 3] ICP30 unregister session Session# [5:7574435833668799728:2050] VirtualId# [5:7574435833668799725:0]
2025-11-19T13:59:30.687033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435833668799728:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:30.687083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:30.687109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:30.687303Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:30.687361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435834019815418:2050] [node 3] ICS01 socket: 487 reason# EndOfStream
2025-11-19T13:59:30.687365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435834019815324:2050] [node 3] ICP30 unregister session Session# [6:7574435834019815418:2050] VirtualId# [6:7574435834019815416:0]
2025-11-19T13:59:30.687369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435834019815418:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:30.687394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:30.687414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:30.715928Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:30.716034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435834019815422:2069] [node 4] ICS07 socket disconnect 474 reason# EndOfStream
2025-11-19T13:59:30.716043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435834019815422:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:30.716094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:30.716114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435834019815422:2069] [node 4] ICS15 start handshake
2025-11-19T13:59:30.717091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435834019815350:2069] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14659) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:30.717107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435834019815422:2069] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:30.717112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435834019815350:2069] [node 4] ICP30 unregister session Session# [6:7574435834019815422:2069] VirtualId# [6:7574435834019815415:0]
2025-11-19T13:59:30.717134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:30.717141Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435834019815350:2069] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14659) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:30.717144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14659) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:30.727945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435834019815350:2069] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:30.907971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435833096674037:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:30.907988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435833096674038:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:30.928170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435831831797690:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:30.928188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435831831797691:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:30.947901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435834204493738:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:30.947923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435834204493739:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:30.985508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435832446786107:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:30.985530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435832446786108:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:30.985534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435832446786109:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:31.019976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435834812537256:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:31.019992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435834812537257:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:31.019997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435834812537258:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:31.064084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435838933570089:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:31.064099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435838933570090:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:31.064103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435838933570091:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:31.132499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435839107504598:2092] [node 6] ICP01 ready to work
2025-11-19T13:59:31.133047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435837391641394:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.136138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435839107504602:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60412
2025-11-19T13:59:31.135993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435838933570109:2062] [node 5] ICP01 ready to work
2025-11-19T13:59:31.136453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435838933570121:2069] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.140001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838499461094:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50695
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.146172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435836741753464:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37127
2025-11-19T13:59:31.144142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435839107504613:2101] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.149045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838499461095:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50695
2025-11-19T13:59:31.154292Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838499461095:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.154762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435834204493739:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.159885Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838499461094:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.160144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435838499461098:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:31.160438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435838499461098:2109] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.160820Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435836126765049:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.160848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435836126765049:2108] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.164196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435837391641400:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:31.164273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435837391641401:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:31.165241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435837391641402:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18732
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.173863Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435836126765049:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:31.172645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435838933570109:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(36143137ns) {Configure:104(36142467ns) {ProcessConfigured:135(35761306ns) {StartInitialHandshake:157(35760735ns) {DropHandshakes:469(447ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(67ns) PrepareNewSessionHandshake:476(1818ns) {SwitchToState:178(1373ns)}}}}}
2025-11-19T13:59:31.173470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435838933570120:2077] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18287
2025-11-19T13:59:31.177887Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435837391641402:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.178490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435838499461103:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.179342Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435839107504605:2096] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.179360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435839107504605:2096] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.178060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435837391641413:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18732
2025-11-19T13:59:31.178219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435836741753465:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37127
2025-11-19T13:59:31.179122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838499461101:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50695
2025-11-19T13:59:31.179173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838499461102:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50695
2025-11-19T13:59:31.180448Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435836741753465:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.180896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435832446786107:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.181632Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435838499461095:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:31.181723Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435837391641406:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.181733Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435837391641406:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.183942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435831831797691:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.183963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:31.183971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435831831797691:2049] [node 3] ICP22 created new session: [2:7574435836126765057:2049]
2025-11-19T13:59:31.183979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435836126765057:2049] [node 3] ICS09 handshake done sender: [2:7574435836126765049:2108] self: [2:7574435836126765048:0] peer: [3:7574435838499461096:0] socket: 465 qp: -1
2025-11-19T13:59:31.183986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435836126765057:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:31.183995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:31.184302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435836126765059:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:31.184375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435836126765060:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:31.184449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435836126765061:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:31.184510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435836126765056:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60202
2025-11-19T13:59:31.187315Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435837391641406:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.194351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435836741753467:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37127
2025-11-19T13:59:31.195041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435839107504627:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60412
2025-11-19T13:59:31.195060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435839107504628:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60412
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.195177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435834204493739:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.195200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:31.195207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435834204493739:2049] [node 2] ICP22 created new session: [3:7574435838499461109:2049]
2025-11-19T13:59:31.195214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435838499461109:2049] [node 2] ICS09 handshake done sender: [3:7574435838499461095:2108] self: [3:7574435838499461096:0] peer: [2:7574435836126765048:0] socket: 466 qp: -1
2025-11-19T13:59:31.195220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435838499461109:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:31.195240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:31.195419Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838499461101:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.195567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435838499461111:2115] [node 6] ICP01 ready to work
2025-11-19T13:59:31.196034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435834204493738:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.198670Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435838933570120:2077] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.198767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435838933570109:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.198798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435838933570168:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18287
2025-11-19T13:59:31.198973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435837391641400:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.198993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:31.198999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435837391641400:2109] [node 4] ICP22 created new session: [1:7574435837391641416:2109]
2025-11-19T13:59:31.199004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435837391641416:2109] [node 4] ICS09 handshake done sender: [1:7574435837391641406:2113] self: [1:7574435837391641405:0] peer: [4:7574435836741753466:0] socket: 469 qp: -1
2025-11-19T13:59:31.199011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435837391641416:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:31.199020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:31.199685Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435839107504605:2096] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.200462Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435837391641404:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.200473Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435837391641404:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.201272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435837391641418:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18732
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.202366Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435837391641404:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:31.200214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435836741753468:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37127
2025-11-19T13:59:31.200286Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435836741753464:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.200352Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435836741753465:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:31.201456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435832446786107:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.201468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:31.201473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435832446786107:2048] [node 1] ICP22 created new session: [4:7574435836741753473:2048]
2025-11-19T13:59:31.201479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435836741753473:2048] [node 1] ICS09 handshake done sender: [4:7574435836741753465:2109] self: [4:7574435836741753466:0] peer: [1:7574435837391641405:0] socket: 470 qp: -1
2025-11-19T13:59:31.201485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435836741753473:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:31.201505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:31.201834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435836741753476:2114] [node 5] ICP01 ready to work
2025-11-19T13:59:31.201859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435836741753471:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37127
2025-11-19T13:59:31.202213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435836741753476:2114] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.202557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435833096674038:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.202570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:31.202576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435833096674038:2049] [node 3] ICP22 created new session: [1:7574435837391641421:2049]
2025-11-19T13:59:31.202581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435837391641421:2049] [node 3] ICS09 handshake done sender: [1:7574435837391641404:2112] self: [1:7574435837391641403:0] peer: [3:7574435838499461116:0] socket: 468 qp: -1
2025-11-19T13:59:31.202585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435837391641421:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:31.202590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.204550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435836126765070:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60202
2025-11-19T13:59:31.204572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435836126765071:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60202
2025-11-19T13:59:31.204593Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435839107504600:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.204604Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435839107504600:2093] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.206608Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435839107504628:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.206645Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435839107504602:2094] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.206673Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435839107504615:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.206676Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435839107504615:2103] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.207356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435834812537258:2050] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.207374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:31.207379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435834812537258:2050] [node 3] ICP22 created new session: [5:7574435839107504630:2050]
2025-11-19T13:59:31.207385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435839107504630:2050] [node 3] ICS09 handshake done sender: [5:7574435839107504605:2096] self: [5:7574435839107504604:0] peer: [3:7574435838499461099:0] socket: 460 qp: -1
2025-11-19T13:59:31.207392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435839107504630:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:31.207399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:31.204242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435836741753479:2115] [node 6] ICP01 ready to work
2025-11-19T13:59:31.208157Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435836741753468:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.209961Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435836126765056:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.210133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435832446786108:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.210156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435836741753484:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37127
2025-11-19T13:59:31.210420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838499461118:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50695
2025-11-19T13:59:31.210852Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435837391641413:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.211064Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435836126765071:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.211082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435831831797690:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.211171Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435837391641410:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.211176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435837391641410:2115] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.211640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435836126765074:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60202
2025-11-19T13:59:31.211721Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435837391641410:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:31.211922Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435836126765071:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:31.211988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435831831797690:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.212007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:31.212013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435831831797690:2048] [node 1] ICP22 created new session: [2:7574435836126765075:2048]
2025-11-19T13:59:31.212030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435836126765075:2048] [node 1] ICS09 handshake done sender: [2:7574435836126765071:2117] self: [2:7574435836126765073:0] peer: [1:7574435837391641409:0] socket: 498 qp: -1
2025-11-19T13:59:31.212037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435836126765075:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:31.212133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:31.213190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435833096674037:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.213209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:31.213213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435833096674037:2048] [node 2] ICP22 created new session: [1:7574435837391641426:2048]
2025-11-19T13:59:31.213218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435837391641426:2048] [node 2] ICS09 handshake done sender: [1:7574435837391641410:2115] self: [1:7574435837391641409:0] peer: [2:7574435836126765073:0] socket: 496 qp: -1
2025-11-19T13:59:31.213223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435837391641426:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:31.213229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:31.213544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435834812537257:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.213822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435834812537256:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.213870Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435839107504600:2093] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:31.218922Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435836126765063:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.218934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435836126765063:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.229801Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435839107504615:2103] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.231990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838499461119:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50695
2025-11-19T13:59:31.232952Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435836126765070:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.234642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435839107504598:2092] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.234663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:31.234673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435839107504598:2092] [node 6] ICP22 created new session: [5:7574435839107504636:2092]
2025-11-19T13:59:31.234679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435839107504636:2092] [node 6] ICS09 handshake done sender: [5:7574435839107504600:2093] self: [5:7574435839107504599:0] peer: [6:7574435838933570169:0] socket: 456 qp: -1
2025-11-19T13:59:31.234686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435839107504636:2092] [node 6] ICS10 traffic start
2025-11-19T13:59:31.234696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:31.237973Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435836126765063:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:31.243379Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435836126765065:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.243394Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435836126765065:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.243883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435836126765059:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.243915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:31.243925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435836126765059:2109] [node 4] ICP22 created new session: [2:7574435836126765080:2109]
2025-11-19T13:59:31.243932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435836126765080:2109] [node 4] ICS09 handshake done sender: [2:7574435836126765063:2113] self: [2:7574435836126765062:0] peer: [4:7574435836741753487:0] socket: 481 qp: -1
2025-11-19T13:59:31.243942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435836126765080:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:31.243958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:31.244687Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435836126765065:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:31.244776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435836126765060:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.244804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:31.244810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435836126765060:2110] [node 5] ICP22 created new session: [2:7574435836126765083:2110]
2025-11-19T13:59:31.244815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435836126765083:2110] [node 5] ICS09 handshake done sender: [2:7574435836126765065:2114] self: [2:7574435836126765064:0] peer: [5:7574435839107504634:0] socket: 482 qp: -1
2025-11-19T13:59:31.244820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435836126765083:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:31.244827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:31.243976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838499461120:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50695
2025-11-19T13:59:31.252206Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435837391641396:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.252228Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435837391641396:2108] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.255911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435839107504598:2092] [node 6] ICP35 event processing took too much time PendingConnection:278(21232977ns) {HandleHandshakeStatus:329(21231945ns) {DropIncomingHandshake:441(96ns) DropOutgoingHandshake:456(257ns) SwitchToState:178(180ns) ProcessPendingSessionEvents:495(21142868ns) {ForwardSessionEventToSession:612(21129772ns) ForwardSessionEventToSession:612(2319ns) ForwardSessionEventToSession:612(801ns) ForwardSessionEventToSession:612(1119ns) ForwardSessionEventToSession:612(846ns) ForwardSessionEventToSession:612(950ns) ForwardSessionEventToSession:612(887ns) ForwardSessionEventToSession:612(1451ns) ForwardSessionEventToSession:612(610ns)}}}
2025-11-19T13:59:31.256037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435839107504613:2101] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.256050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:31.256056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435839107504613:2101] [node 4] ICP22 created new session: [5:7574435839107504638:2101]
2025-11-19T13:59:31.256075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435839107504638:2101] [node 4] ICS09 handshake done sender: [5:7574435839107504615:2103] self: [5:7574435839107504614:0] peer: [4:7574435836741753477:0] socket: 462 qp: -1
2025-11-19T13:59:31.256084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435839107504638:2101] [node 4] ICS10 traffic start
2025-11-19T13:59:31.256095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:31.256348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435839107504640:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60412
2025-11-19T13:59:31.257214Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435837391641396:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:31.257311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435837391641394:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.257324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:31.257329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435837391641394:2107] [node 5] ICP22 created new session: [1:7574435837391641429:2107]
2025-11-19T13:59:31.257335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435837391641429:2107] [node 5] ICS09 handshake done sender: [1:7574435837391641396:2108] self: [1:7574435837391641395:0] peer: [5:7574435839107504635:0] socket: 457 qp: -1
2025-11-19T13:59:31.257340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435837391641429:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:31.257347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.257991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435838933570170:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18287
2025-11-19T13:59:31.258041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435838933570171:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18287
2025-11-19T13:59:31.257792Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435838499461094:2107] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:31.258180Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435836741753464:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:31.258311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435832446786108:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10987) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435836741753468:2111] held: no
2025-11-19T13:59:31.258333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435836741753476:2114] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.258343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:31.258349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435836741753476:2114] [node 5] ICP22 created new session: [4:7574435836741753489:2114]
2025-11-19T13:59:31.258354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435836741753489:2114] [node 5] ICS09 handshake done sender: [4:7574435836741753464:2108] self: [4:7574435836741753477:0] peer: [5:7574435839107504614:0] socket: 463 qp: -1
2025-11-19T13:59:31.258359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435836741753489:2114] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.259108Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435838499461101:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:31.258364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:31.258536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435836741753488:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37127
2025-11-19T13:59:31.258766Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435836741753468:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:31.258821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435832446786108:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.258827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:31.258832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435832446786108:2049] [node 2] ICP22 created new session: [4:7574435836741753491:2049]
2025-11-19T13:59:31.258836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435836741753491:2049] [node 2] ICS09 handshake done sender: [4:7574435836741753468:2111] self: [4:7574435836741753487:0] peer: [2:7574435836126765062:0] socket: 483 qp: -1
2025-11-19T13:59:31.258840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435836741753491:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:31.258846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:31.259244Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838499461120:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.259331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435838499461098:2109] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.259341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:31.259345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435838499461098:2109] [node 5] ICP22 created new session: [3:7574435838499461121:2109]
2025-11-19T13:59:31.259350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435838499461121:2109] [node 5] ICS09 handshake done sender: [3:7574435838499461094:2107] self: [3:7574435838499461099:0] peer: [5:7574435839107504604:0] socket: 461 qp: -1
2025-11-19T13:59:31.259354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435838499461121:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:31.259360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:31.259465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435834204493738:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.259472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:31.259475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435834204493738:2048] [node 1] ICP22 created new session: [3:7574435838499461123:2048]
2025-11-19T13:59:31.259479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435838499461123:2048] [node 1] ICS09 handshake done sender: [3:7574435838499461101:2111] self: [3:7574435838499461116:0] peer: [1:7574435837391641403:0] socket: 471 qp: -1
2025-11-19T13:59:31.259483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435838499461123:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:31.259487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:31.260422Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435837391641418:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.260525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435833096674038:2049] [node 3] ICP18 handshake (actor [1:7574435837391641418:2117]) is too old
2025-11-19T13:59:31.260955Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838499461107:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.260963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435838499461107:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.261739Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435838499461107:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:31.262877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435838499461103:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.262892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:31.262897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435838499461103:2110] [node 4] ICP22 created new session: [3:7574435838499461127:2110]
2025-11-19T13:59:31.262901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435838499461127:2110] [node 4] ICS09 handshake done sender: [3:7574435838499461107:2114] self: [3:7574435838499461106:0] peer: [4:7574435836741753493:0] socket: 488 qp: -1
2025-11-19T13:59:31.262906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435838499461127:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:31.262911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:31.260756Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435836741753471:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.260786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435832446786109:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.261668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435836741753494:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37127
2025-11-19T13:59:31.261874Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435836741753471:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:31.261911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435832446786109:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.261920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:31.261924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435832446786109:2050] [node 3] ICP22 created new session: [4:7574435836741753495:2050]
2025-11-19T13:59:31.261929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435836741753495:2050] [node 3] ICS09 handshake done sender: [4:7574435836741753471:2113] self: [4:7574435836741753493:0] peer: [3:7574435838499461106:0] socket: 492 qp: -1
2025-11-19T13:59:31.261933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435836741753495:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:31.261939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.264000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435838933570172:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18287
2025-11-19T13:59:31.264068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435838933570173:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18287
2025-11-19T13:59:31.264136Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435838933570170:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.264153Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435838933570120:2077] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:31.264383Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435839107504627:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.264536Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435839107504628:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:31.264645Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435838933570172:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.264658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435838933570109:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.264668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:31.264673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435838933570109:2062] [node 5] ICP22 created new session: [6:7574435838933570174:2062]
2025-11-19T13:59:31.264679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435838933570174:2062] [node 5] ICS09 handshake done sender: [6:7574435838933570120:2077] self: [6:7574435838933570169:0] peer: [5:7574435839107504599:0] socket: 459 qp: -1
2025-11-19T13:59:31.264683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435838933570174:2062] [node 5] ICS10 traffic start
2025-11-19T13:59:31.264689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:31.264803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435838933570089:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.265811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435839107504598:2092] [node 6] ICP18 handshake (actor [5:7574435839107504627:2114]) is too old
2025-11-19T13:59:31.265846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435834812537257:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.265853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:31.265857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435834812537257:2049] [node 2] ICP22 created new session: [5:7574435839107504642:2049]
2025-11-19T13:59:31.265861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435839107504642:2049] [node 2] ICS09 handshake done sender: [5:7574435839107504628:2115] self: [5:7574435839107504634:0] peer: [2:7574435836126765064:0] socket: 485 qp: -1
2025-11-19T13:59:31.265866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435839107504642:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:31.265870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:31.265972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435839107504641:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60412
2025-11-19T13:59:31.266426Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435839107504602:2094] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:31.266483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435834812537256:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.266494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:31.266498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435834812537256:2048] [node 1] ICP22 created new session: [5:7574435839107504644:2048]
2025-11-19T13:59:31.266503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435839107504644:2048] [node 1] ICS09 handshake done sender: [5:7574435839107504602:2094] self: [5:7574435839107504635:0] peer: [1:7574435837391641395:0] socket: 458 qp: -1
2025-11-19T13:59:31.266507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435839107504644:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:31.266512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:31.266968Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435838933570173:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.267000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435838933570090:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.267118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435838933570121:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.267242Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435836126765067:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.267252Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435836126765067:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.266972Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435837391641408:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.266981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435837391641408:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.268393Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435836126765067:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.268593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435838933570180:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18287
2025-11-19T13:59:31.268611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435838933570181:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18287
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.268513Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435836741753483:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.268523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435836741753483:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.269145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435836741753497:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37127
2025-11-19T13:59:31.269291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435836126765061:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.269307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:31.269312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435836126765061:2111] [node 6] ICP22 created new session: [2:7574435836126765086:2111]
2025-11-19T13:59:31.269317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435836126765086:2111] [node 6] ICS09 handshake done sender: [2:7574435836126765067:2115] self: [2:7574435836126765066:0] peer: [6:7574435838933570177:0] socket: 464 qp: -1
2025-11-19T13:59:31.269321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435836126765086:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:31.269327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:31.269606Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435837391641408:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:31.269684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435837391641401:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.269699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:31.269703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435837391641401:2110] [node 6] ICP22 created new session: [1:7574435837391641432:2110]
2025-11-19T13:59:31.269707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435837391641432:2110] [node 6] ICS09 handshake done sender: [1:7574435837391641408:2114] self: [1:7574435837391641407:0] peer: [6:7574435838933570176:0] socket: 494 qp: -1
2025-11-19T13:59:31.269711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435837391641432:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:31.269716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:31.273441Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435836741753483:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:31.273577Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435836741753497:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.273589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435836741753479:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.273597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:31.273603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435836741753479:2115] [node 6] ICP22 created new session: [4:7574435836741753499:2115]
2025-11-19T13:59:31.273607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435836741753499:2115] [node 6] ICS09 handshake done sender: [4:7574435836741753483:2117] self: [4:7574435836741753482:0] peer: [6:7574435838933570179:0] socket: 509 qp: -1
2025-11-19T13:59:31.273610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435836741753499:2115] [node 6] ICS10 traffic start
2025-11-19T13:59:31.273617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:31.273718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435836741753479:2115] [node 6] ICP18 handshake (actor [4:7574435836741753497:2121]) is too old
2025-11-19T13:59:31.273924Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435838933570168:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.273944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435838933570090:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10987) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435838933570172:2119] held: no
2025-11-19T13:59:31.273971Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435838933570089:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11337) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435838933570170:2117] held: no
2025-11-19T13:59:31.273991Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435838933570172:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:31.274082Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435838933570170:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.274222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435838933570090:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.274232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:31.274236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435838933570090:2049] [node 2] ICP22 created new session: [6:7574435838933570183:2049]
2025-11-19T13:59:31.274240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435838933570183:2049] [node 2] ICS09 handshake done sender: [6:7574435838933570172:2119] self: [6:7574435838933570177:0] peer: [2:7574435836126765066:0] socket: 512 qp: -1
2025-11-19T13:59:31.274244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435838933570183:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:31.274248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:31.274327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435838933570182:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18287
2025-11-19T13:59:31.275248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435838933570089:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.275259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:31.275264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435838933570089:2048] [node 1] ICP22 created new session: [6:7574435838933570185:2048]
2025-11-19T13:59:31.275268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435838933570185:2048] [node 1] ICS09 handshake done sender: [6:7574435838933570170:2117] self: [6:7574435838933570176:0] peer: [1:7574435837391641407:0] socket: 503 qp: -1
2025-11-19T13:59:31.275273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435838933570185:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:31.275279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:31.275358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435838933570091:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.275394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435838933570121:2069] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1937) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435838933570173:2120] held: no
2025-11-19T13:59:31.275522Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435838933570173:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:31.275627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435838933570121:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.275635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:31.275639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435838933570121:2069] [node 4] ICP22 created new session: [6:7574435838933570189:2069]
2025-11-19T13:59:31.275643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435838933570189:2069] [node 4] ICS09 handshake done sender: [6:7574435838933570173:2120] self: [6:7574435838933570179:0] peer: [4:7574435836741753482:0] socket: 514 qp: -1
2025-11-19T13:59:31.275646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435838933570189:2069] [node 4] ICS10 traffic start
2025-11-19T13:59:31.275650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.279967Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838499461115:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.279979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435838499461115:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.280660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838499461129:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50695
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.280885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435838933570191:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18287
2025-11-19T13:59:31.280981Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435838499461115:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:31.281127Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435838933570168:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:31.281178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435838933570091:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.281190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:31.281194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435838933570091:2050] [node 3] ICP22 created new session: [6:7574435838933570192:2050]
2025-11-19T13:59:31.281198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435838933570192:2050] [node 3] ICS09 handshake done sender: [6:7574435838933570168:2116] self: [6:7574435838933570188:0] peer: [3:7574435838499461114:0] socket: 490 qp: -1
2025-11-19T13:59:31.281203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435838933570192:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:31.281210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:31.281961Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838499461129:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.281973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435838499461111:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.281991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:31.281996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435838499461111:2115] [node 6] ICP22 created new session: [3:7574435838499461131:2115]
2025-11-19T13:59:31.282000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435838499461131:2115] [node 6] ICS09 handshake done sender: [3:7574435838499461115:2116] self: [3:7574435838499461114:0] peer: [6:7574435838933570188:0] socket: 489 qp: -1
2025-11-19T13:59:31.282003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435838499461131:2115] [node 6] ICS10 traffic start
2025-11-19T13:59:31.282008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:31.282134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435838499461111:2115] [node 6] ICP18 handshake (actor [3:7574435838499461129:2120]) is too old
2025-11-19T13:59:31.288499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:31.288581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574435836126765075:2048] [node 1] ICS07 socket disconnect 498 reason# EndOfStream
2025-11-19T13:59:31.288588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435836126765075:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.288619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:31.288636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435836126765075:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:31.295367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:31.295451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435838499461123:2048] [node 1] ICS01 socket: 471 reason# EndOfStream
2025-11-19T13:59:31.295460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435834204493738:2048] [node 1] ICP30 unregister session Session# [3:7574435838499461123:2048] VirtualId# [3:7574435838499461116:0]
2025-11-19T13:59:31.295468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435838499461123:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.295498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:31.295518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:31.296414Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435831831797690:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11337) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:31.296429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435836126765075:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:31.296692Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435831831797690:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11337) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:31.296702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435836126765075:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:31.296706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435831831797690:2048] [node 1] ICP30 unregister session Session# [2:7574435836126765075:2048] VirtualId# [2:7574435836126765073:0]
2025-11-19T13:59:31.296719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:31.296725Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574435831831797690:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:31.296727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:31.304111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7574435831831797690:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:31.322319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:31.322415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435838933570185:2048] [node 1] ICS01 socket: 503 reason# EndOfStream
2025-11-19T13:59:31.322425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435838933570089:2048] [node 1] ICP30 unregister session Session# [6:7574435838933570185:2048] VirtualId# [6:7574435838933570176:0]
2025-11-19T13:59:31.322433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435838933570185:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.322477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:31.322501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:31.324071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:31.324149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435839107504644:2048] [node 1] ICS01 socket: 458 reason# EndOfStream
2025-11-19T13:59:31.324157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435834812537256:2048] [node 1] ICP30 unregister session Session# [5:7574435839107504644:2048] VirtualId# [5:7574435839107504635:0]
2025-11-19T13:59:31.324162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435839107504644:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.324204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:31.324226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:31.324331Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:31.324391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435836741753473:2048] [node 1] ICS07 socket disconnect 470 reason# EndOfStream
2025-11-19T13:59:31.324396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435836741753473:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.324419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:31.324437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435836741753473:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:31.325027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435832446786107:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11337) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:31.325038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435836741753473:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:31.325042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435832446786107:2048] [node 1] ICP30 unregister session Session# [4:7574435836741753473:2048] VirtualId# [4:7574435836741753466:0]
2025-11-19T13:59:31.325055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:31.325061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435832446786107:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:31.325064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11337) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:31.329149Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:31.329252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435838499461109:2049] [node 2] ICS07 socket disconnect 466 reason# EndOfStream
2025-11-19T13:59:31.329258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435838499461109:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.329296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:31.329312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435838499461109:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:31.329809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435834204493739:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10987) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:31.329821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435838499461109:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:31.329826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435834204493739:2049] [node 2] ICP30 unregister session Session# [3:7574435838499461109:2049] VirtualId# [3:7574435838499461096:0]
2025-11-19T13:59:31.329838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:31.329842Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435834204493739:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10987) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:31.329846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10987) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:31.335921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435832446786107:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:31.350316Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:31.350420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435838933570183:2049] [node 2] ICS01 socket: 512 reason# EndOfStream
2025-11-19T13:59:31.350430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435838933570090:2049] [node 2] ICP30 unregister session Session# [6:7574435838933570183:2049] VirtualId# [6:7574435838933570177:0]
2025-11-19T13:59:31.350436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435838933570183:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.350478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:31.350500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:31.350637Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:31.350697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435836741753491:2049] [node 2] ICS01 socket: 483 reason# EndOfStream
2025-11-19T13:59:31.350701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435832446786108:2049] [node 2] ICP30 unregister session Session# [4:7574435836741753491:2049] VirtualId# [4:7574435836741753487:0]
2025-11-19T13:59:31.350705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435836741753491:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.350728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:31.350745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:31.352148Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:31.352244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435839107504642:2049] [node 2] ICS01 socket: 485 reason# EndOfStream
2025-11-19T13:59:31.352252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435834812537257:2049] [node 2] ICP30 unregister session Session# [5:7574435839107504642:2049] VirtualId# [5:7574435839107504634:0]
2025-11-19T13:59:31.352258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435839107504642:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.352309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:31.352335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:31.360440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:31.360550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435839107504630:2050] [node 3] ICS01 socket: 460 reason# EndOfStream
2025-11-19T13:59:31.360559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435834812537258:2050] [node 3] ICP30 unregister session Session# [5:7574435839107504630:2050] VirtualId# [5:7574435839107504604:0]
2025-11-19T13:59:31.360565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435839107504630:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.360618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:31.360638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:31.364024Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:31.364098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435838933570192:2050] [node 3] ICS01 socket: 490 reason# EndOfStream
2025-11-19T13:59:31.364106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435838933570091:2050] [node 3] ICP30 unregister session Session# [6:7574435838933570192:2050] VirtualId# [6:7574435838933570188:0]
2025-11-19T13:59:31.364112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435838933570192:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.364156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:31.364180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:31.370068Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:31.370158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435838933570189:2069] [node 4] ICS01 socket: 514 reason# EndOfStream
2025-11-19T13:59:31.370167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435838933570121:2069] [node 4] ICP30 unregister session Session# [6:7574435838933570189:2069] VirtualId# [6:7574435838933570179:0]
2025-11-19T13:59:31.370174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435838933570189:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.370219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:31.370241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:31.552438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435836903930057:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:31.552460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435836903930058:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:31.603936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435839245221301:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:31.603964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435839245221302:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:31.648349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435838557174691:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:31.648364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435838557174692:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:31.687915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435838542696742:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:31.687935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435838542696743:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:31.687940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435838542696744:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:31.711952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435836613782681:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:31.711973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435836613782682:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:31.711979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435836613782683:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:31.728396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435839680345742:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:31.728410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435839680345743:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:31.728414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435839680345744:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:31.764700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435836903930119:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:31.764791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435836903930120:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:31.764858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435839680345766:2066] [node 4] ICP01 ready to work
2025-11-19T13:59:31.765238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435839680345772:2071] [node 5] ICP01 ready to work
2025-11-19T13:59:31.764509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435838557174752:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.766497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435838542696802:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29751
2025-11-19T13:59:31.766520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435838542696803:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29751
2025-11-19T13:59:31.766531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435838542696804:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29751
2025-11-19T13:59:31.764797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435836613782706:2066] [node 6] ICP01 ready to work
2025-11-19T13:59:31.765148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435836613782711:2072] [node 4] ICP01 ready to work
2025-11-19T13:59:31.766395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435836613782722:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47910
2025-11-19T13:59:31.771970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435836903930133:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.774330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838557174756:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58391
2025-11-19T13:59:31.774350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838557174757:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58391
2025-11-19T13:59:31.776429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435838557174759:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.780341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435839245221368:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:31.780432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435839245221369:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:31.780495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435839245221370:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:31.781456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435839245221365:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58736
2025-11-19T13:59:31.781468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435839245221366:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58736
2025-11-19T13:59:31.781478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435839245221367:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58736
2025-11-19T13:59:31.782159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435839680345806:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49214
2025-11-19T13:59:31.782540Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435839245221367:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.795815Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435839245221365:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.799073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435836613782753:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47910
2025-11-19T13:59:31.799329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435836613782681:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24359648ns) {Configure:104(24358872ns) {ProcessConfigured:135(24017408ns) {StartInitialHandshake:157(24016801ns) {DropHandshakes:469(644ns) {DropIncomingHandshake:441(92ns) DropOutgoingHandshake:456(149ns)} GenerateSessionVirtualId:988(159ns) PrepareNewSessionHandshake:476(454ns) {SwitchToState:178(388ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.799901Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435838542696743:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24336628ns) {Configure:104(24335629ns) {ProcessConfigured:135(23961907ns) {StartInitialHandshake:157(23961344ns) {DropHandshakes:469(537ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(95ns)} GenerateSessionVirtualId:988(408ns) PrepareNewSessionHandshake:476(1116ns) {SwitchToState:178(548ns)}}}}}
2025-11-19T13:59:31.800101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435838542696811:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:31.800139Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435838542696804:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.800267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435838542696813:2112] [node 6] ICP01 ready to work
2025-11-19T13:59:31.801120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435838542696742:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.801145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435838542696812:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29751
2025-11-19T13:59:31.801512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435838542696744:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(25430613ns) {Configure:104(25429263ns) {ProcessConfigured:135(25214939ns) {StartInitialHandshake:157(25214699ns) {DropHandshakes:469(220ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(78ns) PrepareNewSessionHandshake:476(725ns) {SwitchToState:178(184ns)}}}}}
2025-11-19T13:59:31.802086Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435839680345806:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.802124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435839680345772:2071] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.801289Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435836613782753:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.801424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435836613782682:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.802489Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435838542696803:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.802511Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435838542696812:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.803369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435838542696743:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.811847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435838557174691:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(27453584ns) {Configure:104(27452712ns) {ProcessConfigured:135(27114952ns) {StartInitialHandshake:157(27114318ns) {DropHandshakes:469(671ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(1809ns) {SwitchToState:178(940ns)}}}}}
2025-11-19T13:59:31.812445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435838557174766:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:31.812498Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838557174757:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.814143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435838557174691:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.814174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838557174767:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58391
2025-11-19T13:59:31.814815Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435836903930126:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.814831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435836903930126:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.815991Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435836613782716:2077] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.816006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435836613782716:2077] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.820036Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435836903930124:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.820051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435836903930124:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.822597Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435838542696802:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.822642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435838542696744:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.831805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435836903930136:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45576
2025-11-19T13:59:31.832011Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435839245221366:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.832049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435839245221372:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.832052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435839245221372:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.836394Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435836903930136:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.836656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435838557174759:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(60166140ns) {Configure:104(60165599ns) {ProcessConfigured:135(59891820ns) {StartInitialHandshake:157(59890795ns) {DropHandshakes:469(175ns) {DropIncomingHandshake:441(46ns) DropOutgoingHandshake:456(24ns)} GenerateSessionVirtualId:988(139ns) PrepareNewSessionHandshake:476(1000ns) {SwitchToState:178(497ns)}}}}}
2025-11-19T13:59:31.836709Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838557174754:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.836713Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435838557174754:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.837839Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435836903930124:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.838168Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435838557174754:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:31.840949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435838557174752:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.840965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:31.840974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435838557174752:2107] [node 4] ICP22 created new session: [3:7574435838557174776:2107]
2025-11-19T13:59:31.840981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435838557174776:2107] [node 4] ICS09 handshake done sender: [3:7574435838557174754:2108] self: [3:7574435838557174753:0] peer: [4:7574435838542696827:0] socket: 462 qp: -1
2025-11-19T13:59:31.840990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435838557174776:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:31.841003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:31.841256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838557174775:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58391
2025-11-19T13:59:31.840878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435836903930137:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45576
2025-11-19T13:59:31.840997Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435836903930126:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.841603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435836613782762:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47910
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.842090Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435839245221376:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.842101Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435839245221376:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.842877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435839245221384:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58736
2025-11-19T13:59:31.842909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435839245221385:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58736
2025-11-19T13:59:31.843469Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435839245221301:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(62892964ns) {Configure:104(62892108ns) {ProcessConfigured:135(57117349ns) {StartInitialHandshake:157(57116843ns) {DropHandshakes:469(771ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(181ns)} GenerateSessionVirtualId:988(291ns) PrepareNewSessionHandshake:476(801ns) {SwitchToState:178(280ns)}}}}}
2025-11-19T13:59:31.843533Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435839245221372:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.845038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435838542696828:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29751
2025-11-19T13:59:31.844122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435836903930058:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.844140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:31.844147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435836903930058:2049] [node 3] ICP22 created new session: [1:7574435836903930141:2049]
2025-11-19T13:59:31.844154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435836903930141:2049] [node 3] ICS09 handshake done sender: [1:7574435836903930124:2110] self: [1:7574435836903930123:0] peer: [3:7574435838557174770:0] socket: 468 qp: -1
2025-11-19T13:59:31.844161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435836903930141:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:31.844169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:31.844308Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435836903930137:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.844315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435836903930119:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.844326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:31.844332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435836903930119:2107] [node 4] ICP22 created new session: [1:7574435836903930143:2107]
2025-11-19T13:59:31.844336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435836903930143:2107] [node 4] ICS09 handshake done sender: [1:7574435836903930126:2111] self: [1:7574435836903930125:0] peer: [4:7574435838542696820:0] socket: 470 qp: -1
2025-11-19T13:59:31.844340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435836903930143:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:31.844345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:31.844642Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838557174756:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.844668Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838557174767:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.844856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435838557174692:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.845350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435838542696829:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29751
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.845401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435838542696830:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29751
2025-11-19T13:59:31.845415Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435838542696804:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:31.845470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435838542696742:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.845482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:31.845487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435838542696742:2048] [node 1] ICP22 created new session: [4:7574435838542696831:2048]
2025-11-19T13:59:31.845492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435838542696831:2048] [node 1] ICS09 handshake done sender: [4:7574435838542696804:2109] self: [4:7574435838542696820:0] peer: [1:7574435836903930125:0] socket: 475 qp: -1
2025-11-19T13:59:31.845497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435838542696831:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:31.845503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.846847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435839680345822:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49214
2025-11-19T13:59:31.847109Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435838542696817:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.847117Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435838542696817:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.847622Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435838542696802:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:31.846565Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435836613782762:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.846596Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435836613782722:2079] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.846683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435836613782711:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.847879Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435838557174757:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:31.848072Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435839680345806:2102] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:31.848263Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435839245221376:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:31.847975Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435836613782716:2077] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:31.853124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435836903930058:2049] [node 3] ICP18 handshake (actor [1:7574435836903930137:2116]) is too old
2025-11-19T13:59:31.853385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435839245221301:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.853424Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435839245221384:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.853431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435839245221368:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.853444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:31.853456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435839245221368:2107] [node 5] ICP22 created new session: [2:7574435839245221390:2107]
2025-11-19T13:59:31.853463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435839245221390:2107] [node 5] ICS09 handshake done sender: [2:7574435839245221372:2113] self: [2:7574435839245221371:0] peer: [5:7574435836613782755:0] socket: 482 qp: -1
2025-11-19T13:59:31.853469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435839245221390:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:31.853485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:31.856336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435839680345772:2071] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.856355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:31.856362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435839680345772:2071] [node 5] ICP22 created new session: [6:7574435839680345823:2071]
2025-11-19T13:59:31.856368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435839680345823:2071] [node 5] ICS09 handshake done sender: [6:7574435839680345806:2102] self: [6:7574435839680345821:0] peer: [5:7574435836613782715:0] socket: 478 qp: -1
2025-11-19T13:59:31.856375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435839680345823:2071] [node 5] ICS10 traffic start
2025-11-19T13:59:31.856382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.858505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435839680345825:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49214
2025-11-19T13:59:31.859889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435838557174691:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.859909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:31.859917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435838557174691:2048] [node 1] ICP22 created new session: [3:7574435838557174781:2048]
2025-11-19T13:59:31.859924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435838557174781:2048] [node 1] ICS09 handshake done sender: [3:7574435838557174757:2110] self: [3:7574435838557174770:0] peer: [1:7574435836903930123:0] socket: 479 qp: -1
2025-11-19T13:59:31.859930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435838557174781:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:31.859937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.863417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435836903930147:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45576
2025-11-19T13:59:31.863493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435836613782706:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.863514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:31.863519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435836613782706:2066] [node 6] ICP22 created new session: [5:7574435836613782766:2066]
2025-11-19T13:59:31.863524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435836613782766:2066] [node 6] ICS09 handshake done sender: [5:7574435836613782716:2077] self: [5:7574435836613782715:0] peer: [6:7574435839680345821:0] socket: 471 qp: -1
2025-11-19T13:59:31.863530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435836613782766:2066] [node 6] ICS10 traffic start
2025-11-19T13:59:31.863536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:31.863991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435838542696744:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.864012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:31.864019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435838542696744:2050] [node 3] ICP22 created new session: [4:7574435838542696835:2050]
2025-11-19T13:59:31.864025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435838542696835:2050] [node 3] ICS09 handshake done sender: [4:7574435838542696802:2107] self: [4:7574435838542696827:0] peer: [3:7574435838557174753:0] socket: 473 qp: -1
2025-11-19T13:59:31.864029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435838542696835:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:31.864034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.864802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435839680345827:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49214
2025-11-19T13:59:31.867302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435838557174691:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.867669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435839680345828:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49214
2025-11-19T13:59:31.868017Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435838542696817:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:31.868307Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435839680345825:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.868341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435839680345742:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.868188Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435839245221385:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.872169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435836613782769:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47910
2025-11-19T13:59:31.872306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435836613782770:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47910
2025-11-19T13:59:31.872353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435836613782771:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47910
2025-11-19T13:59:31.872383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435836613782772:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47910
2025-11-19T13:59:31.872805Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435836613782770:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.872823Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435836613782771:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.872840Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435836613782762:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:31.876320Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435839245221374:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.876348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435839245221374:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.879817Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435836903930135:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.879840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435836903930135:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.880378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435839245221368:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(26911710ns) {HandleHandshakeStatus:329(26911052ns) {DropIncomingHandshake:441(2878ns) DropOutgoingHandshake:456(568ns) SwitchToState:178(41ns) ProcessPendingSessionEvents:495(11865363ns) {ForwardSessionEventToSession:612(11850760ns) ForwardSessionEventToSession:612(2529ns) ForwardSessionEventToSession:612(884ns) ForwardSessionEventToSession:612(689ns) ForwardSessionEventToSession:612(849ns) ForwardSessionEventToSession:612(950ns) ForwardSessionEventToSession:612(599ns)}}}
2025-11-19T13:59:31.880404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435839245221369:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.880420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:31.880427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435839245221369:2108] [node 4] ICP22 created new session: [2:7574435839245221392:2108]
2025-11-19T13:59:31.880432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435839245221392:2108] [node 4] ICS09 handshake done sender: [2:7574435839245221376:2115] self: [2:7574435839245221375:0] peer: [4:7574435838542696826:0] socket: 484 qp: -1
2025-11-19T13:59:31.880442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435839245221392:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:31.880452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:31.883199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435839245221369:2108] [node 4] ICP18 handshake (actor [2:7574435839245221384:2118]) is too old
2025-11-19T13:59:31.880487Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435836903930147:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.880890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435838542696811:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.880908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:31.880915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435838542696811:2111] [node 5] ICP22 created new session: [4:7574435838542696837:2111]
2025-11-19T13:59:31.880920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435838542696837:2111] [node 5] ICS09 handshake done sender: [4:7574435838542696817:2116] self: [4:7574435838542696816:0] peer: [5:7574435836613782764:0] socket: 489 qp: -1
2025-11-19T13:59:31.880925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435838542696837:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:31.880931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:31.881035Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435838542696812:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:31.882333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435838542696743:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.882342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:31.882348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435838542696743:2049] [node 2] ICP22 created new session: [4:7574435838542696839:2049]
2025-11-19T13:59:31.882354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435838542696839:2049] [node 2] ICS09 handshake done sender: [4:7574435838542696812:2114] self: [4:7574435838542696826:0] peer: [2:7574435839245221375:0] socket: 486 qp: -1
2025-11-19T13:59:31.882359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435838542696839:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:31.882364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:31.884111Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435836903930122:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.884128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435836903930122:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.889945Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435839245221374:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:31.890071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435839245221302:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.890094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:31.890102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435839245221302:2049] [node 3] ICP22 created new session: [2:7574435839245221398:2049]
2025-11-19T13:59:31.890108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435839245221398:2049] [node 3] ICS09 handshake done sender: [2:7574435839245221374:2114] self: [2:7574435839245221373:0] peer: [3:7574435838557174780:0] socket: 483 qp: -1
2025-11-19T13:59:31.890114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435839245221398:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:31.890122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:31.891930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435836613782681:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.891959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435836613782711:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.891971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:31.891979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435836613782711:2072] [node 4] ICP22 created new session: [5:7574435836613782774:2072]
2025-11-19T13:59:31.891984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435836613782774:2072] [node 4] ICS09 handshake done sender: [5:7574435836613782762:2117] self: [5:7574435836613782764:0] peer: [4:7574435838542696816:0] socket: 492 qp: -1
2025-11-19T13:59:31.891992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435836613782774:2072] [node 4] ICS10 traffic start
2025-11-19T13:59:31.892000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.893356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435836903930149:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45576
2025-11-19T13:59:31.894172Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435836903930135:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:31.893594Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435838557174692:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28901) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435838557174767:2115] held: no
2025-11-19T13:59:31.900409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435836903930133:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.900433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:31.900440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435836903930133:2113] [node 6] ICP22 created new session: [1:7574435836903930151:2113]
2025-11-19T13:59:31.900447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435836903930151:2113] [node 6] ICS09 handshake done sender: [1:7574435836903930135:2114] self: [1:7574435836903930134:0] peer: [6:7574435839680345829:0] socket: 490 qp: -1
2025-11-19T13:59:31.900455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435836903930151:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:31.900465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:31.900911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435836613782683:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.900978Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435836613782753:2109] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.902082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435839245221400:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58736
2025-11-19T13:59:31.901394Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435836903930149:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.901426Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435836903930131:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.901429Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435836903930131:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.902607Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435836903930122:2109] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.903370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435838542696841:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29751
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.903591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435839680345742:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2226) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435839680345825:2117] held: no
2025-11-19T13:59:31.903658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435839680345830:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49214
2025-11-19T13:59:31.903698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435839680345831:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49214
2025-11-19T13:59:31.903076Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435839245221367:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:31.908010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838557174784:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58391
2025-11-19T13:59:31.908354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435839245221301:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.908376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:31.908383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435839245221301:2048] [node 1] ICP22 created new session: [2:7574435839245221401:2048]
2025-11-19T13:59:31.908389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435839245221401:2048] [node 1] ICS09 handshake done sender: [2:7574435839245221367:2112] self: [2:7574435839245221389:0] peer: [1:7574435836903930121:0] socket: 481 qp: -1
2025-11-19T13:59:31.908396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435839245221401:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:31.908403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:31.911159Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435838557174767:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:31.911414Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435836903930131:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:31.911961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435836903930057:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.911992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:31.911999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435836903930057:2048] [node 2] ICP22 created new session: [1:7574435836903930156:2048]
2025-11-19T13:59:31.912005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435836903930156:2048] [node 2] ICS09 handshake done sender: [1:7574435836903930122:2109] self: [1:7574435836903930121:0] peer: [2:7574435839245221389:0] socket: 466 qp: -1
2025-11-19T13:59:31.912011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435836903930156:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:31.912019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:31.916053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435838557174692:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.916071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:31.916080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435838557174692:2049] [node 2] ICP22 created new session: [3:7574435838557174785:2049]
2025-11-19T13:59:31.916086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435838557174785:2049] [node 2] ICS09 handshake done sender: [3:7574435838557174767:2115] self: [3:7574435838557174780:0] peer: [2:7574435839245221373:0] socket: 493 qp: -1
2025-11-19T13:59:31.916092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435838557174785:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:31.916099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:31.916307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435836903930120:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.916318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:31.916323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435836903930120:2108] [node 5] ICP22 created new session: [1:7574435836903930158:2108]
2025-11-19T13:59:31.916327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435836903930158:2108] [node 5] ICS09 handshake done sender: [1:7574435836903930131:2112] self: [1:7574435836903930127:0] peer: [5:7574435836613782773:0] socket: 488 qp: -1
2025-11-19T13:59:31.916331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435836903930158:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:31.916337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:31.916592Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435839680345827:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.916632Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435839680345830:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.916646Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435839680345828:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.916668Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435839680345825:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:31.917078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435839680345742:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.917090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:31.917094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435839680345742:2048] [node 1] ICP22 created new session: [6:7574435839680345832:2048]
2025-11-19T13:59:31.917099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435839680345832:2048] [node 1] ICS09 handshake done sender: [6:7574435839680345825:2117] self: [6:7574435839680345829:0] peer: [1:7574435836903930134:0] socket: 496 qp: -1
2025-11-19T13:59:31.917103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435839680345832:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:31.917108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:31.917195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435839680345766:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.917206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435839680345743:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.917213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435839680345744:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:31.917432Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435838542696819:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.917446Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435838542696819:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.918327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435836613782682:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28901) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435836613782753:2109] held: no
2025-11-19T13:59:31.918354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435836613782682:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.918361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:31.918366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435836613782682:2049] [node 2] ICP22 created new session: [5:7574435836613782777:2049]
2025-11-19T13:59:31.918370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435836613782777:2049] [node 2] ICS09 handshake done sender: [5:7574435836613782753:2109] self: [5:7574435836613782755:0] peer: [2:7574435839245221371:0] socket: 485 qp: -1
2025-11-19T13:59:31.918375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435836613782777:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:31.918381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.918869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435839680345838:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49214
2025-11-19T13:59:31.919211Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435838542696819:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:31.919509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435838542696813:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.919524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:31.919529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435838542696813:2112] [node 6] ICP22 created new session: [4:7574435838542696843:2112]
2025-11-19T13:59:31.919535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435838542696843:2112] [node 6] ICS09 handshake done sender: [4:7574435838542696819:2117] self: [4:7574435838542696818:0] peer: [6:7574435839680345835:0] socket: 513 qp: -1
2025-11-19T13:59:31.919539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435838542696843:2112] [node 6] ICS10 traffic start
2025-11-19T13:59:31.919544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:31.919635Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435838542696811:2111] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:31.924096Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435839245221378:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.924110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435839245221378:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.924171Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838557174773:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.924181Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435838557174773:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.927832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435836613782779:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47910
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.930751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435836613782781:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47910
2025-11-19T13:59:31.931029Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435836613782771:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:31.931601Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435836613782770:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:31.932329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435836613782681:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.932342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:31.932347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435836613782681:2048] [node 1] ICP22 created new session: [5:7574435836613782782:2048]
2025-11-19T13:59:31.932351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435836613782782:2048] [node 1] ICS09 handshake done sender: [5:7574435836613782771:2120] self: [5:7574435836613782773:0] peer: [1:7574435836903930127:0] socket: 518 qp: -1
2025-11-19T13:59:31.932355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435836613782782:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:31.932361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:31.933065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435836613782683:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.933077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:31.933082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435836613782683:2050] [node 3] ICP22 created new session: [5:7574435836613782784:2050]
2025-11-19T13:59:31.933087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435836613782784:2050] [node 3] ICS09 handshake done sender: [5:7574435836613782770:2119] self: [5:7574435836613782776:0] peer: [3:7574435838557174768:0] socket: 517 qp: -1
2025-11-19T13:59:31.933091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435836613782784:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:31.933095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.929293Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838557174769:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.929304Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435838557174769:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:31.930848Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435838557174773:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:31.931210Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435838557174769:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:31.932005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435838557174759:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.932013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:31.932017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435838557174759:2111] [node 6] ICP22 created new session: [3:7574435838557174790:2111]
2025-11-19T13:59:31.932021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435838557174790:2111] [node 6] ICS09 handshake done sender: [3:7574435838557174773:2117] self: [3:7574435838557174760:0] peer: [6:7574435839680345837:0] socket: 494 qp: -1
2025-11-19T13:59:31.932025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435838557174790:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:31.932029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: 2025-11-19T13:59:31.932105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435838557174766:2112] [node 5] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-11-19T13:59:31.932113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:31.932116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435838557174766:2112] [node 5] ICP22 created new session: [3:7574435838557174792:2112]
2025-11-19T13:59:31.932120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435838557174792:2112] [node 5] ICS09 handshake done sender: [3:7574435838557174769:2116] self: [3:7574435838557174768:0] peer: [5:7574435836613782776:0] socket: 506 qp: -1
2025-11-19T13:59:31.932123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435838557174792:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:31.932129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:31.934171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435838557174789:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58391
2025-11-19T13:59:31.930422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435839680345766:2066] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14196) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435839680345827:2118] held: no
2025-11-19T13:59:31.930473Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435839680345827:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:31.931717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435839680345766:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.931725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:31.931730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435839680345766:2066] [node 4] ICP22 created new session: [6:7574435839680345841:2066]
2025-11-19T13:59:31.931735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435839680345841:2066] [node 4] ICS09 handshake done sender: [6:7574435839680345827:2118] self: [6:7574435839680345835:0] peer: [4:7574435838542696818:0] socket: 514 qp: -1
2025-11-19T13:59:31.931739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435839680345841:2066] [node 4] ICS10 traffic start
2025-11-19T13:59:31.931754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:31.934045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435839680345839:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49214
2025-11-19T13:59:31.930555Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435839245221378:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:31.931269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435839245221370:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:31.931283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:31.931289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435839245221370:2109] [node 6] ICP22 created new session: [2:7574435839245221404:2109]
2025-11-19T13:59:31.931294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435839245221404:2109] [node 6] ICS09 handshake done sender: [2:7574435839245221378:2116] self: [2:7574435839245221377:0] peer: [6:7574435839680345836:0] socket: 480 qp: -1
2025-11-19T13:59:31.931299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435839245221404:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:31.931307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:31.930652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435836903930160:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45576
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:31.934486Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435836903930160:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.934371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435839680345840:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49214
2025-11-19T13:59:31.934638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435836903930120:2108] [node 5] ICP18 handshake (actor [1:7574435836903930160:2119]) is too old
2025-11-19T13:59:31.935355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435839680345743:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28901) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435839680345830:2120] held: no
2025-11-19T13:59:31.935381Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435839680345830:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:31.939320Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435839680345828:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:31.935429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435838557174789:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:31.935455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435838557174759:2111] [node 6] ICP18 handshake (actor [3:7574435838557174789:2120]) is too old
2025-11-19T13:59:31.940074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435839680345743:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.940095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:31.940102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435839680345743:2049] [node 2] ICP22 created new session: [6:7574435839680345843:2049]
2025-11-19T13:59:31.940108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435839680345843:2049] [node 2] ICS09 handshake done sender: [6:7574435839680345830:2120] self: [6:7574435839680345836:0] peer: [2:7574435839245221377:0] socket: 487 qp: -1
2025-11-19T13:59:31.940115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435839680345843:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:31.940115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435839680345744:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:31.940125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:31.940130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:31.940138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435839680345744:2050] [node 3] ICP22 created new session: [6:7574435839680345845:2050]
2025-11-19T13:59:31.940144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435839680345845:2050] [node 3] ICS09 handshake done sender: [6:7574435839680345828:2119] self: [6:7574435839680345837:0] peer: [3:7574435838557174760:0] socket: 515 qp: -1
2025-11-19T13:59:31.940152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435839680345845:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:31.940160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:31.940253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435839680345744:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:31.952093Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:31.952199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435838557174781:2048] [node 1] ICS01 socket: 479 reason# EndOfStream
2025-11-19T13:59:31.952209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435838557174691:2048] [node 1] ICP30 unregister session Session# [3:7574435838557174781:2048] VirtualId# [3:7574435838557174770:0]
2025-11-19T13:59:31.952217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435838557174781:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.952274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:31.952299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:31.952299Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:31.952375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435839245221401:2048] [node 1] ICS01 socket: 481 reason# EndOfStream
2025-11-19T13:59:31.952382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435839245221301:2048] [node 1] ICP30 unregister session Session# [2:7574435839245221401:2048] VirtualId# [2:7574435839245221389:0]
2025-11-19T13:59:31.952388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435839245221401:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.952433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:31.952456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:31.956026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:31.956112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435838542696831:2048] [node 1] ICS07 socket disconnect 475 reason# EndOfStream
2025-11-19T13:59:31.956120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435838542696831:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.956163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:31.956183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435838542696831:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:31.956743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:31.956804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435839680345832:2048] [node 1] ICS01 socket: 496 reason# EndOfStream
2025-11-19T13:59:31.956809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435839680345742:2048] [node 1] ICP30 unregister session Session# [6:7574435839680345832:2048] VirtualId# [6:7574435839680345829:0]
2025-11-19T13:59:31.956814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435839680345832:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.956851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:31.956874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:31.956649Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435838542696742:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2226) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:31.956678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435838542696831:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:31.956684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435838542696742:2048] [node 1] ICP30 unregister session Session# [4:7574435838542696831:2048] VirtualId# [4:7574435838542696820:0]
2025-11-19T13:59:31.956701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:31.956709Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435838542696742:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2226) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:31.956713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2226) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:31.959852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:31.959931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435836613782782:2048] [node 1] ICS01 socket: 518 reason# EndOfStream
2025-11-19T13:59:31.959937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435836613782681:2048] [node 1] ICP30 unregister session Session# [5:7574435836613782782:2048] VirtualId# [5:7574435836613782773:0]
2025-11-19T13:59:31.959943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435836613782782:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.959983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:31.960005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:31.964263Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:31.964329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435838557174785:2049] [node 2] ICS01 socket: 493 reason# EndOfStream
2025-11-19T13:59:31.964335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435838557174692:2049] [node 2] ICP30 unregister session Session# [3:7574435838557174785:2049] VirtualId# [3:7574435838557174780:0]
2025-11-19T13:59:31.964340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435838557174785:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.964367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:31.964383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:31.967878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435838542696742:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:31.986349Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:31.986457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435836613782777:2049] [node 2] ICS01 socket: 485 reason# EndOfStream
2025-11-19T13:59:31.986468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435836613782682:2049] [node 2] ICP30 unregister session Session# [5:7574435836613782777:2049] VirtualId# [5:7574435836613782755:0]
2025-11-19T13:59:31.986478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435836613782777:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.986524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:31.986546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:31.987908Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:31.987998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435838542696839:2049] [node 2] ICS01 socket: 486 reason# EndOfStream
2025-11-19T13:59:31.988005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435838542696743:2049] [node 2] ICP30 unregister session Session# [4:7574435838542696839:2049] VirtualId# [4:7574435838542696826:0]
2025-11-19T13:59:31.988010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435838542696839:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.988051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:31.988072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:31.990198Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:31.990270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435839680345843:2049] [node 2] ICS01 socket: 487 reason# EndOfStream
2025-11-19T13:59:31.990276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435839680345743:2049] [node 2] ICP30 unregister session Session# [6:7574435839680345843:2049] VirtualId# [6:7574435839680345836:0]
2025-11-19T13:59:31.990282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435839680345843:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.990326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:31.990351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:31.998564Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:31.998650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435836613782784:2050] [node 3] ICS01 socket: 517 reason# EndOfStream
2025-11-19T13:59:31.998661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435836613782683:2050] [node 3] ICP30 unregister session Session# [5:7574435836613782784:2050] VirtualId# [5:7574435836613782776:0]
2025-11-19T13:59:31.998667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435836613782784:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.998708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:31.998727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:31.998812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:31.998866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435839680345845:2050] [node 3] ICS01 socket: 515 reason# EndOfStream
2025-11-19T13:59:31.998870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435839680345744:2050] [node 3] ICP30 unregister session Session# [6:7574435839680345845:2050] VirtualId# [6:7574435839680345837:0]
2025-11-19T13:59:31.998873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435839680345845:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:31.998897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:31.998913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:32.011899Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:32.012000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435839680345841:2066] [node 4] ICS01 socket: 514 reason# EndOfStream
2025-11-19T13:59:32.012012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435839680345766:2066] [node 4] ICP30 unregister session Session# [6:7574435839680345841:2066] VirtualId# [6:7574435839680345835:0]
2025-11-19T13:59:32.012020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435839680345841:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:32.012067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:32.012089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:32.132200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435842040466188:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:32.132228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435842040466189:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:32.220270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435841242910733:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:32.220297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435841242910734:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:32.303874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435843104024355:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:32.303874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435843104024356:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:32.388568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435840997423556:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:32.388586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435840997423557:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:32.388591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435840997423558:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:32.447891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435842848880417:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:32.447905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435842848880418:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:32.447910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435842848880419:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:32.551882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435842575314302:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:32.551904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435842575314303:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:32.551909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435842575314304:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.592518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435842040466248:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2677
2025-11-19T13:59:32.592539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435842040466249:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2677
2025-11-19T13:59:32.592291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435840997423595:2085] [node 5] ICP01 ready to work
2025-11-19T13:59:32.592519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435843104024406:2095] [node 4] ICP01 ready to work
2025-11-19T13:59:32.592102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435842848880445:2068] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.595729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435841242910792:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62730
2025-11-19T13:59:32.594224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435840997423602:2088] [node 6] ICP01 ready to work
2025-11-19T13:59:32.596056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435842575314327:2067] [node 5] ICP01 ready to work
2025-11-19T13:59:32.598114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435842848880462:2080] [node 4] ICP01 ready to work
2025-11-19T13:59:32.596096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435842040466254:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:32.596743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435842040466259:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:32.598998Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435842040466248:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.601834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435842575314347:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6467
2025-11-19T13:59:32.604811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435843104024417:2102] [node 5] ICP01 ready to work
2025-11-19T13:59:32.604891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435843104024418:2103] [node 6] ICP01 ready to work
2025-11-19T13:59:32.605020Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435842575314347:2084] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.608426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435840997423625:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2132
2025-11-19T13:59:32.609757Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435842040466249:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.616383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435842040466251:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.621648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435841242910797:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:32.621939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435841242910805:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:32.622707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435841242910802:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62730
2025-11-19T13:59:32.622765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435841242910803:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62730
2025-11-19T13:59:32.622775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435841242910804:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62730
2025-11-19T13:59:32.626007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435842575314355:2061] [node 4] ICP01 ready to work
2025-11-19T13:59:32.625956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435841242910734:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(28803944ns) {Configure:104(28803585ns) {ProcessConfigured:135(28576862ns) {StartInitialHandshake:157(28576599ns) {DropHandshakes:469(348ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:476(1298ns) {SwitchToState:178(25ns)}}}}}
2025-11-19T13:59:32.626008Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435841242910802:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.631792Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435841242910792:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.632723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435842848880493:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46678
2025-11-19T13:59:32.632783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435842848880494:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46678
2025-11-19T13:59:32.632814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435842848880495:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46678
2025-11-19T13:59:32.633557Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435842848880495:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.633594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435842848880417:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:32.633712Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435842040466261:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.633724Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435842040466261:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.634470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435843104024434:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6716
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.634528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435843104024435:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6716
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.634561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435843104024436:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6716
2025-11-19T13:59:32.632104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435841242910801:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:32.636077Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435842848880494:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.636098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435842848880493:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.636130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435842848880462:2080] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:32.636432Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435842040466261:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.637767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435842040466254:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.637791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:32.637797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435842040466254:2110] [node 5] ICP22 created new session: [1:7574435842040466273:2110]
2025-11-19T13:59:32.637804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435842040466273:2110] [node 5] ICS09 handshake done sender: [1:7574435842040466261:2114] self: [1:7574435842040466260:0] peer: [5:7574435842848880496:0] socket: 485 qp: -1
2025-11-19T13:59:32.637811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435842040466273:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:32.637820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:32.637943Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435841242910803:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.637970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435841242910733:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:32.637256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435840997423635:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2132
2025-11-19T13:59:32.639146Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435840997423597:2086] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.639158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435840997423597:2086] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:32.637159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435842575314377:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6467
2025-11-19T13:59:32.641488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435842575314355:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:32.644325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435840997423611:2095] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.644337Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435840997423611:2095] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.645651Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435843104024435:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.645680Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435843104024434:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.645695Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435843104024436:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.645747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435843104024355:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.646019Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435842040466256:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.646028Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435842040466256:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:32.646513Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435842040466258:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.646516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435842040466258:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:32.647105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435842040466272:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2677
2025-11-19T13:59:32.647624Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435840997423611:2095] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:32.648866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435842848880501:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46678
2025-11-19T13:59:32.649963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435842848880417:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29962) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435842848880495:2115] held: no
2025-11-19T13:59:32.650017Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435842848880495:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:32.651821Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435842040466256:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:32.651952Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435842040466272:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.651974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435842040466188:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.651978Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435842040466258:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:32.651990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:32.651996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435842040466188:2048] [node 2] ICP22 created new session: [1:7574435842040466279:2048]
2025-11-19T13:59:32.652001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435842040466279:2048] [node 2] ICS09 handshake done sender: [1:7574435842040466256:2112] self: [1:7574435842040466255:0] peer: [2:7574435841242910816:0] socket: 483 qp: -1
2025-11-19T13:59:32.652007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435842040466279:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:32.652015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.657167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435842575314384:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6467
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.657664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435841242910819:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62730
2025-11-19T13:59:32.657741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435843104024440:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6716
2025-11-19T13:59:32.657884Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435840997423597:2086] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:32.658075Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435842040466254:2110] [node 5] ICP35 event processing took too much time PendingConnection:278(20280001ns) {HandleHandshakeStatus:329(20278722ns) {DropIncomingHandshake:441(3932ns) DropOutgoingHandshake:456(338ns) SwitchToState:178(250ns) ProcessPendingSessionEvents:495(10607296ns) {ForwardSessionEventToSession:612(10596135ns) ForwardSessionEventToSession:612(1948ns) ForwardSessionEventToSession:612(872ns) ForwardSessionEventToSession:612(781ns) ForwardSessionEventToSession:612(587ns) ForwardSessionEventToSession:612(1310ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(517ns) ForwardSessionEventToSession:612(473ns) ForwardSessionEventToSession:612(809ns)}}}
2025-11-19T13:59:32.658269Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435843104024436:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:32.658336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435843104024355:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.658352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:32.658359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435843104024355:2048] [node 1] ICP22 created new session: [3:7574435843104024441:2048]
2025-11-19T13:59:32.658364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435843104024441:2048] [node 1] ICS09 handshake done sender: [3:7574435843104024436:2117] self: [3:7574435843104024439:0] peer: [1:7574435842040466257:0] socket: 503 qp: -1
2025-11-19T13:59:32.658369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435843104024441:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:32.658386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:32.658526Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435841242910804:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.658577Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435841242910803:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:32.662071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435842575314383:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6467
2025-11-19T13:59:32.662433Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435842575314347:2084] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:32.662494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435842575314355:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.662517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:32.662525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435842575314355:2061] [node 4] ICP22 created new session: [6:7574435842575314385:2061]
2025-11-19T13:59:32.662531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435842575314385:2061] [node 4] ICS09 handshake done sender: [6:7574435842575314347:2084] self: [6:7574435842575314380:0] peer: [4:7574435840997423610:0] socket: 487 qp: -1
2025-11-19T13:59:32.662538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435842575314385:2061] [node 4] ICS10 traffic start
2025-11-19T13:59:32.662548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:32.664196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435842848880417:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.664218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:32.664227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435842848880417:2048] [node 1] ICP22 created new session: [5:7574435842848880502:2048]
2025-11-19T13:59:32.664235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435842848880502:2048] [node 1] ICS09 handshake done sender: [5:7574435842848880495:2115] self: [5:7574435842848880496:0] peer: [1:7574435842040466260:0] socket: 500 qp: -1
2025-11-19T13:59:32.664242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435842848880502:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:32.664250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:32.664206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435840997423602:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.664230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:32.664240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435840997423602:2088] [node 6] ICP22 created new session: [4:7574435840997423639:2088]
2025-11-19T13:59:32.664247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435840997423639:2088] [node 6] ICS09 handshake done sender: [4:7574435840997423611:2095] self: [4:7574435840997423610:0] peer: [6:7574435842575314380:0] socket: 486 qp: -1
2025-11-19T13:59:32.664253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435840997423639:2088] [node 6] ICS10 traffic start
2025-11-19T13:59:32.664262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.667852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435841242910733:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.667878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:32.667885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435841242910733:2048] [node 1] ICP22 created new session: [2:7574435841242910820:2048]
2025-11-19T13:59:32.667891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435841242910820:2048] [node 1] ICS09 handshake done sender: [2:7574435841242910803:2113] self: [2:7574435841242910816:0] peer: [1:7574435842040466255:0] socket: 492 qp: -1
2025-11-19T13:59:32.667898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435841242910820:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:32.667919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.669494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435842040466278:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2677
2025-11-19T13:59:32.669537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435842040466189:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.669547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:32.669555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435842040466189:2049] [node 3] ICP22 created new session: [1:7574435842040466282:2049]
2025-11-19T13:59:32.669561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435842040466282:2049] [node 3] ICS09 handshake done sender: [1:7574435842040466258:2113] self: [1:7574435842040466257:0] peer: [3:7574435843104024439:0] socket: 484 qp: -1
2025-11-19T13:59:32.669566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435842040466282:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:32.669574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:32.669626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435842040466189:2049] [node 3] ICP18 handshake (actor [1:7574435842040466272:2117]) is too old
2025-11-19T13:59:32.670065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435842040466281:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2677
2025-11-19T13:59:32.670183Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435842575314377:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.670260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435842575314303:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:32.670259Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435840997423635:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.671373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435840997423557:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:32.671548Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435842040466188:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(19540216ns) {HandleHandshakeStatus:329(19539932ns) {DropIncomingHandshake:441(158ns) DropOutgoingHandshake:456(280ns) SwitchToState:178(73ns) ProcessPendingSessionEvents:495(13266342ns) {ForwardSessionEventToSession:612(13255936ns) ForwardSessionEventToSession:612(1426ns) ForwardSessionEventToSession:612(826ns) ForwardSessionEventToSession:612(527ns) ForwardSessionEventToSession:612(560ns) ForwardSessionEventToSession:612(1353ns) ForwardSessionEventToSession:612(559ns) ForwardSessionEventToSession:612(494ns) ForwardSessionEventToSession:612(535ns) ForwardSessionEventToSession:612(1073ns)}}}
2025-11-19T13:59:32.672815Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435842575314384:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.672857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435842575314302:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.672999Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435841242910810:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.673007Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435841242910810:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:32.673668Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435841242910808:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.673671Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435841242910808:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:32.675418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435841242910824:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62730
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.675681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435842575314390:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6467
2025-11-19T13:59:32.676948Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435841242910810:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:32.677162Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435841242910808:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:32.677233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435841242910805:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.677250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:32.677254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435841242910805:2110] [node 6] ICP22 created new session: [2:7574435841242910827:2110]
2025-11-19T13:59:32.677259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435841242910827:2110] [node 6] ICS09 handshake done sender: [2:7574435841242910810:2116] self: [2:7574435841242910809:0] peer: [6:7574435842575314388:0] socket: 496 qp: -1
2025-11-19T13:59:32.677264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435841242910827:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:32.677271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:32.677357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435841242910797:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.677361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:32.677364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435841242910797:2107] [node 4] ICP22 created new session: [2:7574435841242910829:2107]
2025-11-19T13:59:32.677367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435841242910829:2107] [node 4] ICS09 handshake done sender: [2:7574435841242910808:2115] self: [2:7574435841242910807:0] peer: [4:7574435840997423642:0] socket: 495 qp: -1
2025-11-19T13:59:32.677381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435841242910829:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:32.677386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.676846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435840997423595:2085] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.676869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:32.676877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435840997423595:2085] [node 5] ICP22 created new session: [4:7574435840997423643:2085]
2025-11-19T13:59:32.676883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435840997423643:2085] [node 5] ICS09 handshake done sender: [4:7574435840997423597:2086] self: [4:7574435840997423596:0] peer: [5:7574435842848880498:0] socket: 472 qp: -1
2025-11-19T13:59:32.676890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435840997423643:2085] [node 5] ICS10 traffic start
2025-11-19T13:59:32.676898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:32.676978Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435840997423595:2085] [node 5] ICP35 event processing took too much time PendingConnection:278(18461737ns) {HandleHandshakeStatus:329(18461261ns) {DropIncomingHandshake:441(538ns) DropOutgoingHandshake:456(407ns) SwitchToState:178(380ns) ProcessPendingSessionEvents:495(24763ns) {ForwardSessionEventToSession:612(15895ns) ForwardSessionEventToSession:612(827ns) ForwardSessionEventToSession:612(849ns) ForwardSessionEventToSession:612(582ns) ForwardSessionEventToSession:612(525ns) ForwardSessionEventToSession:612(698ns) ForwardSessionEventToSession:612(571ns) ForwardSessionEventToSession:612(1045ns) ForwardSessionEventToSession:612(514ns) ForwardSessionEventToSession:612(1034ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.678401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435843104024443:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6716
2025-11-19T13:59:32.678442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435843104024444:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6716
2025-11-19T13:59:32.679999Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435843104024355:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(21630676ns) {HandleHandshakeStatus:329(21630242ns) {DropIncomingHandshake:441(320ns) DropOutgoingHandshake:456(1989ns) SwitchToState:178(195ns) ProcessPendingSessionEvents:495(15526701ns) {ForwardSessionEventToSession:612(15513562ns) ForwardSessionEventToSession:612(2772ns) ForwardSessionEventToSession:612(1205ns) ForwardSessionEventToSession:612(678ns) ForwardSessionEventToSession:612(914ns) ForwardSessionEventToSession:612(962ns) ForwardSessionEventToSession:612(823ns) ForwardSessionEventToSession:612(538ns) ForwardSessionEventToSession:612(669ns) ForwardSessionEventToSession:612(845ns)}}}
2025-11-19T13:59:32.681011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435842848880504:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46678
2025-11-19T13:59:32.681255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435842848880505:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46678
2025-11-19T13:59:32.681234Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435842040466263:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.681256Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435842040466263:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:32.682197Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435842575314303:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435842575314377:2112] held: no
2025-11-19T13:59:32.682233Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435842575314377:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-11-19T13:59:32.684316Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435842848880493:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:32.684144Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435840997423625:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.684336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435840997423602:2088] [node 6] ICP35 event processing took too much time PendingConnection:278(36575730ns) {HandleHandshakeStatus:329(36574981ns) {DropIncomingHandshake:441(485ns) DropOutgoingHandshake:456(551ns) SwitchToState:178(232ns) ProcessPendingSessionEvents:495(7474414ns) {ForwardSessionEventToSession:612(7463531ns) ForwardSessionEventToSession:612(1155ns) ForwardSessionEventToSession:612(1271ns) ForwardSessionEventToSession:612(586ns) ForwardSessionEventToSession:612(622ns) ForwardSessionEventToSession:612(862ns) ForwardSessionEventToSession:612(583ns) ForwardSessionEventToSession:612(531ns) ForwardSessionEventToSession:612(581ns) ForwardSessionEventToSession:612(1072ns)}}}
::ffff:127.1.0.0
2025-11-19T13:59:32.682395Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435843104024444:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.683888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435843104024356:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:32.684392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435840997423646:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2132
2025-11-19T13:59:32.684470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435842575314303:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.684494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:32.684503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435842575314303:2049] [node 2] ICP22 created new session: [6:7574435842575314391:2049]
2025-11-19T13:59:32.684508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435842575314391:2049] [node 2] ICS09 handshake done sender: [6:7574435842575314377:2112] self: [6:7574435842575314388:0] peer: [2:7574435841242910809:0] socket: 497 qp: -1
2025-11-19T13:59:32.684515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435842575314391:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:32.684528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:32.684500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435842848880462:2080] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.684515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:32.684520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435842848880462:2080] [node 4] ICP22 created new session: [5:7574435842848880507:2080]
2025-11-19T13:59:32.684526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435842848880507:2080] [node 4] ICS09 handshake done sender: [5:7574435842848880493:2113] self: [5:7574435842848880498:0] peer: [4:7574435840997423596:0] socket: 476 qp: -1
2025-11-19T13:59:32.684532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435842848880507:2080] [node 4] ICS10 traffic start
2025-11-19T13:59:32.684538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:32.684927Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435842040466281:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.685002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435842040466188:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:32.685061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435840997423558:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:32.685235Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435840997423635:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:32.685263Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435843104024443:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.685341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435842848880506:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46678
2025-11-19T13:59:32.685370Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435842040466263:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:32.685461Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435842848880505:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.685507Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435842848880506:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.685532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435842848880418:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:32.686396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435840997423645:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2132
2025-11-19T13:59:32.686447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435840997423557:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.686460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:32.686465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435840997423557:2049] [node 2] ICP22 created new session: [4:7574435840997423648:2049]
2025-11-19T13:59:32.686471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435840997423648:2049] [node 2] ICS09 handshake done sender: [4:7574435840997423635:2116] self: [4:7574435840997423642:0] peer: [2:7574435841242910807:0] socket: 498 qp: -1
2025-11-19T13:59:32.686477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435840997423648:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:32.686484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:32.686655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435842848880419:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:32.687605Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435841242910813:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.687619Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435841242910813:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:32.688103Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435843104024416:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.688114Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435843104024416:2104] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:32.688944Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435843104024416:2104] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:32.688691Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435841242910824:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.688767Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435841242910817:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.688770Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435841242910817:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:32.689250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435841242910797:2107] [node 4] ICP18 handshake (actor [2:7574435841242910824:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.690565Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435842040466269:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.690576Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435842040466269:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:32.692459Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435841242910817:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:32.692574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435841242910801:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.692594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:32.692601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435841242910801:2108] [node 5] ICP22 created new session: [2:7574435841242910833:2108]
2025-11-19T13:59:32.692606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435841242910833:2108] [node 5] ICS09 handshake done sender: [2:7574435841242910817:2118] self: [2:7574435841242910815:0] peer: [5:7574435842848880510:0] socket: 478 qp: -1
2025-11-19T13:59:32.692613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435841242910833:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:32.692620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:32.692937Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435842040466269:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:32.693035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435842040466251:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.693054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:32.693060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435842040466251:2109] [node 4] ICP22 created new session: [1:7574435842040466288:2109]
2025-11-19T13:59:32.693065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435842040466288:2109] [node 4] ICS09 handshake done sender: [1:7574435842040466269:2116] self: [1:7574435842040466268:0] peer: [4:7574435840997423651:0] socket: 517 qp: -1
2025-11-19T13:59:32.693069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435842040466288:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:32.693077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:32.693288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435843104024406:2095] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.693307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:32.693311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435843104024406:2095] [node 4] ICP22 created new session: [3:7574435843104024449:2095]
2025-11-19T13:59:32.693315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435843104024449:2095] [node 4] ICS09 handshake done sender: [3:7574435843104024416:2104] self: [3:7574435843104024407:0] peer: [4:7574435840997423647:0] socket: 488 qp: -1
2025-11-19T13:59:32.693319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435843104024449:2095] [node 4] ICS10 traffic start
2025-11-19T13:59:32.693323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.690216Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435840997423645:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.690290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435840997423556:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:32.690307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435840997423650:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2132
2025-11-19T13:59:32.690509Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435840997423625:2106] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:32.691443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435840997423558:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.691455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:32.691460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435840997423558:2050] [node 3] ICP22 created new session: [4:7574435840997423653:2050]
2025-11-19T13:59:32.691466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435840997423653:2050] [node 3] ICS09 handshake done sender: [4:7574435840997423625:2106] self: [4:7574435840997423647:0] peer: [3:7574435843104024407:0] socket: 490 qp: -1
2025-11-19T13:59:32.691473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435840997423653:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:32.691478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:32.695996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435840997423652:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2132
2025-11-19T13:59:32.695982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435842575314393:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6467
2025-11-19T13:59:32.696033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435842575314394:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6467
2025-11-19T13:59:32.696048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435842575314395:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6467
2025-11-19T13:59:32.696291Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435842575314384:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:32.696392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435842575314302:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.696406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:32.696411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435842575314302:2048] [node 1] ICP22 created new session: [6:7574435842575314396:2048]
2025-11-19T13:59:32.696420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435842575314396:2048] [node 1] ICS09 handshake done sender: [6:7574435842575314384:2118] self: [6:7574435842575314389:0] peer: [1:7574435842040466262:0] socket: 514 qp: -1
2025-11-19T13:59:32.696425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435842575314396:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:32.696431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:32.697225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435843104024422:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.697238Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435843104024422:2106] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:32.697006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435840997423556:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29962) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435840997423645:2117] held: no
2025-11-19T13:59:32.701138Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435840997423645:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:32.701237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435840997423556:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.701249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:32.701255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435840997423556:2048] [node 1] ICP22 created new session: [4:7574435840997423655:2048]
2025-11-19T13:59:32.701260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435840997423655:2048] [node 1] ICS09 handshake done sender: [4:7574435840997423645:2117] self: [4:7574435840997423651:0] peer: [1:7574435842040466268:0] socket: 475 qp: -1
2025-11-19T13:59:32.701265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435840997423655:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:32.701272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:32.701654Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435842575314395:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.701687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435842575314304:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:32.701785Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435843104024424:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.701796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435843104024424:2107] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.702605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435842575314399:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6467
2025-11-19T13:59:32.702690Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435843104024424:2107] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:32.702850Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435842575314395:2122] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:32.702910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435842575314304:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.702923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:32.702928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435842575314304:2050] [node 3] ICP22 created new session: [6:7574435842575314400:2050]
2025-11-19T13:59:32.702933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435842575314400:2050] [node 3] ICS09 handshake done sender: [6:7574435842575314395:2122] self: [6:7574435842575314398:0] peer: [3:7574435843104024423:0] socket: 513 qp: -1
2025-11-19T13:59:32.702937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435842575314400:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:32.702943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:32.704854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435843104024418:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.704874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:32.704882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435843104024418:2103] [node 6] ICP22 created new session: [3:7574435843104024453:2103]
2025-11-19T13:59:32.704891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435843104024453:2103] [node 6] ICS09 handshake done sender: [3:7574435843104024424:2107] self: [3:7574435843104024423:0] peer: [6:7574435842575314398:0] socket: 479 qp: -1
2025-11-19T13:59:32.704897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435843104024453:2103] [node 6] ICS10 traffic start
2025-11-19T13:59:32.704904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:32.705609Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435842848880418:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435842848880505:2120] held: no
2025-11-19T13:59:32.705686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435842040466259:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.705712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:32.705718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435842040466259:2111] [node 6] ICP22 created new session: [1:7574435842040466290:2111]
2025-11-19T13:59:32.705723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435842040466290:2111] [node 6] ICS09 handshake done sender: [1:7574435842040466263:2115] self: [1:7574435842040466262:0] peer: [6:7574435842575314389:0] socket: 508 qp: -1
2025-11-19T13:59:32.705728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435842040466290:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:32.705737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:32.705829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435842040466259:2111] [node 6] ICP35 event processing took too much time PendingConnection:278(18273623ns) {HandleHandshakeStatus:329(18272699ns) {DropIncomingHandshake:441(9939ns) DropOutgoingHandshake:456(480ns) SwitchToState:178(239ns) ProcessPendingSessionEvents:495(17255ns) {ForwardSessionEventToSession:612(10011ns) ForwardSessionEventToSession:612(509ns) ForwardSessionEventToSession:612(608ns) ForwardSessionEventToSession:612(592ns) ForwardSessionEventToSession:612(499ns) ForwardSessionEventToSession:612(953ns) ForwardSessionEventToSession:612(587ns) ForwardSessionEventToSession:612(580ns) ForwardSessionEventToSession:612(614ns) ForwardSessionEventToSession:612(720ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.707018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435842848880512:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46678
2025-11-19T13:59:32.707107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435842848880513:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46678
2025-11-19T13:59:32.707444Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435842848880505:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:32.707532Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435842575314394:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.707562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435842575314327:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:32.707652Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435842848880454:2078] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:32.707657Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435842848880454:2078] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:32.707908Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435841242910813:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:32.707910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435842848880418:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.707922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:32.708004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435841242910734:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.708023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:32.708029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435841242910734:2049] [node 3] ICP22 created new session: [2:7574435841242910836:2049]
2025-11-19T13:59:32.708034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435841242910836:2049] [node 3] ICS09 handshake done sender: [2:7574435841242910813:2117] self: [2:7574435841242910798:0] peer: [3:7574435843104024446:0] socket: 519 qp: -1
2025-11-19T13:59:32.708039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435841242910836:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:32.708047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:32.707947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435842848880418:2049] [node 2] ICP22 created new session: [5:7574435842848880514:2049]
2025-11-19T13:59:32.707952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435842848880514:2049] [node 2] ICS09 handshake done sender: [5:7574435842848880505:2120] self: [5:7574435842848880510:0] peer: [2:7574435841242910815:0] socket: 526 qp: -1
2025-11-19T13:59:32.707958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435842848880514:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:32.707963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:32.708827Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435843104024422:2106] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.708951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435842575314403:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6467
2025-11-19T13:59:32.709085Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435842848880454:2078] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:32.709494Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435842575314394:2121] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:32.709751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435842848880445:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.709768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:32.709773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435842848880445:2068] [node 6] ICP22 created new session: [5:7574435842848880517:2068]
2025-11-19T13:59:32.709778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435842848880517:2068] [node 6] ICS09 handshake done sender: [5:7574435842848880454:2078] self: [5:7574435842848880453:0] peer: [6:7574435842575314402:0] socket: 528 qp: -1
2025-11-19T13:59:32.709782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435842848880517:2068] [node 6] ICS10 traffic start
2025-11-19T13:59:32.709787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:32.710102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435842575314327:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.710117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:32.710122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435842575314327:2067] [node 5] ICP22 created new session: [6:7574435842575314404:2067]
2025-11-19T13:59:32.710126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435842575314404:2067] [node 5] ICS09 handshake done sender: [6:7574435842575314394:2121] self: [6:7574435842575314402:0] peer: [5:7574435842848880453:0] socket: 502 qp: -1
2025-11-19T13:59:32.710131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435842575314404:2067] [node 5] ICS10 traffic start
2025-11-19T13:59:32.710135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:32.710951Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435842848880506:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:32.711102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435842848880419:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.711113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:32.711117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435842848880419:2050] [node 3] ICP22 created new session: [5:7574435842848880519:2050]
2025-11-19T13:59:32.711122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435842848880519:2050] [node 3] ICS09 handshake done sender: [5:7574435842848880506:2121] self: [5:7574435842848880511:0] peer: [3:7574435843104024421:0] socket: 527 qp: -1
2025-11-19T13:59:32.711127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435842848880519:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:32.711132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:32.712715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435843104024417:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:32.712733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:32.712739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435843104024417:2102] [node 5] ICP22 created new session: [3:7574435843104024456:2102]
2025-11-19T13:59:32.712746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435843104024456:2102] [node 5] ICS09 handshake done sender: [3:7574435843104024422:2106] self: [3:7574435843104024421:0] peer: [5:7574435842848880511:0] socket: 525 qp: -1
2025-11-19T13:59:32.712750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435843104024456:2102] [node 5] ICS10 traffic start
2025-11-19T13:59:32.712757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:32.712888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435843104024356:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435843104024444:2120] held: no
2025-11-19T13:59:32.713084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435843104024458:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6716
2025-11-19T13:59:32.713456Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435843104024444:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:32.713513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435843104024356:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:32.713525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:32.713529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435843104024356:2049] [node 2] ICP22 created new session: [3:7574435843104024459:2049]
2025-11-19T13:59:32.713534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435843104024459:2049] [node 2] ICS09 handshake done sender: [3:7574435843104024444:2120] self: [3:7574435843104024446:0] peer: [2:7574435841242910798:0] socket: 494 qp: -1
2025-11-19T13:59:32.713539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435843104024459:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:32.713543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:32.729148Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:32.729254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435841242910820:2048] [node 1] ICS01 socket: 492 reason# EndOfStream
2025-11-19T13:59:32.729265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435841242910733:2048] [node 1] ICP30 unregister session Session# [2:7574435841242910820:2048] VirtualId# [2:7574435841242910816:0]
2025-11-19T13:59:32.729272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435841242910820:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:32.729304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:32.729324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:32.729734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:32.729797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435843104024441:2048] [node 1] ICS07 socket disconnect 503 reason# EndOfStream
2025-11-19T13:59:32.729803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435843104024441:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:32.729826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:32.729840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435843104024441:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:32.731763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435843104024355:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29962) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:32.731777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435843104024441:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:32.733761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:32.733837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435842575314396:2048] [node 1] ICS01 socket: 514 reason# EndOfStream
2025-11-19T13:59:32.733844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435842575314302:2048] [node 1] ICP30 unregister session Session# [6:7574435842575314396:2048] VirtualId# [6:7574435842575314389:0]
2025-11-19T13:59:32.733850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435842575314396:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:32.733887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:32.733909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:32.735431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435843104024355:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29962) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:32.735445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435843104024441:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:32.735449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435843104024355:2048] [node 1] ICP30 unregister session Session# [3:7574435843104024441:2048] VirtualId# [3:7574435843104024439:0]
2025-11-19T13:59:32.735464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:32.735471Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435843104024355:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29962) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:32.735474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29962) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:32.735902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:32.735982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435840997423655:2048] [node 1] ICS01 socket: 475 reason# EndOfStream
2025-11-19T13:59:32.735989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435840997423556:2048] [node 1] ICP30 unregister session Session# [4:7574435840997423655:2048] VirtualId# [4:7574435840997423651:0]
2025-11-19T13:59:32.735994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435840997423655:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:32.736033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:32.736059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:32.736090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:32.736149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435842848880502:2048] [node 1] ICS01 socket: 500 reason# EndOfStream
2025-11-19T13:59:32.736153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435842848880417:2048] [node 1] ICP30 unregister session Session# [5:7574435842848880502:2048] VirtualId# [5:7574435842848880496:0]
2025-11-19T13:59:32.736157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435842848880502:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:32.736181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:32.736201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:32.742983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574435843104024355:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:32.744583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:32.744670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435843104024459:2049] [node 2] ICS01 socket: 494 reason# EndOfStream
2025-11-19T13:59:32.744680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435843104024356:2049] [node 2] ICP30 unregister session Session# [3:7574435843104024459:2049] VirtualId# [3:7574435843104024446:0]
2025-11-19T13:59:32.744686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435843104024459:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:32.744716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:32.744735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:32.771872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:32.771956Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:32.772046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435842848880514:2049] [node 2] ICS01 socket: 526 reason# EndOfStream
2025-11-19T13:59:32.772054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435842848880418:2049] [node 2] ICP30 unregister session Session# [5:7574435842848880514:2049] VirtualId# [5:7574435842848880510:0]
2025-11-19T13:59:32.771872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:32.772060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435842848880514:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:32.772106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:32.772127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:32.772018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435842575314391:2049] [node 2] ICS01 socket: 497 reason# EndOfStream
2025-11-19T13:59:32.772032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435842575314303:2049] [node 2] ICP30 unregister session Session# [6:7574435842575314391:2049] VirtualId# [6:7574435842575314388:0]
2025-11-19T13:59:32.772041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435842575314391:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:32.772092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:32.772114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:32.775016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435840997423648:2049] [node 2] ICS07 socket disconnect 498 reason# EndOfStream
2025-11-19T13:59:32.775029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435840997423648:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:32.775071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:32.775091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435840997423648:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:32.775450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435840997423557:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2805) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:32.775463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435840997423648:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:32.775468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435840997423557:2049] [node 2] ICP30 unregister session Session# [4:7574435840997423648:2049] VirtualId# [4:7574435840997423642:0]
2025-11-19T13:59:32.775486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:32.775493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435840997423557:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2805) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:32.775496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2805) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:32.789561Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:32.789668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435842848880519:2050] [node 3] ICS01 socket: 527 reason# EndOfStream
2025-11-19T13:59:32.789679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435842848880419:2050] [node 3] ICP30 unregister session Session# [5:7574435842848880519:2050] VirtualId# [5:7574435842848880511:0]
2025-11-19T13:59:32.789687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435842848880519:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:32.789733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:32.789758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:32.791123Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:32.791196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435842575314400:2050] [node 3] ICS01 socket: 513 reason# EndOfStream
2025-11-19T13:59:32.791203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435842575314304:2050] [node 3] ICP30 unregister session Session# [6:7574435842575314400:2050] VirtualId# [6:7574435842575314398:0]
2025-11-19T13:59:32.791208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435842575314400:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:32.791251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:32.791271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:32.816983Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:32.817095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435842575314385:2061] [node 4] ICS01 socket: 487 reason# EndOfStream
2025-11-19T13:59:32.817109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435842575314355:2061] [node 4] ICP30 unregister session Session# [6:7574435842575314385:2061] VirtualId# [6:7574435842575314380:0]
2025-11-19T13:59:32.817119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435842575314385:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:32.817169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:32.817193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:32.912010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435844279160013:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:32.912023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435844279160014:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:32.939876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435843783291305:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:32.939888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435843783291306:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:32.971864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435844517197108:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:32.971882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435844517197109:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:32.971886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435844517197110:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:32.956909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435844324421409:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:32.956933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435844324421410:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:32.987872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435841846889532:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:32.987883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435841846889533:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:32.987888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435841846889534:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:33.003851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435847022885148:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:33.003854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435847022885147:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:33.003863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435847022885149:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:33.025770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435848574127368:2105] [node 4] ICP01 ready to work
2025-11-19T13:59:33.028089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435848619388744:2084] [node 6] ICP01 ready to work
2025-11-19T13:59:33.028342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435847022885169:2064] [node 5] ICP01 ready to work
2025-11-19T13:59:33.028420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435847022885170:2065] [node 4] ICP01 ready to work
2025-11-19T13:59:33.028573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435848078258652:2093] [node 6] ICP01 ready to work
2025-11-19T13:59:33.028731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435846141856863:2080] [node 6] ICP01 ready to work
2025-11-19T13:59:33.029697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435846141856856:2073] [node 4] ICP01 ready to work
2025-11-19T13:59:33.034440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435848078258661:2101] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.035373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435847022885196:2087] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7254
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.043829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435848078258669:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25110
2025-11-19T13:59:33.045486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435848812164445:2083] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.047545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435847022885217:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7254
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.052689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848812164471:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41016
2025-11-19T13:59:33.052713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848812164472:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41016
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.051840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435844279160013:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23891761ns) {Configure:104(23891195ns) {ProcessConfigured:135(23597232ns) {StartInitialHandshake:157(23596739ns) {DropHandshakes:469(330ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(52ns)} GenerateSessionVirtualId:988(143ns) PrepareNewSessionHandshake:476(1028ns) {SwitchToState:178(768ns)}}}}}
2025-11-19T13:59:33.051998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435848574127381:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:33.052760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848812164473:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41016
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.053297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435848574127385:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40697
2025-11-19T13:59:33.053311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435848574127386:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40697
2025-11-19T13:59:33.053320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435848574127387:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40697
2025-11-19T13:59:33.055958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435848574127377:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:33.055945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435848078258670:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25110
2025-11-19T13:59:33.057002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435848078258653:2091] [node 5] ICP01 ready to work
2025-11-19T13:59:33.060056Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435848812164472:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.064181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435848619388777:2092] [node 5] ICP01 ready to work
2025-11-19T13:59:33.064833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435848619388774:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2415
2025-11-19T13:59:33.064846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435848619388775:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2415
2025-11-19T13:59:33.064857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435848619388776:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2415
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.065803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435846141856902:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21350
2025-11-19T13:59:33.065127Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435848078258652:2093] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(36152590ns) {Configure:104(24673245ns) {ProcessConfigured:135(24405370ns) {StartInitialHandshake:157(24404849ns) {DropHandshakes:469(429ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(193ns)} GenerateSessionVirtualId:988(806ns) PrepareNewSessionHandshake:476(972ns) {SwitchToState:178(475ns)}}}}}
2025-11-19T13:59:33.065484Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848078258670:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.069008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435843783291306:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17066264ns) {Configure:104(17065482ns) {ProcessConfigured:135(16729700ns) {StartInitialHandshake:157(16729446ns) {DropHandshakes:469(537ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:476(641ns) {SwitchToState:178(268ns)}}}}}
2025-11-19T13:59:33.068387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435848574127390:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40697
2025-11-19T13:59:33.072026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435848619388763:2089] [node 4] ICP01 ready to work
2025-11-19T13:59:33.071779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435848812164461:2081] [node 5] ICP01 ready to work
2025-11-19T13:59:33.079890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435848574127377:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23843332ns) {Configure:104(23842657ns) {ProcessConfigured:135(23558406ns) {StartInitialHandshake:157(23557563ns) {DropHandshakes:469(812ns) {DropIncomingHandshake:441(91ns) DropOutgoingHandshake:456(170ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:476(1161ns) {SwitchToState:178(477ns)}}}}}
2025-11-19T13:59:33.079970Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435848574127387:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.083851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435844324421410:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(55167638ns) {Configure:104(55166325ns) {ProcessConfigured:135(54962977ns) {StartInitialHandshake:157(54961881ns) {DropHandshakes:469(123ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(13ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:476(1297ns) {SwitchToState:178(1026ns)}}}}}
2025-11-19T13:59:33.086387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435848078258683:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25110
2025-11-19T13:59:33.086408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435848078258684:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25110
2025-11-19T13:59:33.088373Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435846141856902:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.088436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435841846889534:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.088517Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435848574127390:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.092751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435848619388786:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2415
2025-11-19T13:59:33.093431Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435848812164471:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.099899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435844517197108:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(71223545ns) {Configure:104(68053289ns) {ProcessConfigured:135(67781508ns) {StartInitialHandshake:157(67780786ns) {DropHandshakes:469(229ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(42ns)} GenerateSessionVirtualId:988(159ns) PrepareNewSessionHandshake:476(1033ns) {SwitchToState:178(849ns)}}}}}
2025-11-19T13:59:33.099959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435844517197108:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.105654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435848619388763:2089] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(33549890ns) {Configure:104(33548544ns) {ProcessConfigured:135(33217783ns) {StartInitialHandshake:157(33216773ns) {DropHandshakes:469(1357ns) {DropIncomingHandshake:441(130ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(457ns) PrepareNewSessionHandshake:476(555ns) {SwitchToState:178(207ns)}}}}}
2025-11-19T13:59:33.107166Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848078258683:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.104392Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435848574127370:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.104408Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435848574127370:2106] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.107594Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435848574127370:2106] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:33.107496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848812164487:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41016
2025-11-19T13:59:33.107802Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435848574127386:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.107817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435848574127368:2105] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.107831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:33.107839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435848574127368:2105] [node 4] ICP22 created new session: [1:7574435848574127395:2105]
2025-11-19T13:59:33.107845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435848574127395:2105] [node 4] ICS09 handshake done sender: [1:7574435848574127370:2106] self: [1:7574435848574127369:0] peer: [4:7574435848812164486:0] socket: 474 qp: -1
2025-11-19T13:59:33.107854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435848574127395:2105] [node 4] ICS10 traffic start
2025-11-19T13:59:33.107875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:33.108715Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435848619388786:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.108742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435844324421410:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.108862Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848078258677:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.108869Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435848078258677:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.109688Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435848078258677:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:33.109848Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435848812164473:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.109872Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435848812164471:2105] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.109976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435848619388789:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2415
2025-11-19T13:59:33.110184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435843783291306:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.110200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:33.110207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435843783291306:2049] [node 3] ICP22 created new session: [2:7574435848078258687:2049]
2025-11-19T13:59:33.110212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435848078258687:2049] [node 3] ICS09 handshake done sender: [2:7574435848078258677:2114] self: [2:7574435848078258654:0] peer: [3:7574435848619388788:0] socket: 505 qp: -1
2025-11-19T13:59:33.110217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435848078258687:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:33.110223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:33.111576Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435848574127385:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.111840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435844517197109:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.110455Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435847022885196:2087] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.110500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435847022885149:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.111994Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435848619388751:2090] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.112002Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435848619388751:2090] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.113087Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435848619388751:2090] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:33.113163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435844517197108:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.113186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:33.113192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435844517197108:2048] [node 1] ICP22 created new session: [4:7574435848812164489:2048]
2025-11-19T13:59:33.113197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435848812164489:2048] [node 1] ICS09 handshake done sender: [4:7574435848812164471:2105] self: [4:7574435848812164486:0] peer: [1:7574435848574127369:0] socket: 477 qp: -1
2025-11-19T13:59:33.113202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435848812164489:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:33.113209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.115405Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435848619388786:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:33.115672Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435847022885217:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.115953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435847022885170:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.115846Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435848812164483:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.115853Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435848812164483:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.116285Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848078258663:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.116296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435848078258663:2104] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.116322Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435848619388781:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.116330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435848619388781:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.116540Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848078258669:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.117039Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435848574127373:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.117061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435848574127373:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.117021Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435848812164463:2101] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.117027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435848812164463:2101] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.116887Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435848619388774:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.116927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435844324421409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.117623Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435848619388776:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.118411Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435848812164463:2101] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:33.118253Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435847022885169:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(58155112ns) {Configure:104(57846563ns) {ProcessConfigured:135(57599876ns) {StartInitialHandshake:157(57599105ns) {DropHandshakes:469(352ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(142ns) PrepareNewSessionHandshake:476(645ns) {SwitchToState:178(442ns)}}}}}
2025-11-19T13:59:33.118287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435847022885229:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7254
2025-11-19T13:59:33.118680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435847022885230:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7254
2025-11-19T13:59:33.118749Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435847022885229:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.118767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435847022885147:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.118689Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435848812164483:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:33.118865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435848812164445:2083] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.118881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:33.118889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435848812164445:2083] [node 6] ICP22 created new session: [4:7574435848812164494:2083]
2025-11-19T13:59:33.118933Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435848574127383:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.118941Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435848574127383:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.119001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435848812164461:2081] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.119017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:33.119022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435848812164461:2081] [node 5] ICP22 created new session: [4:7574435848812164495:2081]
2025-11-19T13:59:33.119027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435848812164495:2081] [node 5] ICS09 handshake done sender: [4:7574435848812164483:2116] self: [4:7574435848812164482:0] peer: [5:7574435846141856909:0] socket: 518 qp: -1
2025-11-19T13:59:33.119031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435848812164495:2081] [node 5] ICS10 traffic start
2025-11-19T13:59:33.119046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:33.119222Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435847022885230:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.119239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435847022885148:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.119409Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848078258684:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.119433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435843783291305:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.113723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435846141856908:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21350
2025-11-19T13:59:33.115655Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435846141856908:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.115681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435846141856856:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.120225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435848619388775:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.120241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435844324421410:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.120253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:33.120259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435844324421410:2049] [node 2] ICP22 created new session: [3:7574435848619388793:2049]
2025-11-19T13:59:33.120264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435848619388793:2049] [node 2] ICS09 handshake done sender: [3:7574435848619388786:2117] self: [3:7574435848619388788:0] peer: [2:7574435848078258654:0] socket: 508 qp: -1
2025-11-19T13:59:33.120271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435848619388793:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:33.120287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:33.120719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435848619388744:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.120735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:33.120741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435848619388744:2084] [node 6] ICP22 created new session: [3:7574435848619388797:2084]
2025-11-19T13:59:33.120746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435848619388797:2084] [node 6] ICS09 handshake done sender: [3:7574435848619388751:2090] self: [3:7574435848619388749:0] peer: [6:7574435847022885227:0] socket: 480 qp: -1
2025-11-19T13:59:33.120750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435848619388797:2084] [node 6] ICS10 traffic start
2025-11-19T13:59:33.120772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:33.120229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435848812164494:2083] [node 6] ICS09 handshake done sender: [4:7574435848812164463:2101] self: [4:7574435848812164462:0] peer: [6:7574435847022885228:0] socket: 492 qp: -1
2025-11-19T13:59:33.120251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435848812164494:2083] [node 6] ICS10 traffic start
2025-11-19T13:59:33.120264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:33.125079Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435847022885149:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28425) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435847022885196:2087] held: no
2025-11-19T13:59:33.125023Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435848078258663:2104] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:33.125528Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435848619388781:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:33.125537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435848078258661:2101] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.125560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:33.125570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435848078258661:2101] [node 4] ICP22 created new session: [2:7574435848078258693:2101]
2025-11-19T13:59:33.125577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435848078258693:2101] [node 4] ICS09 handshake done sender: [2:7574435848078258663:2104] self: [2:7574435848078258662:0] peer: [4:7574435848812164488:0] socket: 483 qp: -1
2025-11-19T13:59:33.125585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435848078258693:2101] [node 4] ICS10 traffic start
2025-11-19T13:59:33.125594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:33.126149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435848619388777:2092] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.126171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:33.126178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435848619388777:2092] [node 5] ICP22 created new session: [3:7574435848619388800:2092]
2025-11-19T13:59:33.126184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435848619388800:2092] [node 5] ICS09 handshake done sender: [3:7574435848619388781:2115] self: [3:7574435848619388780:0] peer: [5:7574435846141856907:0] socket: 504 qp: -1
2025-11-19T13:59:33.126191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435848619388800:2092] [node 5] ICS10 traffic start
2025-11-19T13:59:33.126216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:33.127191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435847022885231:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7254
2025-11-19T13:59:33.127094Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848078258678:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.127108Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435848078258678:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.127230Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435848574127380:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.127241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435848574127380:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.127770Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435847022885196:2087] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:33.127892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435847022885235:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7254
2025-11-19T13:59:33.128013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435847022885149:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.128028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:33.128035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435847022885149:2050] [node 3] ICP22 created new session: [6:7574435847022885236:2050]
2025-11-19T13:59:33.128040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435847022885236:2050] [node 3] ICS09 handshake done sender: [6:7574435847022885196:2087] self: [6:7574435847022885227:0] peer: [3:7574435848619388749:0] socket: 486 qp: -1
2025-11-19T13:59:33.128046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435847022885236:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:33.128053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.129365Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435848574127380:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:33.129963Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435847022885217:2107] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:33.130102Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435844517197109:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5730) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435848812164473:2107] held: no
2025-11-19T13:59:33.130782Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435848574127383:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:33.131411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435848574127381:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.131428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:33.131437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435848574127381:2109] [node 6] ICP22 created new session: [1:7574435848574127402:2109]
2025-11-19T13:59:33.131442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435848574127402:2109] [node 6] ICS09 handshake done sender: [1:7574435848574127383:2112] self: [1:7574435848574127382:0] peer: [6:7574435847022885232:0] socket: 512 qp: -1
2025-11-19T13:59:33.131446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435848574127402:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:33.131451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:33.128768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435846141856910:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21350
2025-11-19T13:59:33.129402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435846141856911:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21350
2025-11-19T13:59:33.129432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435846141856912:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21350
2025-11-19T13:59:33.129782Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435846141856902:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:33.130873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435846141856856:2073] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14496) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435846141856908:2116] held: no
2025-11-19T13:59:33.130889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435841846889534:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.130898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:33.130903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435841846889534:2050] [node 3] ICP22 created new session: [5:7574435846141856913:2050]
2025-11-19T13:59:33.130908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435846141856913:2050] [node 3] ICS09 handshake done sender: [5:7574435846141856902:2111] self: [5:7574435846141856907:0] peer: [3:7574435848619388780:0] socket: 506 qp: -1
2025-11-19T13:59:33.130913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435846141856913:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:33.130919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:33.131663Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435846141856908:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:33.133576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435844279160013:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.133594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:33.133600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435844279160013:2048] [node 2] ICP22 created new session: [1:7574435848574127404:2048]
2025-11-19T13:59:33.133606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435848574127404:2048] [node 2] ICS09 handshake done sender: [1:7574435848574127380:2111] self: [1:7574435848574127375:0] peer: [2:7574435848078258691:0] socket: 495 qp: -1
2025-11-19T13:59:33.133611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435848574127404:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:33.133629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:33.135863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435848812164461:2081] [node 5] ICP35 event processing took too much time PendingConnection:278(16824378ns) {HandleHandshakeStatus:329(16823768ns) {DropIncomingHandshake:441(3134ns) DropOutgoingHandshake:456(334ns) SwitchToState:178(73ns) ProcessPendingSessionEvents:495(8744591ns) {ForwardSessionEventToSession:612(8733659ns) ForwardSessionEventToSession:612(936ns) ForwardSessionEventToSession:612(828ns) ForwardSessionEventToSession:612(553ns) ForwardSessionEventToSession:612(605ns) ForwardSessionEventToSession:612(894ns) ForwardSessionEventToSession:612(918ns) ForwardSessionEventToSession:612(467ns) ForwardSessionEventToSession:612(579ns) ForwardSessionEventToSession:612(931ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.137430Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435848574127373:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:33.138645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435844279160014:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.138661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:33.138668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435844279160014:2049] [node 3] ICP22 created new session: [1:7574435848574127408:2049]
2025-11-19T13:59:33.138674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435848574127408:2049] [node 3] ICS09 handshake done sender: [1:7574435848574127373:2107] self: [1:7574435848574127372:0] peer: [3:7574435848619388791:0] socket: 475 qp: -1
2025-11-19T13:59:33.138678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435848574127408:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:33.138683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:33.140245Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435848078258678:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:33.140521Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435843783291305:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63902) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435848078258684:2118] held: no
2025-11-19T13:59:33.140545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435848078258652:2093] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.140556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:33.140563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435848078258652:2093] [node 6] ICP22 created new session: [2:7574435848078258697:2093]
2025-11-19T13:59:33.140569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435848078258697:2093] [node 6] ICS09 handshake done sender: [2:7574435848078258678:2115] self: [2:7574435848078258672:0] peer: [6:7574435847022885233:0] socket: 516 qp: -1
2025-11-19T13:59:33.140574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435848078258697:2093] [node 6] ICS10 traffic start
2025-11-19T13:59:33.140582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:33.137242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435848619388802:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2415
2025-11-19T13:59:33.137591Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435848619388774:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:33.137682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435844324421409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.137696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:33.137702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435844324421409:2048] [node 1] ICP22 created new session: [3:7574435848619388803:2048]
2025-11-19T13:59:33.137707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435848619388803:2048] [node 1] ICS09 handshake done sender: [3:7574435848619388774:2111] self: [3:7574435848619388791:0] peer: [1:7574435848574127372:0] socket: 479 qp: -1
2025-11-19T13:59:33.137712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435848619388803:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:33.137721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.144038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435847022885170:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.144058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:33.144064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435847022885170:2065] [node 4] ICP22 created new session: [6:7574435847022885239:2065]
2025-11-19T13:59:33.144071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435847022885239:2065] [node 4] ICS09 handshake done sender: [6:7574435847022885217:2107] self: [6:7574435847022885228:0] peer: [4:7574435848812164462:0] socket: 494 qp: -1
2025-11-19T13:59:33.144076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435847022885239:2065] [node 4] ICS10 traffic start
2025-11-19T13:59:33.144084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:33.144275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435847022885238:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7254
2025-11-19T13:59:33.144373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435846141856856:2073] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.144390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:33.144395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435846141856856:2073] [node 4] ICP22 created new session: [5:7574435846141856916:2073]
2025-11-19T13:59:33.144400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435846141856916:2073] [node 4] ICS09 handshake done sender: [5:7574435846141856908:2116] self: [5:7574435846141856909:0] peer: [4:7574435848812164482:0] socket: 519 qp: -1
2025-11-19T13:59:33.144405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435846141856916:2073] [node 4] ICS10 traffic start
2025-11-19T13:59:33.144411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:33.144608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435846141856918:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21350
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.146585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435848078258699:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25110
2025-11-19T13:59:33.146442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848812164498:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41016
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.147570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848812164499:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41016
2025-11-19T13:59:33.147621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848812164500:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41016
2025-11-19T13:59:33.147858Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435846141856912:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.147896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435841846889533:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.148878Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435848812164499:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.148902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435844517197110:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.149069Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435848619388787:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.149080Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435848619388787:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.150558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848812164502:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41016
2025-11-19T13:59:33.150756Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435848812164473:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:33.151172Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435848812164499:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.147474Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435848078258684:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:33.147990Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848078258680:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.147993Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435848078258680:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.150446Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435848078258680:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:33.150889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435843783291305:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.150898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:33.150904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435843783291305:2048] [node 1] ICP22 created new session: [2:7574435848078258701:2048]
2025-11-19T13:59:33.150909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435848078258701:2048] [node 1] ICS09 handshake done sender: [2:7574435848078258684:2118] self: [2:7574435848078258691:0] peer: [1:7574435848574127375:0] socket: 509 qp: -1
2025-11-19T13:59:33.150913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435848078258701:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:33.150919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:33.150996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435848078258653:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.151004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:33.151007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435848078258653:2091] [node 5] ICP22 created new session: [2:7574435848078258703:2091]
2025-11-19T13:59:33.151011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435848078258703:2091] [node 5] ICS09 handshake done sender: [2:7574435848078258680:2116] self: [2:7574435848078258679:0] peer: [5:7574435846141856919:0] socket: 500 qp: -1
2025-11-19T13:59:33.151014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435848078258703:2091] [node 5] ICS10 traffic start
2025-11-19T13:59:33.151019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:33.151275Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435846141856918:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.151330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435846141856920:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21350
2025-11-19T13:59:33.151262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435844517197110:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.151273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:33.151279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435844517197110:2050] [node 3] ICP22 created new session: [4:7574435848812164503:2050]
2025-11-19T13:59:33.151283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435848812164503:2050] [node 3] ICS09 handshake done sender: [4:7574435848812164499:2120] self: [4:7574435848812164501:0] peer: [3:7574435848619388784:0] socket: 484 qp: -1
2025-11-19T13:59:33.151288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435848812164503:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:33.151292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:33.149980Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435848619388787:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:33.150076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435848619388763:2089] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.150088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:33.150096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435848619388763:2089] [node 4] ICP22 created new session: [3:7574435848619388806:2089]
2025-11-19T13:59:33.150101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435848619388806:2089] [node 4] ICS09 handshake done sender: [3:7574435848619388787:2118] self: [3:7574435848619388784:0] peer: [4:7574435848812164501:0] socket: 527 qp: -1
2025-11-19T13:59:33.150106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435848619388806:2089] [node 4] ICS10 traffic start
2025-11-19T13:59:33.150112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:33.152311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435841846889532:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.152392Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435841846889533:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5730) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435846141856912:2119] held: no
2025-11-19T13:59:33.152443Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435846141856912:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:33.157039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435848078258652:2093] [node 6] ICP35 event processing took too much time PendingConnection:278(16455231ns) {HandleHandshakeStatus:329(16454568ns) {DropIncomingHandshake:441(2902ns) DropOutgoingHandshake:456(92ns) SwitchToState:178(231ns) ProcessPendingSessionEvents:495(16373221ns) {ForwardSessionEventToSession:612(16361056ns) ForwardSessionEventToSession:612(1476ns) ForwardSessionEventToSession:612(759ns) ForwardSessionEventToSession:612(1511ns) ForwardSessionEventToSession:612(613ns) ForwardSessionEventToSession:612(597ns) ForwardSessionEventToSession:612(680ns) ForwardSessionEventToSession:612(577ns) ForwardSessionEventToSession:612(537ns) ForwardSessionEventToSession:612(1191ns)}}}
2025-11-19T13:59:33.157433Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435848574127391:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.157447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435848574127391:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.158164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435844517197109:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.158184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:33.158191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435844517197109:2049] [node 2] ICP22 created new session: [4:7574435848812164505:2049]
2025-11-19T13:59:33.158196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435848812164505:2049] [node 2] ICS09 handshake done sender: [4:7574435848812164473:2107] self: [4:7574435848812164488:0] peer: [2:7574435848078258662:0] socket: 497 qp: -1
2025-11-19T13:59:33.158202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435848812164505:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:33.158209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:33.158453Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435847022885148:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5730) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435847022885230:2118] held: no
2025-11-19T13:59:33.158561Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435847022885147:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63902) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435847022885229:2117] held: no
2025-11-19T13:59:33.158825Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435848574127391:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:33.158916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435848574127377:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.158934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:33.158939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435848574127377:2108] [node 5] ICP22 created new session: [1:7574435848574127411:2108]
2025-11-19T13:59:33.158945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435848574127411:2108] [node 5] ICS09 handshake done sender: [1:7574435848574127391:2118] self: [1:7574435848574127389:0] peer: [5:7574435846141856921:0] socket: 498 qp: -1
2025-11-19T13:59:33.158950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435848574127411:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:33.158956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:33.159823Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435848812164445:2083] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.162107Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435847022885229:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.162179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435847022885241:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7254
2025-11-19T13:59:33.162227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435847022885242:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7254
2025-11-19T13:59:33.162524Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435847022885241:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.162607Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435847022885230:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.162835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435847022885147:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.162846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:33.162851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435847022885147:2048] [node 1] ICP22 created new session: [6:7574435847022885244:2048]
2025-11-19T13:59:33.162857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435847022885244:2048] [node 1] ICS09 handshake done sender: [6:7574435847022885229:2117] self: [6:7574435847022885232:0] peer: [1:7574435848574127382:0] socket: 513 qp: -1
2025-11-19T13:59:33.162861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435847022885244:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:33.162866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:33.162955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435847022885169:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.160773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435841846889533:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.160792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:33.160799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435841846889533:2049] [node 2] ICP22 created new session: [5:7574435846141856922:2049]
2025-11-19T13:59:33.160804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435846141856922:2049] [node 2] ICS09 handshake done sender: [5:7574435846141856912:2119] self: [5:7574435846141856919:0] peer: [2:7574435848078258679:0] socket: 531 qp: -1
2025-11-19T13:59:33.160809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435846141856922:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:33.160816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:33.161007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435846141856924:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21350
2025-11-19T13:59:33.162258Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435841846889532:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63902) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435846141856918:2120] held: no
2025-11-19T13:59:33.162700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435846141856925:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21350
2025-11-19T13:59:33.163103Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435846141856876:2089] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.163107Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435846141856876:2089] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.164347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435847022885148:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.164364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:33.164370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435847022885148:2049] [node 2] ICP22 created new session: [6:7574435847022885248:2049]
2025-11-19T13:59:33.164375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435847022885248:2049] [node 2] ICS09 handshake done sender: [6:7574435847022885230:2118] self: [6:7574435847022885233:0] peer: [2:7574435848078258672:0] socket: 520 qp: -1
2025-11-19T13:59:33.164380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435847022885248:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:33.164386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:33.165670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435847022885247:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7254
2025-11-19T13:59:33.164210Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435846141856876:2089] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:33.166008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435846141856863:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.166016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:33.166021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435846141856863:2080] [node 6] ICP22 created new session: [5:7574435846141856927:2080]
2025-11-19T13:59:33.166025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435846141856927:2080] [node 6] ICS09 handshake done sender: [5:7574435846141856876:2089] self: [5:7574435846141856875:0] peer: [6:7574435847022885246:0] socket: 496 qp: -1
2025-11-19T13:59:33.166029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435846141856927:2080] [node 6] ICS10 traffic start
2025-11-19T13:59:33.166035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:33.166111Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435846141856918:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:33.166318Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435846141856925:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.166347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435846141856863:2080] [node 6] ICP18 handshake (actor [5:7574435846141856925:2123]) is too old
2025-11-19T13:59:33.166487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435841846889532:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.166494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:33.166498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435841846889532:2048] [node 1] ICP22 created new session: [5:7574435846141856929:2048]
2025-11-19T13:59:33.166503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435846141856929:2048] [node 1] ICS09 handshake done sender: [5:7574435846141856918:2120] self: [5:7574435846141856921:0] peer: [1:7574435848574127389:0] socket: 478 qp: -1
2025-11-19T13:59:33.166508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435846141856929:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:33.166513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:33.167385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435847022885169:2064] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26195) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435847022885241:2122] held: no
2025-11-19T13:59:33.172818Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435847022885241:2122] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:33.175915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435847022885169:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.175933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:33.175942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435847022885169:2064] [node 5] ICP22 created new session: [6:7574435847022885250:2064]
2025-11-19T13:59:33.175948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435847022885250:2064] [node 5] ICS09 handshake done sender: [6:7574435847022885241:2122] self: [6:7574435847022885246:0] peer: [5:7574435846141856875:0] socket: 503 qp: -1
2025-11-19T13:59:33.175957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435847022885250:2064] [node 5] ICS10 traffic start
2025-11-19T13:59:33.175973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:33.208330Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:33.208449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435848078258701:2048] [node 1] ICS01 socket: 509 reason# EndOfStream
2025-11-19T13:59:33.208461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435843783291305:2048] [node 1] ICP30 unregister session Session# [2:7574435848078258701:2048] VirtualId# [2:7574435848078258691:0]
2025-11-19T13:59:33.208470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435848078258701:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.208504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:33.208525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:33.209097Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:33.209169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435848619388803:2048] [node 1] ICS01 socket: 479 reason# EndOfStream
2025-11-19T13:59:33.209175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435844324421409:2048] [node 1] ICP30 unregister session Session# [3:7574435848619388803:2048] VirtualId# [3:7574435848619388791:0]
2025-11-19T13:59:33.209180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435848619388803:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.209207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:33.209225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:33.241230Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:33.241340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435846141856929:2048] [node 1] ICS01 socket: 478 reason# EndOfStream
2025-11-19T13:59:33.241351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435841846889532:2048] [node 1] ICP30 unregister session Session# [5:7574435846141856929:2048] VirtualId# [5:7574435846141856921:0]
2025-11-19T13:59:33.241359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435846141856929:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.241407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:33.241430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:33.241466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:33.241523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435847022885244:2048] [node 1] ICS01 socket: 513 reason# EndOfStream
2025-11-19T13:59:33.241527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435847022885147:2048] [node 1] ICP30 unregister session Session# [6:7574435847022885244:2048] VirtualId# [6:7574435847022885232:0]
2025-11-19T13:59:33.241532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435847022885244:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.241555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:33.241574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:33.241683Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:33.241738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435848812164489:2048] [node 1] ICS01 socket: 477 reason# EndOfStream
2025-11-19T13:59:33.241742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435844517197108:2048] [node 1] ICP30 unregister session Session# [4:7574435848812164489:2048] VirtualId# [4:7574435848812164486:0]
2025-11-19T13:59:33.241746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435848812164489:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.241770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:33.241789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:33.247076Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:33.247173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435848619388793:2049] [node 2] ICS01 socket: 508 reason# EndOfStream
2025-11-19T13:59:33.247182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435844324421410:2049] [node 2] ICP30 unregister session Session# [3:7574435848619388793:2049] VirtualId# [3:7574435848619388788:0]
2025-11-19T13:59:33.247189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435848619388793:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.247221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:33.247238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:33.254537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:33.254630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435847022885248:2049] [node 2] ICS01 socket: 520 reason# EndOfStream
2025-11-19T13:59:33.254638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435847022885148:2049] [node 2] ICP30 unregister session Session# [6:7574435847022885248:2049] VirtualId# [6:7574435847022885233:0]
2025-11-19T13:59:33.254645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435847022885248:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.254685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:33.254706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:33.254734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:33.254796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435848812164505:2049] [node 2] ICS01 socket: 497 reason# EndOfStream
2025-11-19T13:59:33.254800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435844517197109:2049] [node 2] ICP30 unregister session Session# [4:7574435848812164505:2049] VirtualId# [4:7574435848812164488:0]
2025-11-19T13:59:33.254804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435848812164505:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.254838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:33.254855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:33.254954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:33.255010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435846141856922:2049] [node 2] ICS01 socket: 531 reason# EndOfStream
2025-11-19T13:59:33.255014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435841846889533:2049] [node 2] ICP30 unregister session Session# [5:7574435846141856922:2049] VirtualId# [5:7574435846141856919:0]
2025-11-19T13:59:33.255017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435846141856922:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.255038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:33.255055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:33.282259Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:33.282371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435847022885236:2050] [node 3] ICS01 socket: 486 reason# EndOfStream
2025-11-19T13:59:33.282383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435847022885149:2050] [node 3] ICP30 unregister session Session# [6:7574435847022885236:2050] VirtualId# [6:7574435847022885227:0]
2025-11-19T13:59:33.282391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435847022885236:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.282446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:33.282471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:33.282508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:33.282568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435846141856913:2050] [node 3] ICS01 socket: 506 reason# EndOfStream
2025-11-19T13:59:33.282573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435841846889534:2050] [node 3] ICP30 unregister session Session# [5:7574435846141856913:2050] VirtualId# [5:7574435846141856907:0]
2025-11-19T13:59:33.282578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435846141856913:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.282602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:33.282622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:33.310838Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:33.310951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435847022885239:2065] [node 4] ICS01 socket: 494 reason# EndOfStream
2025-11-19T13:59:33.310962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435847022885170:2065] [node 4] ICP30 unregister session Session# [6:7574435847022885239:2065] VirtualId# [6:7574435847022885228:0]
2025-11-19T13:59:33.310971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435847022885239:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.311029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:33.311059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:33.424540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435847992940683:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:33.424552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435847992940684:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:33.467882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435846363703209:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:33.467912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435846363703210:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:33.506213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435847665355546:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:33.506228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435847665355547:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:33.536308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435847029769323:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:33.536326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435847029769324:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:33.536331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435847029769325:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:33.552091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435844932626241:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:33.552107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435844932626242:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:33.552112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435844932626243:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:33.567893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435845364908823:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:33.567901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435845364908822:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:33.567907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435845364908824:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.619247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435846363703267:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34869
2025-11-19T13:59:33.620161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435844932626298:2101] [node 4] ICP01 ready to work
2025-11-19T13:59:33.620194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435847992940748:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:33.620307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435847992940750:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.624648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435847992940752:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.631306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435847992940760:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39539
2025-11-19T13:59:33.631874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435847665355606:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42571
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.639161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435846363703275:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:33.639217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435847665355608:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:33.638074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435847029769385:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:33.639023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435845364908887:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8957
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.640499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435845364908888:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8957
2025-11-19T13:59:33.639268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435846363703276:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.643959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435847992940752:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19244088ns) {Configure:104(19243557ns) {ProcessConfigured:135(18940514ns) {StartInitialHandshake:157(18939961ns) {DropHandshakes:469(556ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(286ns) PrepareNewSessionHandshake:476(1407ns) {SwitchToState:178(660ns)}}}}}
2025-11-19T13:59:33.644782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435845364908823:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23599366ns) {Configure:104(23598293ns) {ProcessConfigured:135(23370417ns) {StartInitialHandshake:157(23370039ns) {DropHandshakes:469(240ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(24ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:476(670ns) {SwitchToState:178(493ns)}}}}}
2025-11-19T13:59:33.644346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435846363703283:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34869
2025-11-19T13:59:33.645490Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435845364908887:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.653288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435847992940750:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(32667771ns) {Configure:104(32666985ns) {ProcessConfigured:135(32400594ns) {StartInitialHandshake:157(32400159ns) {DropHandshakes:469(243ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(1283ns) {SwitchToState:178(981ns)}}}}}
2025-11-19T13:59:33.653731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435847665355617:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42571
2025-11-19T13:59:33.653785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435847665355618:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42571
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.654244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435847992940763:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39539
2025-11-19T13:59:33.654325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435845364908888:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.654361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435845364908823:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.655568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435847665355611:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:33.655651Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435847665355606:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.653796Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435846363703267:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.653997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435846363703285:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34869
2025-11-19T13:59:33.653895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435844932626298:2101] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(33651364ns) {Configure:104(33650707ns) {ProcessConfigured:135(33183227ns) {StartInitialHandshake:157(33182556ns) {DropHandshakes:469(530ns) {DropIncomingHandshake:441(95ns) DropOutgoingHandshake:456(85ns)} GenerateSessionVirtualId:988(330ns) PrepareNewSessionHandshake:476(811ns) {SwitchToState:178(622ns)}}}}}
2025-11-19T13:59:33.655841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435847665355614:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.655197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435845364908886:2105] [node 4] ICP01 ready to work
2025-11-19T13:59:33.655971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435845364908885:2099] [node 5] ICP01 ready to work
2025-11-19T13:59:33.663841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435847029769323:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23587967ns) {Configure:104(23587323ns) {ProcessConfigured:135(23380775ns) {StartInitialHandshake:157(23379951ns) {DropHandshakes:469(201ns) {DropIncomingHandshake:441(28ns) DropOutgoingHandshake:456(24ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:476(947ns) {SwitchToState:178(509ns)}}}}}
2025-11-19T13:59:33.664024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435847029769399:2113] [node 5] ICP01 ready to work
2025-11-19T13:59:33.664014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435847665355608:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24739317ns) {Configure:104(24738693ns) {ProcessConfigured:135(22686860ns) {StartInitialHandshake:157(22685899ns) {DropHandshakes:469(331ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(112ns) PrepareNewSessionHandshake:476(1329ns) {SwitchToState:178(320ns)}}}}}
2025-11-19T13:59:33.664071Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435847665355617:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.668223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435844932626314:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63225
2025-11-19T13:59:33.668372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435846363703210:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(47180059ns) {Configure:104(47179338ns) {ProcessConfigured:135(46772060ns) {StartInitialHandshake:157(46771558ns) {DropHandshakes:469(535ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(87ns)} GenerateSessionVirtualId:988(261ns) PrepareNewSessionHandshake:476(1591ns) {SwitchToState:178(340ns)}}}}}
2025-11-19T13:59:33.671516Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435847665355547:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(30100636ns) {Configure:104(30099848ns) {ProcessConfigured:135(29828901ns) {StartInitialHandshake:157(29826422ns) {DropHandshakes:469(385ns) {DropIncomingHandshake:441(77ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:476(1167ns) {SwitchToState:178(978ns)}}}}}
2025-11-19T13:59:33.669749Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435846363703209:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(48113532ns) {Configure:104(44525166ns) {ProcessConfigured:135(44205079ns) {StartInitialHandshake:157(44204692ns) {DropHandshakes:469(646ns) {DropIncomingHandshake:441(90ns) DropOutgoingHandshake:456(94ns)} GenerateSessionVirtualId:988(277ns) PrepareNewSessionHandshake:476(672ns) {SwitchToState:178(273ns)}}}}}
2025-11-19T13:59:33.669900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435846363703209:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.672126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435844932626310:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:33.672143Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435846363703283:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.675114Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435846363703281:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.675129Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435846363703281:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.675939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435846363703284:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-19T13:59:33.675850Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435844932626314:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.677350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435844932626242:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(21914452ns) {Configure:104(21913786ns) {ProcessConfigured:135(21575364ns) {StartInitialHandshake:157(21574897ns) {DropHandshakes:469(534ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(248ns) PrepareNewSessionHandshake:476(598ns) {SwitchToState:178(192ns)}}}}}
2025-11-19T13:59:33.677380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435844932626242:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.676161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435847029769396:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-19T13:59:33.677217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435847029769403:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-19T13:59:33.677234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435847029769402:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-19T13:59:33.677322Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435847029769387:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.677325Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435847029769387:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.677632Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435847029769396:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.676688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435845364908886:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.676745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435845364908886:2105] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(21467666ns) {Configure:104(21466978ns) {ProcessConfigured:135(21151306ns) {StartInitialHandshake:157(21115539ns) {DropHandshakes:469(664ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(109ns)} GenerateSessionVirtualId:988(515ns) PrepareNewSessionHandshake:476(1524ns) {SwitchToState:178(1137ns)}} IncomingHandshake:261(24535ns) {DropIncomingHandshake:441(60ns) PrepareNewSessionHandshake:476(156ns) {SwitchToState:178(122ns)} GenerateSessionVirtualId:988(253ns) IssueIncomingHandshakeReply:193(17812ns) {SwitchToState:178(38ns)}}}}}
2025-11-19T13:59:33.679420Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435845364908885:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23367710ns) {Configure:104(23366969ns) {ProcessConfigured:135(23092758ns) {StartInitialHandshake:157(23092252ns) {DropHandshakes:469(599ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:476(1290ns) {SwitchToState:178(284ns)}}}}}
2025-11-19T13:59:33.680587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435847665355546:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.680626Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435847665355546:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(23817618ns) {IncomingHandshake:261(23817173ns) {DropIncomingHandshake:441(195ns) PrepareNewSessionHandshake:476(448ns) {SwitchToState:178(255ns)} GenerateSessionVirtualId:988(235ns) IssueIncomingHandshakeReply:193(23810147ns) {SwitchToState:178(971ns)}}}
2025-11-19T13:59:33.684007Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435847029769402:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.684047Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435847029769387:2109] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:33.683866Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435847992940744:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.683884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435847992940744:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.683909Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435847992940757:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.683921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435847992940757:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.683943Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435847992940753:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.683951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435847992940753:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.684428Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435847665355618:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.684364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435844932626241:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(51535260ns) {Configure:104(51533777ns) {ProcessConfigured:135(51184959ns) {StartInitialHandshake:157(51184431ns) {DropHandshakes:469(783ns) {DropIncomingHandshake:441(160ns) DropOutgoingHandshake:456(125ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:476(691ns) {SwitchToState:178(350ns)}}}}}
2025-11-19T13:59:33.684301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435847029769385:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.684324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:33.684331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435847029769385:2108] [node 6] ICP22 created new session: [4:7574435847029769407:2108]
2025-11-19T13:59:33.684338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435847029769407:2108] [node 6] ICS09 handshake done sender: [4:7574435847029769387:2109] self: [4:7574435847029769386:0] peer: [6:7574435845364908900:0] socket: 490 qp: -1
2025-11-19T13:59:33.684347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435847029769407:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:33.684357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:33.686398Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435846363703281:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:33.684551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435847665355628:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42571
2025-11-19T13:59:33.685300Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435847665355628:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.685325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435847665355547:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.687219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435847665355630:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42571
2025-11-19T13:59:33.686557Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435846363703286:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.686566Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435846363703286:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.686886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435846363703276:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.686899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:33.686906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435846363703276:2110] [node 6] ICP22 created new session: [2:7574435846363703292:2110]
2025-11-19T13:59:33.686912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435846363703292:2110] [node 6] ICS09 handshake done sender: [2:7574435846363703281:2112] self: [2:7574435846363703280:0] peer: [6:7574435845364908896:0] socket: 493 qp: -1
2025-11-19T13:59:33.686920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435846363703292:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:33.686939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:33.687453Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435846363703286:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.688311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435845364908903:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8957
2025-11-19T13:59:33.688520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435845364908904:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8957
2025-11-19T13:59:33.688868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435847665355614:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(32950894ns) {Configure:104(32949994ns) {ProcessConfigured:135(32649645ns) {StartInitialHandshake:157(32648219ns) {DropHandshakes:469(951ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:476(824ns) {SwitchToState:178(594ns)}}}}}
2025-11-19T13:59:33.687560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435846363703210:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.687576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:33.687582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435846363703210:2049] [node 3] ICP22 created new session: [2:7574435846363703296:2049]
2025-11-19T13:59:33.687588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435846363703296:2049] [node 3] ICS09 handshake done sender: [2:7574435846363703286:2115] self: [2:7574435846363703270:0] peer: [3:7574435847665355629:0] socket: 506 qp: -1
2025-11-19T13:59:33.687593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435846363703296:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:33.687599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:33.688891Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435846363703278:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.688903Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435846363703278:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.689220Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435847665355628:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:33.689325Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435845364908888:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:33.689296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435847665355547:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.689311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:33.689317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435847665355547:2049] [node 2] ICP22 created new session: [3:7574435847665355632:2049]
2025-11-19T13:59:33.689323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435847665355632:2049] [node 2] ICS09 handshake done sender: [3:7574435847665355628:2116] self: [3:7574435847665355629:0] peer: [2:7574435846363703270:0] socket: 507 qp: -1
2025-11-19T13:59:33.689329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435847665355632:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:33.689350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:33.691825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435847029769323:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.697403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435844932626321:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63225
2025-11-19T13:59:33.694793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435845364908824:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(42118974ns) {Configure:104(42117220ns) {ProcessConfigured:135(41710177ns) {StartInitialHandshake:157(41709057ns) {DropHandshakes:469(1790ns) {DropIncomingHandshake:441(392ns) DropOutgoingHandshake:456(133ns)} GenerateSessionVirtualId:988(824ns) PrepareNewSessionHandshake:476(1769ns) {SwitchToState:178(1065ns)}}}}}
2025-11-19T13:59:33.694848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435845364908823:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.694868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:33.694881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435845364908823:2049] [node 2] ICP22 created new session: [6:7574435845364908906:2049]
2025-11-19T13:59:33.694888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435845364908906:2049] [node 2] ICS09 handshake done sender: [6:7574435845364908888:2108] self: [6:7574435845364908896:0] peer: [2:7574435846363703280:0] socket: 494 qp: -1
2025-11-19T13:59:33.694898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435845364908906:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:33.694919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:33.695129Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435845364908887:2107] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:33.696078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435845364908886:2105] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.696098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:33.696108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435845364908886:2105] [node 4] ICP22 created new session: [6:7574435845364908908:2105]
2025-11-19T13:59:33.696118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435845364908908:2105] [node 4] ICS09 handshake done sender: [6:7574435845364908887:2107] self: [6:7574435845364908900:0] peer: [4:7574435847029769386:0] socket: 491 qp: -1
2025-11-19T13:59:33.696129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435845364908908:2105] [node 4] ICS10 traffic start
2025-11-19T13:59:33.696158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:33.696731Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435847665355611:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(41032712ns) {Configure:104(41031639ns) {ProcessConfigured:135(40721682ns) {StartInitialHandshake:157(40721352ns) {DropHandshakes:469(255ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(37ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:476(1086ns) {SwitchToState:178(96ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.699637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435846363703298:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34869
2025-11-19T13:59:33.699683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435846363703299:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34869
2025-11-19T13:59:33.699792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435846363703284:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(23736787ns) {Configure:104(23736352ns) {ProcessConfigured:135(23398160ns) {StartInitialHandshake:157(23385934ns) {DropHandshakes:469(729ns) {DropIncomingHandshake:441(166ns) DropOutgoingHandshake:456(177ns)} GenerateSessionVirtualId:988(587ns) PrepareNewSessionHandshake:476(763ns) {SwitchToState:178(177ns)}} IncomingHandshake:261(4585ns) {DropIncomingHandshake:441(114ns) PrepareNewSessionHandshake:476(192ns) {SwitchToState:178(133ns)} GenerateSessionVirtualId:988(226ns) IssueIncomingHandshakeReply:193(180ns)}}}}
2025-11-19T13:59:33.701776Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435847992940760:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.701832Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435847992940744:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:33.702009Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435847992940753:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:33.702104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435847992940683:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.702122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:33.702132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435847992940683:2048] [node 2] ICP22 created new session: [1:7574435847992940766:2048]
2025-11-19T13:59:33.702138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435847992940766:2048] [node 2] ICS09 handshake done sender: [1:7574435847992940744:2107] self: [1:7574435847992940743:0] peer: [2:7574435846363703289:0] socket: 480 qp: -1
2025-11-19T13:59:33.702144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435847992940766:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:33.702170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:33.702243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435847992940684:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.702253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:33.702258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435847992940684:2049] [node 3] ICP22 created new session: [1:7574435847992940768:2049]
2025-11-19T13:59:33.702264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435847992940768:2049] [node 3] ICS09 handshake done sender: [1:7574435847992940753:2111] self: [1:7574435847992940751:0] peer: [3:7574435847665355622:0] socket: 482 qp: -1
2025-11-19T13:59:33.702268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435847992940768:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:33.702281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:33.701329Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435846363703298:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.703877Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435844932626321:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.705602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435847029769410:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-19T13:59:33.705622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435847029769411:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-19T13:59:33.705666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435847029769412:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-19T13:59:33.705858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435847992940774:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39539
2025-11-19T13:59:33.705901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435847992940775:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39539
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.709004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435847665355636:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42571
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.712039Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435847992940757:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:33.712115Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435847992940774:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.712169Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435847992940763:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.712349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435847992940748:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.712369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:33.712376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435847992940748:2108] [node 4] ICP22 created new session: [1:7574435847992940777:2108]
2025-11-19T13:59:33.712383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435847992940777:2108] [node 4] ICS09 handshake done sender: [1:7574435847992940757:2112] self: [1:7574435847992940754:0] peer: [4:7574435847029769404:0] socket: 483 qp: -1
2025-11-19T13:59:33.712390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435847992940777:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:33.712400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.715225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435846363703304:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34869
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.709425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435847665355637:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42571
2025-11-19T13:59:33.715978Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435847029769411:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.716340Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435847029769323:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435847029769396:2112] held: no
2025-11-19T13:59:33.716384Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435847992940775:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.715872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435844932626323:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63225
2025-11-19T13:59:33.716761Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435844932626323:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.716887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435844932626241:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.716906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435844932626324:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63225
2025-11-19T13:59:33.716567Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435846363703267:2105] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:33.716735Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435846363703278:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:33.716806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435846363703275:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.716822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:33.716828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435846363703275:2109] [node 5] ICP22 created new session: [2:7574435846363703306:2109]
2025-11-19T13:59:33.716835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435846363703306:2109] [node 5] ICS09 handshake done sender: [2:7574435846363703278:2111] self: [2:7574435846363703277:0] peer: [5:7574435844932626317:0] socket: 492 qp: -1
2025-11-19T13:59:33.716840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435846363703306:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:33.716851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:33.717269Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435847992940762:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.717278Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435847992940762:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.718177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435844932626326:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63225
2025-11-19T13:59:33.717234Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435847665355637:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.718434Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435847665355606:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:33.718527Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435844932626314:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:33.718599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435844932626242:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.718615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:33.718621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435844932626242:2049] [node 2] ICP22 created new session: [5:7574435844932626327:2049]
2025-11-19T13:59:33.718627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435844932626327:2049] [node 2] ICS09 handshake done sender: [5:7574435844932626314:2112] self: [5:7574435844932626317:0] peer: [2:7574435846363703277:0] socket: 497 qp: -1
2025-11-19T13:59:33.718634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435844932626327:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:33.718646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:33.719925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435847665355546:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.719949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:33.719955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435847665355546:2048] [node 1] ICP22 created new session: [3:7574435847665355640:2048]
2025-11-19T13:59:33.719962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435847665355640:2048] [node 1] ICS09 handshake done sender: [3:7574435847665355606:2107] self: [3:7574435847665355622:0] peer: [1:7574435847992940751:0] socket: 484 qp: -1
2025-11-19T13:59:33.719969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435847665355640:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:33.719986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:33.720171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435846363703209:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.720191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:33.720196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435846363703209:2048] [node 1] ICP22 created new session: [2:7574435846363703308:2048]
2025-11-19T13:59:33.720202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435846363703308:2048] [node 1] ICS09 handshake done sender: [2:7574435846363703267:2105] self: [2:7574435846363703289:0] peer: [1:7574435847992940743:0] socket: 481 qp: -1
2025-11-19T13:59:33.720206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435846363703308:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:33.720224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.719971Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435847992940762:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:33.723101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435847992940750:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.723117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:33.723124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435847992940750:2109] [node 5] ICP22 created new session: [1:7574435847992940780:2109]
2025-11-19T13:59:33.723130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435847992940780:2109] [node 5] ICS09 handshake done sender: [1:7574435847992940762:2115] self: [1:7574435847992940755:0] peer: [5:7574435844932626325:0] socket: 531 qp: -1
2025-11-19T13:59:33.723138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435847992940780:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:33.723171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.723548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435844932626330:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63225
2025-11-19T13:59:33.723567Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435844932626323:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:33.721697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435845364908912:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8957
2025-11-19T13:59:33.721766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435845364908913:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8957
2025-11-19T13:59:33.721779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435845364908914:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8957
2025-11-19T13:59:33.724076Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435845364908912:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.724113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435845364908822:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.724225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435847992940761:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.724236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435847992940761:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.724276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435847992940683:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(22137844ns) {HandleHandshakeStatus:329(22137040ns) {DropIncomingHandshake:441(78ns) DropOutgoingHandshake:456(233ns) SwitchToState:178(189ns) ProcessPendingSessionEvents:495(16118026ns) {ForwardSessionEventToSession:612(16105206ns) ForwardSessionEventToSession:612(1579ns) ForwardSessionEventToSession:612(1332ns) ForwardSessionEventToSession:612(621ns) ForwardSessionEventToSession:612(854ns) ForwardSessionEventToSession:612(1068ns) ForwardSessionEventToSession:612(593ns) ForwardSessionEventToSession:612(589ns) ForwardSessionEventToSession:612(896ns) ForwardSessionEventToSession:612(1097ns)}}}
2025-11-19T13:59:33.724327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435847992940683:2048] [node 2] ICP18 handshake (actor [1:7574435847992940775:2118]) is too old
2025-11-19T13:59:33.724577Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435847992940684:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(22302802ns) {HandleHandshakeStatus:329(22301872ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(108ns) SwitchToState:178(137ns) ProcessPendingSessionEvents:495(16365094ns) {ForwardSessionEventToSession:612(16353327ns) ForwardSessionEventToSession:612(1086ns) ForwardSessionEventToSession:612(1069ns) ForwardSessionEventToSession:612(705ns) ForwardSessionEventToSession:612(534ns) ForwardSessionEventToSession:612(1342ns) ForwardSessionEventToSession:612(699ns) ForwardSessionEventToSession:612(581ns) ForwardSessionEventToSession:612(587ns) ForwardSessionEventToSession:612(954ns)}}}
2025-11-19T13:59:33.724644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435847992940684:2049] [node 3] ICP18 handshake (actor [1:7574435847992940763:2116]) is too old
2025-11-19T13:59:33.725218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435844932626241:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.725238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:33.725244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435844932626241:2048] [node 1] ICP22 created new session: [5:7574435844932626331:2048]
2025-11-19T13:59:33.725249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435844932626331:2048] [node 1] ICS09 handshake done sender: [5:7574435844932626323:2117] self: [5:7574435844932626325:0] peer: [1:7574435847992940755:0] socket: 533 qp: -1
2025-11-19T13:59:33.725254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435844932626331:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:33.725261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:33.725744Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435845364908914:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.725793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435845364908885:2099] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.728230Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435844932626319:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.728243Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435844932626319:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.728969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435845364908917:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8957
2025-11-19T13:59:33.729109Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435844932626319:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:33.729226Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435844932626330:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.729238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435844932626310:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.729252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:33.729257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435844932626310:2107] [node 6] ICP22 created new session: [5:7574435844932626334:2107]
2025-11-19T13:59:33.729262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435844932626334:2107] [node 6] ICS09 handshake done sender: [5:7574435844932626319:2114] self: [5:7574435844932626315:0] peer: [6:7574435845364908916:0] socket: 513 qp: -1
2025-11-19T13:59:33.729268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435844932626334:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:33.729275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:33.730948Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435847665355634:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.730962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435847665355634:2119] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.730839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435844932626243:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.731738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435844932626337:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63225
2025-11-19T13:59:33.732048Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435844932626330:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:33.732169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435844932626243:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.732187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:33.732193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435844932626243:2050] [node 3] ICP22 created new session: [5:7574435844932626338:2050]
2025-11-19T13:59:33.732199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435844932626338:2050] [node 3] ICS09 handshake done sender: [5:7574435844932626330:2120] self: [5:7574435844932626336:0] peer: [3:7574435847665355620:0] socket: 543 qp: -1
2025-11-19T13:59:33.732205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435844932626338:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:33.732213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:33.729660Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435845364908914:2121] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:33.729756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435845364908885:2099] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.729768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:33.729774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435845364908885:2099] [node 5] ICP22 created new session: [6:7574435845364908918:2099]
2025-11-19T13:59:33.729779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435845364908918:2099] [node 5] ICS09 handshake done sender: [6:7574435845364908914:2121] self: [6:7574435845364908916:0] peer: [5:7574435844932626315:0] socket: 542 qp: -1
2025-11-19T13:59:33.729787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435845364908918:2099] [node 5] ICS10 traffic start
2025-11-19T13:59:33.729807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:33.731874Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435847665355634:2119] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:33.733122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435847665355611:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.733135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:33.733141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435847665355611:2109] [node 5] ICP22 created new session: [3:7574435847665355644:2109]
2025-11-19T13:59:33.733145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435847665355644:2109] [node 5] ICS09 handshake done sender: [3:7574435847665355634:2119] self: [3:7574435847665355620:0] peer: [5:7574435844932626336:0] socket: 540 qp: -1
2025-11-19T13:59:33.733150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435847665355644:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:33.733157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:33.738273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435847029769385:2108] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:33.738306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [4:7574435847029769385:2108] [node 6] ICP35 event processing took too much time StateWork:293(31605658ns) {HandleHandshakeStatus:395(31604507ns) {LogHandshakeFail:472(3254ns) {UpdateErrorStateLog:544(1198ns)}}}
2025-11-19T13:59:33.739081Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435845364908913:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.739127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435845364908824:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.738262Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435847665355547:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(48921212ns) {HandleHandshakeStatus:329(48920585ns) {DropIncomingHandshake:441(265ns) DropOutgoingHandshake:456(1692ns) SwitchToState:178(153ns) ProcessPendingSessionEvents:495(31676279ns) {ForwardSessionEventToSession:612(31663238ns) ForwardSessionEventToSession:612(1710ns) ForwardSessionEventToSession:612(1358ns) ForwardSessionEventToSession:612(731ns) ForwardSessionEventToSession:612(649ns) ForwardSessionEventToSession:612(947ns) ForwardSessionEventToSession:612(664ns) ForwardSessionEventToSession:612(576ns) ForwardSessionEventToSession:612(529ns) ForwardSessionEventToSession:612(848ns)}}}
2025-11-19T13:59:33.738760Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435847665355546:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(18805527ns) {HandleHandshakeStatus:329(18804639ns) {DropIncomingHandshake:441(237ns) DropOutgoingHandshake:456(1508ns) SwitchToState:178(117ns) ProcessPendingSessionEvents:495(17873600ns) {ForwardSessionEventToSession:612(17861569ns) ForwardSessionEventToSession:612(1993ns) ForwardSessionEventToSession:612(961ns) ForwardSessionEventToSession:612(723ns) ForwardSessionEventToSession:612(593ns) ForwardSessionEventToSession:612(1156ns) ForwardSessionEventToSession:612(947ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(593ns) ForwardSessionEventToSession:612(791ns)}}}
2025-11-19T13:59:33.740201Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435846363703209:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20001370ns) {HandleHandshakeStatus:329(20000673ns) {DropIncomingHandshake:441(136ns) DropOutgoingHandshake:456(5796ns) SwitchToState:178(59ns) ProcessPendingSessionEvents:495(19270436ns) {ForwardSessionEventToSession:612(19258604ns) ForwardSessionEventToSession:612(1197ns) ForwardSessionEventToSession:612(982ns) ForwardSessionEventToSession:612(713ns) ForwardSessionEventToSession:612(914ns) ForwardSessionEventToSession:612(1390ns) ForwardSessionEventToSession:612(824ns) ForwardSessionEventToSession:612(648ns) ForwardSessionEventToSession:612(1024ns) ForwardSessionEventToSession:612(811ns)}}}
2025-11-19T13:59:33.740459Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435847992940761:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:33.741058Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435847665355631:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.741069Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435847665355631:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.741594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435847992940752:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.741613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:33.741620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435847992940752:2110] [node 6] ICP22 created new session: [1:7574435847992940783:2110]
2025-11-19T13:59:33.741625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435847992940783:2110] [node 6] ICS09 handshake done sender: [1:7574435847992940761:2114] self: [1:7574435847992940759:0] peer: [6:7574435845364908915:0] socket: 530 qp: -1
2025-11-19T13:59:33.741631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435847992940783:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:33.741640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:33.741876Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435847665355631:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:33.741978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435847665355614:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.741995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:33.742001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435847665355614:2110] [node 6] ICP22 created new session: [3:7574435847665355647:2110]
2025-11-19T13:59:33.742006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435847665355647:2110] [node 6] ICS09 handshake done sender: [3:7574435847665355631:2118] self: [3:7574435847665355621:0] peer: [6:7574435845364908920:0] socket: 539 qp: -1
2025-11-19T13:59:33.742012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435847665355647:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:33.742018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:33.742963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435845364908822:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435845364908912:2119] held: no
2025-11-19T13:59:33.762038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435847029769324:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.762107Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435847029769410:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.762186Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435847029769396:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:33.762382Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435846363703300:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.762394Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435846363703300:2119] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.764325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435847665355623:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.764336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435847665355623:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.764294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435844932626340:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63225
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.765221Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435846363703300:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:33.766117Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435847665355623:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:33.763922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435847029769323:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.763949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:33.763960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435847029769323:2048] [node 1] ICP22 created new session: [4:7574435847029769416:2048]
2025-11-19T13:59:33.763962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435847029769325:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.763967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435847029769416:2048] [node 1] ICS09 handshake done sender: [4:7574435847029769396:2112] self: [4:7574435847029769404:0] peer: [1:7574435847992940754:0] socket: 485 qp: -1
2025-11-19T13:59:33.763978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435847029769416:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:33.763988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:33.764113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435847029769414:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-19T13:59:33.766199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435847029769419:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18524
2025-11-19T13:59:33.766493Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435847029769411:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:33.766749Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435847029769410:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:33.767688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435847029769324:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.767710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:33.767719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435847029769324:2049] [node 2] ICP22 created new session: [4:7574435847029769420:2049]
2025-11-19T13:59:33.767726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435847029769420:2049] [node 2] ICS09 handshake done sender: [4:7574435847029769411:2119] self: [4:7574435847029769413:0] peer: [2:7574435846363703290:0] socket: 535 qp: -1
2025-11-19T13:59:33.767735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435847029769420:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:33.767742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:33.768608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435847029769325:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.768620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:33.768627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435847029769325:2050] [node 3] ICP22 created new session: [4:7574435847029769422:2050]
2025-11-19T13:59:33.768632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435847029769422:2050] [node 3] ICS09 handshake done sender: [4:7574435847029769410:2118] self: [4:7574435847029769417:0] peer: [3:7574435847665355615:0] socket: 516 qp: -1
2025-11-19T13:59:33.768639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435847029769422:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:33.768646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:33.769950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435847665355608:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.769974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:33.769991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435847665355608:2108] [node 4] ICP22 created new session: [3:7574435847665355650:2108]
2025-11-19T13:59:33.769997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435847665355650:2108] [node 4] ICS09 handshake done sender: [3:7574435847665355623:2114] self: [3:7574435847665355615:0] peer: [4:7574435847029769417:0] socket: 514 qp: -1
2025-11-19T13:59:33.770003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435847665355650:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:33.770009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.771252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435845364908824:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19366) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435845364908913:2120] held: no
2025-11-19T13:59:33.771298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435845364908921:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8957
2025-11-19T13:59:33.771495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435845364908922:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8957
2025-11-19T13:59:33.771357Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435844932626340:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.771413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435844932626298:2101] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:33.771858Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435845364908913:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:33.770495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435846363703284:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.770552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:33.770561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435846363703284:2108] [node 4] ICP22 created new session: [2:7574435846363703311:2108]
2025-11-19T13:59:33.770568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435846363703311:2108] [node 4] ICS09 handshake done sender: [2:7574435846363703300:2119] self: [2:7574435846363703290:0] peer: [4:7574435847029769413:0] socket: 526 qp: -1
2025-11-19T13:59:33.770575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435846363703311:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:33.770588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:33.771965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435845364908824:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.771977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:33.771982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435845364908824:2050] [node 3] ICP22 created new session: [6:7574435845364908923:2050]
2025-11-19T13:59:33.771988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435845364908923:2050] [node 3] ICS09 handshake done sender: [6:7574435845364908913:2120] self: [6:7574435845364908920:0] peer: [3:7574435847665355621:0] socket: 541 qp: -1
2025-11-19T13:59:33.771993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435845364908923:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:33.772011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:33.772355Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435845364908912:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:33.772490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435844932626342:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63225
2025-11-19T13:59:33.771670Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435847029769401:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:33.771680Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435847029769401:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:33.772675Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435847029769401:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:33.772856Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435844932626340:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:33.772761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435847029769399:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:33.772779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:33.772784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435847029769399:2113] [node 5] ICP22 created new session: [4:7574435847029769425:2113]
2025-11-19T13:59:33.772789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435847029769425:2113] [node 5] ICS09 handshake done sender: [4:7574435847029769401:2115] self: [4:7574435847029769400:0] peer: [5:7574435844932626341:0] socket: 510 qp: -1
2025-11-19T13:59:33.772794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435847029769425:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:33.772800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:33.772983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435845364908822:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.772995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:33.773001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435845364908822:2048] [node 1] ICP22 created new session: [6:7574435845364908925:2048]
2025-11-19T13:59:33.773005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435845364908925:2048] [node 1] ICS09 handshake done sender: [6:7574435845364908912:2119] self: [6:7574435845364908915:0] peer: [1:7574435847992940759:0] socket: 532 qp: -1
2025-11-19T13:59:33.773010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435845364908925:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:33.773019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:33.773319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435844932626298:2101] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:33.773334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:33.773339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435844932626298:2101] [node 4] ICP22 created new session: [5:7574435844932626343:2101]
2025-11-19T13:59:33.773343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435844932626343:2101] [node 4] ICS09 handshake done sender: [5:7574435844932626340:2122] self: [5:7574435844932626341:0] peer: [4:7574435847029769400:0] socket: 537 qp: -1
2025-11-19T13:59:33.773352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435844932626343:2101] [node 4] ICS10 traffic start
2025-11-19T13:59:33.773363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:33.786220Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:33.786309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574435846363703308:2048] [node 1] ICS07 socket disconnect 481 reason# EndOfStream
2025-11-19T13:59:33.786317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435846363703308:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.786350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:33.786367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435846363703308:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:33.789094Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435846363703209:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29594) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:33.789109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435846363703308:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:33.789481Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435846363703209:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29594) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:33.789493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435846363703308:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:33.789500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435846363703209:2048] [node 1] ICP30 unregister session Session# [2:7574435846363703308:2048] VirtualId# [2:7574435846363703289:0]
2025-11-19T13:59:33.789517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:33.789523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574435846363703209:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29594) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:33.789525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29594) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:33.792152Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:33.792252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435847665355640:2048] [node 1] ICS07 socket disconnect 484 reason# EndOfStream
2025-11-19T13:59:33.792261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435847665355640:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.792332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:33.792361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435847665355640:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:33.793261Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435847665355546:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29594) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:33.793272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435847665355640:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:33.793277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435847665355546:2048] [node 1] ICP30 unregister session Session# [3:7574435847665355640:2048] VirtualId# [3:7574435847665355622:0]
2025-11-19T13:59:33.793298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:33.793304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435847665355546:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29594) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:33.793307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29594) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:33.802645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574435847665355546:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:33.803787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7574435846363703209:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:33.807249Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:33.807347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435844932626331:2048] [node 1] ICS01 socket: 533 reason# EndOfStream
2025-11-19T13:59:33.807356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435844932626241:2048] [node 1] ICP30 unregister session Session# [5:7574435844932626331:2048] VirtualId# [5:7574435844932626325:0]
2025-11-19T13:59:33.807364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435844932626331:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.807419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:33.807446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:33.808775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:33.808879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435845364908925:2048] [node 1] ICS01 socket: 532 reason# EndOfStream
2025-11-19T13:59:33.808887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435845364908822:2048] [node 1] ICP30 unregister session Session# [6:7574435845364908925:2048] VirtualId# [6:7574435845364908915:0]
2025-11-19T13:59:33.808893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435845364908925:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.808958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:33.808984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:33.809071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:33.809141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435847029769416:2048] [node 1] ICS01 socket: 485 reason# EndOfStream
2025-11-19T13:59:33.809146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435847029769323:2048] [node 1] ICP30 unregister session Session# [4:7574435847029769416:2048] VirtualId# [4:7574435847029769404:0]
2025-11-19T13:59:33.809152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435847029769416:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.809185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:33.809208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:33.819594Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:33.819685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435847665355632:2049] [node 2] ICS07 socket disconnect 507 reason# EndOfStream
2025-11-19T13:59:33.819692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435847665355632:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.819734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:33.819768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435847665355632:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:33.821816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435847665355547:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13704) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:33.821829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435847665355632:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:33.823331Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:33.823427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435845364908906:2049] [node 2] ICS01 socket: 494 reason# EndOfStream
2025-11-19T13:59:33.823435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435845364908823:2049] [node 2] ICP30 unregister session Session# [6:7574435845364908906:2049] VirtualId# [6:7574435845364908896:0]
2025-11-19T13:59:33.823443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435845364908906:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.823491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:33.823514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:33.824209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:33.824301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435844932626327:2049] [node 2] ICS01 socket: 497 reason# EndOfStream
2025-11-19T13:59:33.824309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435844932626242:2049] [node 2] ICP30 unregister session Session# [5:7574435844932626327:2049] VirtualId# [5:7574435844932626317:0]
2025-11-19T13:59:33.824315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435844932626327:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.824365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:33.824391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:33.827927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:33.828019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435847029769420:2049] [node 2] ICS01 socket: 535 reason# EndOfStream
2025-11-19T13:59:33.828027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435847029769324:2049] [node 2] ICP30 unregister session Session# [4:7574435847029769420:2049] VirtualId# [4:7574435847029769413:0]
2025-11-19T13:59:33.828034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435847029769420:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.828083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:33.828107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:33.862962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:33.863058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435845364908923:2050] [node 3] ICS01 socket: 541 reason# EndOfStream
2025-11-19T13:59:33.863067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435845364908824:2050] [node 3] ICP30 unregister session Session# [6:7574435845364908923:2050] VirtualId# [6:7574435845364908920:0]
2025-11-19T13:59:33.863075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435845364908923:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.863137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:33.863162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:33.864593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:33.864673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435844932626338:2050] [node 3] ICS01 socket: 543 reason# EndOfStream
2025-11-19T13:59:33.864681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435844932626243:2050] [node 3] ICP30 unregister session Session# [5:7574435844932626338:2050] VirtualId# [5:7574435844932626336:0]
2025-11-19T13:59:33.864687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435844932626338:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.864740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:33.864766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:33.882980Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:33.883081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435845364908908:2105] [node 4] ICS01 socket: 491 reason# EndOfStream
2025-11-19T13:59:33.883091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435845364908886:2105] [node 4] ICP30 unregister session Session# [6:7574435845364908908:2105] VirtualId# [6:7574435845364908900:0]
2025-11-19T13:59:33.883098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435845364908908:2105] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:33.883158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:33.883182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:33.962417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435844868926714:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:33.962447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435844868926715:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:33.995721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435846513228503:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:33.995735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435846513228504:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:34.060645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435851602204298:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:34.060663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435851602204299:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:34.092021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435851461296063:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:34.092036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435851461296064:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:34.092041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435851461296065:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:34.121649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435849937413138:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:34.121662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435849937413139:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:34.121666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435849937413140:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:34.175871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435849393550930:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:34.175916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435849393550931:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:34.175922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435849393550932:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:34.215450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435849393550951:2060] [node 4] ICP01 ready to work
2025-11-19T13:59:34.215555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435849393550954:2062] [node 5] ICP01 ready to work
2025-11-19T13:59:34.215787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435849937413174:2079] [node 6] ICP01 ready to work
2025-11-19T13:59:34.218248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435849163894072:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:34.218366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435849163894073:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:34.228564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435849937413203:2092] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.241021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435846513228504:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(16374019ns) {Configure:104(16371870ns) {ProcessConfigured:135(15888110ns) {StartInitialHandshake:157(15887392ns) {DropHandshakes:469(834ns) {DropIncomingHandshake:441(139ns) DropOutgoingHandshake:456(189ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(1512ns) {SwitchToState:178(822ns)}}}}}
2025-11-19T13:59:34.241236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435850808195864:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:34.241326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435850808195865:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:34.241463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435850808195867:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:34.242944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435850808195866:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:578
2025-11-19T13:59:34.243453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435851602204361:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.243575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435851602204363:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:34.244792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435851602204359:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63047
2025-11-19T13:59:34.244871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435851602204360:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63047
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.252079Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435849393550951:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(36448710ns) {Configure:104(36448084ns) {ProcessConfigured:135(36157642ns) {StartInitialHandshake:157(36156870ns) {DropHandshakes:469(530ns) {DropIncomingHandshake:441(99ns) DropOutgoingHandshake:456(126ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:476(1583ns) {SwitchToState:178(971ns)}}}}}
2025-11-19T13:59:34.253251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849393551005:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49463
2025-11-19T13:59:34.253265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849393551006:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49463
2025-11-19T13:59:34.260797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435844868926714:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(42379678ns) {Configure:104(42378755ns) {ProcessConfigured:135(41961642ns) {StartInitialHandshake:157(41960781ns) {DropHandshakes:469(628ns) {DropIncomingHandshake:441(99ns) DropOutgoingHandshake:456(170ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(1924ns) {SwitchToState:178(1086ns)}}}}}
2025-11-19T13:59:34.261847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435849163894083:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.262265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435849163894084:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20569
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.264063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435849163894085:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20569
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.269452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435849937413216:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-19T13:59:34.269473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435849937413217:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-19T13:59:34.272282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435851461296129:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:34.272370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435851461296130:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:34.274653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435849163894086:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20569
2025-11-19T13:59:34.273882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435851602204376:2115] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.286523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435851461296131:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43842
2025-11-19T13:59:34.286942Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435849937413217:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.287001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435849937413218:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-19T13:59:34.287012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435849937413219:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-19T13:59:34.287362Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435849393551005:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.287391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435849393550954:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.287514Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435849937413202:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.287520Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435849937413202:2107] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.288339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849393551011:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49463
2025-11-19T13:59:34.288444Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435849937413202:2107] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:34.288628Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435849393551005:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:34.288686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435849393550954:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.288705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:34.288711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435849393550954:2062] [node 5] ICP22 created new session: [6:7574435849393551012:2062]
2025-11-19T13:59:34.288717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435849393551012:2062] [node 5] ICS09 handshake done sender: [6:7574435849393551005:2115] self: [6:7574435849393551010:0] peer: [5:7574435849937413201:0] socket: 500 qp: -1
2025-11-19T13:59:34.288723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435849393551012:2062] [node 5] ICS10 traffic start
2025-11-19T13:59:34.288733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:34.289436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435849937413174:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.289454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:34.289459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435849937413174:2079] [node 6] ICP22 created new session: [5:7574435849937413222:2079]
2025-11-19T13:59:34.289463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435849937413222:2079] [node 6] ICS09 handshake done sender: [5:7574435849937413202:2107] self: [5:7574435849937413201:0] peer: [6:7574435849393551010:0] socket: 488 qp: -1
2025-11-19T13:59:34.289468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435849937413222:2079] [node 6] ICS10 traffic start
2025-11-19T13:59:34.289473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:34.291498Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435851461296131:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.295007Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435849937413219:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.295036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435849937413203:2092] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.295148Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435851461296135:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.295157Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435851461296135:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.296057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435849937413226:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.301533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435850808195879:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:578
2025-11-19T13:59:34.301731Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435850808195866:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.304969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435850808195880:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:578
2025-11-19T13:59:34.304997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435850808195881:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:578
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.296429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435851602204359:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.296945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435851602204377:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63047
2025-11-19T13:59:34.305037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435850808195882:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:578
2025-11-19T13:59:34.296980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435851602204378:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63047
2025-11-19T13:59:34.307925Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435849393551006:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.308099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435849393550932:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.308835Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435849937413216:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.308866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435849937413139:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.309731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435849937413228:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-19T13:59:34.308443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435849163894092:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20569
2025-11-19T13:59:34.308510Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435850808195882:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.308982Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435850808195871:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.308986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435850808195871:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.312304Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435851602204377:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.312358Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435851602204367:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.312362Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435851602204367:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.313730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849393551016:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49463
2025-11-19T13:59:34.313847Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435851602204367:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:34.314044Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435849393551006:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:34.314214Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435849937413216:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:34.314283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435849937413139:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.314300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:34.314306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435849937413139:2049] [node 2] ICP22 created new session: [5:7574435849937413229:2049]
2025-11-19T13:59:34.314312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435849937413229:2049] [node 2] ICS09 handshake done sender: [5:7574435849937413216:2115] self: [5:7574435849937413227:0] peer: [2:7574435850808195870:0] socket: 508 qp: -1
2025-11-19T13:59:34.314320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435849937413229:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:34.314327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:34.314108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435849393550932:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.314125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:34.314131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435849393550932:2050] [node 3] ICP22 created new session: [6:7574435849393551017:2050]
2025-11-19T13:59:34.314137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435849393551017:2050] [node 3] ICS09 handshake done sender: [6:7574435849393551006:2116] self: [6:7574435849393551015:0] peer: [3:7574435851602204366:0] socket: 501 qp: -1
2025-11-19T13:59:34.314144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435849393551017:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:34.314153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:34.314018Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435850808195871:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:34.316740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435851602204361:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.316761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:34.316768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435851602204361:2107] [node 6] ICP22 created new session: [3:7574435851602204384:2107]
2025-11-19T13:59:34.316774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435851602204384:2107] [node 6] ICS09 handshake done sender: [3:7574435851602204367:2112] self: [3:7574435851602204366:0] peer: [6:7574435849393551015:0] socket: 497 qp: -1
2025-11-19T13:59:34.316781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435851602204384:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:34.316790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:34.316119Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435851461296135:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:34.316324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435851461296129:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.316334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:34.316340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435851461296129:2109] [node 5] ICP22 created new session: [4:7574435851461296144:2109]
2025-11-19T13:59:34.316346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435851461296144:2109] [node 5] ICS09 handshake done sender: [4:7574435851461296135:2113] self: [4:7574435851461296134:0] peer: [5:7574435849937413225:0] socket: 518 qp: -1
2025-11-19T13:59:34.316352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435851461296144:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:34.316359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:34.319850Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435849937413219:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.320033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435849937413203:2092] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.320048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:34.320054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435849937413203:2092] [node 4] ICP22 created new session: [5:7574435849937413233:2092]
2025-11-19T13:59:34.320060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435849937413233:2092] [node 4] ICS09 handshake done sender: [5:7574435849937413219:2118] self: [5:7574435849937413225:0] peer: [4:7574435851461296134:0] socket: 519 qp: -1
2025-11-19T13:59:34.320066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435849937413233:2092] [node 4] ICS10 traffic start
2025-11-19T13:59:34.320075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:34.320190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435849937413231:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-19T13:59:34.320500Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435850808195879:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.320568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435850808195864:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.320586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:34.320590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435850808195864:2107] [node 5] ICP22 created new session: [2:7574435850808195886:2107]
2025-11-19T13:59:34.320596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435850808195886:2107] [node 5] ICS09 handshake done sender: [2:7574435850808195871:2113] self: [2:7574435850808195870:0] peer: [5:7574435849937413227:0] socket: 494 qp: -1
2025-11-19T13:59:34.320600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435850808195886:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:34.320606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:34.324450Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435850808195881:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.325291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435851461296146:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43842
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.326220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849393551020:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49463
2025-11-19T13:59:34.326449Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435851602204360:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.326478Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435851602204378:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.326501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435851602204299:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.326511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435851602204298:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.327841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435851461296148:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43842
2025-11-19T13:59:34.325414Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435849163894085:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.325482Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435849163894092:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.325499Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435849163894086:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.325605Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435849163894084:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.325688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435849163894093:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20569
2025-11-19T13:59:34.329188Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435849937413218:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.329220Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435849937413231:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.329245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435849937413138:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.329256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435849937413140:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.329563Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435849393551020:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.329589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435849393550951:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.329689Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435851461296137:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.329696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435851461296137:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.325515Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435850808195880:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.330315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849393551022:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49463
2025-11-19T13:59:34.325622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435846513228503:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.327024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435850808195863:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.327029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435850808195863:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.332073Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435851602204371:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.332087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435851602204371:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.332581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435851602204389:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63047
2025-11-19T13:59:34.332784Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435851602204371:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:34.332535Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435849163894076:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.332550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435849163894076:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.332993Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435849163894093:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.333015Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435849163894080:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.333017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435849163894080:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.333158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435851602204363:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.333188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:34.333195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435851602204363:2110] [node 5] ICP22 created new session: [3:7574435851602204391:2110]
2025-11-19T13:59:34.333201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435851602204391:2110] [node 5] ICS09 handshake done sender: [3:7574435851602204371:2114] self: [3:7574435851602204370:0] peer: [5:7574435849937413236:0] socket: 511 qp: -1
2025-11-19T13:59:34.333207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435851602204391:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:34.333215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:34.333776Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435851602204360:2109] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:34.333923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435851602204299:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.333934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:34.333940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435851602204299:2049] [node 2] ICP22 created new session: [3:7574435851602204394:2049]
2025-11-19T13:59:34.333946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435851602204394:2049] [node 2] ICS09 handshake done sender: [3:7574435851602204360:2109] self: [3:7574435851602204387:0] peer: [2:7574435850808195859:0] socket: 495 qp: -1
2025-11-19T13:59:34.333950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435851602204394:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:34.333957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:34.332968Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435850808195863:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:34.333115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435846513228504:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.333137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:34.333145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435846513228504:2049] [node 3] ICP22 created new session: [2:7574435850808195893:2049]
2025-11-19T13:59:34.333151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435850808195893:2049] [node 3] ICS09 handshake done sender: [2:7574435850808195863:2110] self: [2:7574435850808195859:0] peer: [3:7574435851602204387:0] socket: 492 qp: -1
2025-11-19T13:59:34.333158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435850808195893:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:34.333167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:34.335320Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435851461296137:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:34.335558Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435851461296148:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.335616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435851461296130:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.335630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:34.335637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435851461296130:2110] [node 6] ICP22 created new session: [4:7574435851461296151:2110]
2025-11-19T13:59:34.335641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435851461296151:2110] [node 6] ICS09 handshake done sender: [4:7574435851461296137:2114] self: [4:7574435851461296136:0] peer: [6:7574435849393551021:0] socket: 491 qp: -1
2025-11-19T13:59:34.335646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435851461296151:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:34.335653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:34.336412Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435849393551020:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:34.336667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435849393550951:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.336680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:34.336688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435849393550951:2060] [node 4] ICP22 created new session: [6:7574435849393551023:2060]
2025-11-19T13:59:34.336693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435849393551023:2060] [node 4] ICS09 handshake done sender: [6:7574435849393551020:2119] self: [6:7574435849393551021:0] peer: [4:7574435851461296136:0] socket: 535 qp: -1
2025-11-19T13:59:34.336698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435849393551023:2060] [node 4] ICS10 traffic start
2025-11-19T13:59:34.336706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:34.338581Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435849163894078:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.338600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435849163894078:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.339766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435851461296150:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43842
2025-11-19T13:59:34.339795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435851461296064:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.340249Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435849937413140:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18422) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435849937413231:2121] held: no
2025-11-19T13:59:34.340393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435849937413237:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.340252Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435849163894076:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:34.340403Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435849163894080:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:34.340643Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435850808195873:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.340657Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435850808195873:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.341175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435850808195895:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:578
2025-11-19T13:59:34.346847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435851602204396:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63047
2025-11-19T13:59:34.347231Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435850808195873:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:34.347821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435850808195865:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.347839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:34.347850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435850808195865:2108] [node 4] ICP22 created new session: [2:7574435850808195898:2108]
2025-11-19T13:59:34.347856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435850808195898:2108] [node 4] ICS09 handshake done sender: [2:7574435850808195873:2114] self: [2:7574435850808195872:0] peer: [4:7574435851461296154:0] socket: 534 qp: -1
2025-11-19T13:59:34.347865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435850808195898:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:34.347875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:34.348051Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435850808195880:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:34.347232Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435849163894078:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:34.347316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435844868926714:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.347337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:34.347344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435844868926714:2048] [node 2] ICP22 created new session: [1:7574435849163894102:2048]
2025-11-19T13:59:34.347352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435849163894102:2048] [node 2] ICS09 handshake done sender: [1:7574435849163894076:2110] self: [1:7574435849163894074:0] peer: [2:7574435850808195891:0] socket: 512 qp: -1
2025-11-19T13:59:34.347358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435849163894102:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:34.347368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:34.347502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435844868926715:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.347512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:34.347527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435844868926715:2049] [node 3] ICP22 created new session: [1:7574435849163894104:2049]
2025-11-19T13:59:34.347530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435849163894104:2049] [node 3] ICS09 handshake done sender: [1:7574435849163894080:2112] self: [1:7574435849163894079:0] peer: [3:7574435851602204388:0] socket: 514 qp: -1
2025-11-19T13:59:34.347534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435849163894104:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:34.347538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:34.347605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435849163894072:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.347613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:34.347616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435849163894072:2107] [node 5] ICP22 created new session: [1:7574435849163894106:2107]
2025-11-19T13:59:34.347619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435849163894106:2107] [node 5] ICS09 handshake done sender: [1:7574435849163894078:2111] self: [1:7574435849163894077:0] peer: [5:7574435849937413235:0] socket: 513 qp: -1
2025-11-19T13:59:34.347622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435849163894106:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:34.347628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.355889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849393551025:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49463
2025-11-19T13:59:34.355957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849393551026:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49463
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.356298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435851602204397:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63047
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.362422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435849937413238:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8037
2025-11-19T13:59:34.362620Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435849937413231:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:34.362697Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435849937413138:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22864) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435849937413218:2117] held: no
2025-11-19T13:59:34.362729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435849937413140:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.362743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:34.362750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435849937413140:2050] [node 3] ICP22 created new session: [5:7574435849937413239:2050]
2025-11-19T13:59:34.362758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435849937413239:2050] [node 3] ICS09 handshake done sender: [5:7574435849937413231:2121] self: [5:7574435849937413236:0] peer: [3:7574435851602204370:0] socket: 487 qp: -1
2025-11-19T13:59:34.362767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435849937413239:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:34.362778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:34.367070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435851461296064:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16898) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435851461296148:2117] held: no
2025-11-19T13:59:34.367180Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435851461296150:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.367371Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435851602204378:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:34.367456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435851602204298:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22864) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435851602204378:2117] held: no
2025-11-19T13:59:34.367477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435851602204298:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.367486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:34.367491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435851602204298:2048] [node 1] ICP22 created new session: [3:7574435851602204398:2048]
2025-11-19T13:59:34.367498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435851602204398:2048] [node 1] ICS09 handshake done sender: [3:7574435851602204378:2117] self: [3:7574435851602204388:0] peer: [1:7574435849163894079:0] socket: 516 qp: -1
2025-11-19T13:59:34.367504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435851602204398:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:34.367511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:34.367657Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435851602204397:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.365137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435846513228503:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.365163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:34.365171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435846513228503:2048] [node 1] ICP22 created new session: [2:7574435850808195900:2048]
2025-11-19T13:59:34.365180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435850808195900:2048] [node 1] ICS09 handshake done sender: [2:7574435850808195880:2117] self: [2:7574435850808195891:0] peer: [1:7574435849163894074:0] socket: 524 qp: -1
2025-11-19T13:59:34.365186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435850808195900:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:34.365195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:34.365278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435846513228503:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:34.367882Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435849393551025:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.367965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435849393550931:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.368120Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435849937413218:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:34.368296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435849937413138:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.368309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:34.368314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435849937413138:2048] [node 1] ICP22 created new session: [5:7574435849937413241:2048]
2025-11-19T13:59:34.368320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435849937413241:2048] [node 1] ICS09 handshake done sender: [5:7574435849937413218:2117] self: [5:7574435849937413235:0] peer: [1:7574435849163894077:0] socket: 515 qp: -1
2025-11-19T13:59:34.368326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435849937413241:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:34.368334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.368609Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435850808195875:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.368621Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435850808195875:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.370175Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435850808195875:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:34.370289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435850808195867:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.370300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:34.370305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435850808195867:2109] [node 6] ICP22 created new session: [2:7574435850808195903:2109]
2025-11-19T13:59:34.370308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435850808195903:2109] [node 6] ICS09 handshake done sender: [2:7574435850808195875:2115] self: [2:7574435850808195874:0] peer: [6:7574435849393551027:0] socket: 544 qp: -1
2025-11-19T13:59:34.370312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435850808195903:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:34.370318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:34.369322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435851461296155:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43842
2025-11-19T13:59:34.369781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435851461296156:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43842
2025-11-19T13:59:34.369842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435851461296065:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.369853Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435851461296148:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:34.369962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435851461296064:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.369971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:34.369976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435851461296064:2049] [node 2] ICP22 created new session: [4:7574435851461296158:2049]
2025-11-19T13:59:34.369980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435851461296158:2049] [node 2] ICS09 handshake done sender: [4:7574435851461296148:2117] self: [4:7574435851461296154:0] peer: [2:7574435850808195872:0] socket: 533 qp: -1
2025-11-19T13:59:34.369986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435851461296158:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:34.369994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:34.372004Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435851602204380:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.372017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435851602204380:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.374512Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435851461296156:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.374567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435851461296160:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43842
2025-11-19T13:59:34.374582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435851461296063:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.374715Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435849163894082:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.374735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435849163894082:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.375500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435851461296162:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43842
2025-11-19T13:59:34.376176Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435851461296156:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:34.376251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435851461296063:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.376268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:34.376274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435851461296063:2048] [node 1] ICP22 created new session: [4:7574435851461296163:2048]
2025-11-19T13:59:34.376281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435851461296163:2048] [node 1] ICS09 handshake done sender: [4:7574435851461296156:2120] self: [4:7574435851461296161:0] peer: [1:7574435849163894081:0] socket: 549 qp: -1
2025-11-19T13:59:34.376287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435851461296163:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:34.376294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:34.375985Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435849163894082:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:34.377226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435849163894073:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.377241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:34.377246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435849163894073:2108] [node 4] ICP22 created new session: [1:7574435849163894111:2108]
2025-11-19T13:59:34.377251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435849163894111:2108] [node 4] ICS09 handshake done sender: [1:7574435849163894082:2113] self: [1:7574435849163894081:0] peer: [4:7574435851461296161:0] socket: 506 qp: -1
2025-11-19T13:59:34.377257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435849163894111:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:34.377263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:34.378653Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435849393551026:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.379416Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435851602204380:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.380747Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435849393550931:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16898) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435849393551025:2121] held: no
2025-11-19T13:59:34.380814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849393551028:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49463
2025-11-19T13:59:34.380906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435851602204376:2115] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.380924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:34.380929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435851602204376:2115] [node 4] ICP22 created new session: [3:7574435851602204402:2115]
2025-11-19T13:59:34.380934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435851602204402:2115] [node 4] ICS09 handshake done sender: [3:7574435851602204380:2118] self: [3:7574435851602204379:0] peer: [4:7574435851461296157:0] socket: 489 qp: -1
2025-11-19T13:59:34.380939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435851602204402:2115] [node 4] ICS10 traffic start
2025-11-19T13:59:34.380947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:34.381297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435849393550930:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.381363Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435849393551025:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:34.381600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435849393550931:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.381612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:34.381629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435849393550931:2049] [node 2] ICP22 created new session: [6:7574435849393551030:2049]
2025-11-19T13:59:34.381634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435849393551030:2049] [node 2] ICS09 handshake done sender: [6:7574435849393551025:2121] self: [6:7574435849393551027:0] peer: [2:7574435850808195874:0] socket: 548 qp: -1
2025-11-19T13:59:34.381639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435849393551030:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:34.381646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:34.379691Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435851461296150:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:34.379816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435851461296065:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.379827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:34.379832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435851461296065:2050] [node 3] ICP22 created new session: [4:7574435851461296165:2050]
2025-11-19T13:59:34.379837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435851461296165:2050] [node 3] ICS09 handshake done sender: [4:7574435851461296150:2118] self: [4:7574435851461296157:0] peer: [3:7574435851602204379:0] socket: 496 qp: -1
2025-11-19T13:59:34.379843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435851461296165:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:34.379850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:34.382140Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435849163894088:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.382152Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435849163894088:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.382895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849393551032:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49463
2025-11-19T13:59:34.383033Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435849163894088:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:34.383195Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435849393551026:2122] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:34.383255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435849393550930:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.383268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:34.383272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435849393550930:2048] [node 1] ICP22 created new session: [6:7574435849393551033:2048]
2025-11-19T13:59:34.383277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435849393551033:2048] [node 1] ICS09 handshake done sender: [6:7574435849393551026:2122] self: [6:7574435849393551029:0] peer: [1:7574435849163894087:0] socket: 525 qp: -1
2025-11-19T13:59:34.383282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435849393551033:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:34.383289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:34.384333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435849163894083:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.384353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:34.384358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435849163894083:2109] [node 6] ICP22 created new session: [1:7574435849163894114:2109]
2025-11-19T13:59:34.384364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435849163894114:2109] [node 6] ICS09 handshake done sender: [1:7574435849163894088:2117] self: [1:7574435849163894087:0] peer: [6:7574435849393551029:0] socket: 507 qp: -1
2025-11-19T13:59:34.384368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435849163894114:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:34.384375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:34.399870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:34.399988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435850808195900:2048] [node 1] ICS01 socket: 524 reason# EndOfStream
2025-11-19T13:59:34.399998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435846513228503:2048] [node 1] ICP30 unregister session Session# [2:7574435850808195900:2048] VirtualId# [2:7574435850808195891:0]
2025-11-19T13:59:34.400006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435850808195900:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:34.400067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:34.400094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:34.400213Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:34.400274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435851602204398:2048] [node 1] ICS01 socket: 516 reason# EndOfStream
2025-11-19T13:59:34.400280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435851602204298:2048] [node 1] ICP30 unregister session Session# [3:7574435851602204398:2048] VirtualId# [3:7574435851602204388:0]
2025-11-19T13:59:34.400284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435851602204398:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:34.400323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:34.400347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:34.407071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:34.407155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435849937413241:2048] [node 1] ICS01 socket: 515 reason# EndOfStream
2025-11-19T13:59:34.407165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435849937413138:2048] [node 1] ICP30 unregister session Session# [5:7574435849937413241:2048] VirtualId# [5:7574435849937413235:0]
2025-11-19T13:59:34.407174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435849937413241:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:34.407224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:34.407253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:34.407289Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:34.407348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435849393551033:2048] [node 1] ICS01 socket: 525 reason# EndOfStream
2025-11-19T13:59:34.407353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435849393550930:2048] [node 1] ICP30 unregister session Session# [6:7574435849393551033:2048] VirtualId# [6:7574435849393551029:0]
2025-11-19T13:59:34.407358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435849393551033:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:34.407382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:34.407401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:34.411036Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:34.411123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435851461296163:2048] [node 1] ICS01 socket: 549 reason# EndOfStream
2025-11-19T13:59:34.411130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435851461296063:2048] [node 1] ICP30 unregister session Session# [4:7574435851461296163:2048] VirtualId# [4:7574435851461296161:0]
2025-11-19T13:59:34.411136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435851461296163:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:34.411185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:34.411206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:34.412605Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:34.412697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435851602204394:2049] [node 2] ICS01 socket: 495 reason# EndOfStream
2025-11-19T13:59:34.412703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435851602204299:2049] [node 2] ICP30 unregister session Session# [3:7574435851602204394:2049] VirtualId# [3:7574435851602204387:0]
2025-11-19T13:59:34.412709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435851602204394:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:34.412754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:34.412774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:34.424195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:34.424302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435851461296158:2049] [node 2] ICS01 socket: 533 reason# EndOfStream
2025-11-19T13:59:34.424326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435851461296064:2049] [node 2] ICP30 unregister session Session# [4:7574435851461296158:2049] VirtualId# [4:7574435851461296154:0]
2025-11-19T13:59:34.424333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435851461296158:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:34.424386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:34.424412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:34.426565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:34.426659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435849937413229:2049] [node 2] ICS01 socket: 508 reason# EndOfStream
2025-11-19T13:59:34.426666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435849937413139:2049] [node 2] ICP30 unregister session Session# [5:7574435849937413229:2049] VirtualId# [5:7574435849937413227:0]
2025-11-19T13:59:34.426673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435849937413229:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:34.426726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:34.426750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:34.426790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:34.426869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435849393551030:2049] [node 2] ICS01 socket: 548 reason# EndOfStream
2025-11-19T13:59:34.426875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435849393550931:2049] [node 2] ICP30 unregister session Session# [6:7574435849393551030:2049] VirtualId# [6:7574435849393551027:0]
2025-11-19T13:59:34.426880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435849393551030:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:34.426921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:34.426945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:34.438628Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:34.438737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435849393551017:2050] [node 3] ICS01 socket: 501 reason# EndOfStream
2025-11-19T13:59:34.438747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435849393550932:2050] [node 3] ICP30 unregister session Session# [6:7574435849393551017:2050] VirtualId# [6:7574435849393551015:0]
2025-11-19T13:59:34.438755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435849393551017:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:34.438808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:34.438847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:34.438882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:34.438941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435849937413239:2050] [node 3] ICS01 socket: 487 reason# EndOfStream
2025-11-19T13:59:34.438946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435849937413140:2050] [node 3] ICP30 unregister session Session# [5:7574435849937413239:2050] VirtualId# [5:7574435849937413236:0]
2025-11-19T13:59:34.438949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435849937413239:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:34.438972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:34.438991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:34.451100Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:34.451209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435849393551023:2060] [node 4] ICS01 socket: 535 reason# EndOfStream
2025-11-19T13:59:34.451219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435849393550951:2060] [node 4] ICP30 unregister session Session# [6:7574435849393551023:2060] VirtualId# [6:7574435849393551021:0]
2025-11-19T13:59:34.451227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435849393551023:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:34.451293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:34.451320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:34.647906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435850839286172:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:34.647922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435850839286173:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:34.698071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435848839756164:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:34.698088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435848839756165:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:34.744108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435852510533349:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:34.744122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435852510533350:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:34.784194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435848918607778:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:34.784208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435848918607779:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:34.784213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435848918607780:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:34.808060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435851759881660:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:34.808076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435851759881661:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:34.808081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435851759881662:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:34.816101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435849423714614:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:34.816111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435849423714615:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:34.816116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435849423714616:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:34.880840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435850839286239:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:34.881524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435851759881717:2099] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.882044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435852510533409:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47920
2025-11-19T13:59:34.882064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435852510533410:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47920
2025-11-19T13:59:34.881078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435849423714653:2082] [node 4] ICP01 ready to work
2025-11-19T13:59:34.881170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435849423714654:2085] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.883128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435848918607839:2104] [node 6] ICP01 ready to work
2025-11-19T13:59:34.883825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848918607836:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:878
2025-11-19T13:59:34.883840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848918607837:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:878
2025-11-19T13:59:34.883850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848918607838:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:878
2025-11-19T13:59:34.883930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435851759881724:2102] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.885581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435851759881725:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36944
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.888894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435850839286244:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:34.889464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849423714684:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7748
2025-11-19T13:59:34.893568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435848918607851:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.895969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435850839286246:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41500
2025-11-19T13:59:34.896225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435851759881725:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.896026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435848839756230:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:34.896114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435848839756231:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:34.896176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435848839756232:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:34.899095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435852510533413:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:34.896645Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435850839286246:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.896807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435850839286249:2114] [node 6] ICP01 ready to work
2025-11-19T13:59:34.897179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574435850839286249:2114] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.907486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435850839286251:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41500
2025-11-19T13:59:34.907706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435848839756243:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30837
2025-11-19T13:59:34.907771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435848839756244:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30837
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.908110Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848839756244:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.908134Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848839756243:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.908168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435848839756164:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.908630Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435852510533349:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23686110ns) {Configure:104(23684505ns) {ProcessConfigured:135(23304391ns) {StartInitialHandshake:157(23303605ns) {DropHandshakes:469(477ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(107ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:476(1162ns) {SwitchToState:178(638ns)}}}}}
2025-11-19T13:59:34.908675Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435852510533409:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.909093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435852510533349:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.910316Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435850839286251:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.910107Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435852510533410:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.910329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435852510533422:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:34.909376Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435851759881660:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16272286ns) {Configure:104(16271606ns) {ProcessConfigured:135(15912739ns) {StartInitialHandshake:157(15912366ns) {DropHandshakes:469(592ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:476(630ns) {SwitchToState:178(291ns)}}}}}
2025-11-19T13:59:34.911863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435848839756245:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30837
2025-11-19T13:59:34.911799Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435849423714664:2092] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.911824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435849423714664:2092] [node 1] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.911915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435851759881739:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36944
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.912549Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435848918607837:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.912579Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435848918607836:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.913730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848918607855:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:878
2025-11-19T13:59:34.914103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435848918607778:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.914600Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435850839286233:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.914616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435850839286233:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.914664Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435850839286235:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.914670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435850839286235:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.914984Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435851759881739:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.915015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435851759881661:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.920008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435848839756248:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30837
2025-11-19T13:59:34.920272Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848839756245:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.920024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435848918607838:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.920078Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435848918607855:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.921152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435848918607779:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.921427Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848839756234:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.921442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435848839756234:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.921381Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435849423714684:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.925911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435848918607851:2112] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(32282489ns) {Configure:104(32282053ns) {ProcessConfigured:135(32008395ns) {StartInitialHandshake:157(32007852ns) {DropHandshakes:469(227ns) {DropIncomingHandshake:441(26ns) DropOutgoingHandshake:456(32ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:476(1246ns) {SwitchToState:178(363ns)}}}}}
2025-11-19T13:59:34.926462Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435850839286241:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.926476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435850839286241:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.927102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435850839286252:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41500
2025-11-19T13:59:34.928351Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435850839286233:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:34.928527Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435850839286241:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:34.928051Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848839756236:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.928067Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435848839756236:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.931862Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435849423714664:2092] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:34.932096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435852510533413:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(32845193ns) {Configure:104(32844346ns) {ProcessConfigured:135(32522769ns) {StartInitialHandshake:157(32522039ns) {DropHandshakes:469(478ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(64ns)} GenerateSessionVirtualId:988(341ns) PrepareNewSessionHandshake:476(1612ns) {SwitchToState:178(1124ns)}}}}}
2025-11-19T13:59:34.932197Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435848918607841:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.932209Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435848918607841:2108] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.936872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435849423714653:2082] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.936902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435849423714614:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.936915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:34.936921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435849423714614:2048] [node 1] ICP22 created new session: [6:7574435849423714695:2048]
2025-11-19T13:59:34.936928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435849423714695:2048] [node 1] ICS09 handshake done sender: [6:7574435849423714664:2092] self: [6:7574435849423714663:0] peer: [1:7574435850839286250:0] socket: 502 qp: -1
2025-11-19T13:59:34.936935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435849423714695:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:34.936944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:34.936511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435848839756250:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30837
2025-11-19T13:59:34.936719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435851759881742:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36944
2025-11-19T13:59:34.942859Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435848839756234:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:34.943045Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435850839286235:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:34.943208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435850839286172:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.943225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:34.943232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435850839286172:2048] [node 2] ICP22 created new session: [1:7574435850839286258:2048]
2025-11-19T13:59:34.943240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435850839286258:2048] [node 2] ICS09 handshake done sender: [1:7574435850839286233:2107] self: [1:7574435850839286232:0] peer: [2:7574435848839756246:0] socket: 493 qp: -1
2025-11-19T13:59:34.943245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435850839286258:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:34.943255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:34.943370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435850839286239:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.943376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:34.943396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435850839286239:2109] [node 4] ICP22 created new session: [1:7574435850839286260:2109]
2025-11-19T13:59:34.943401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435850839286260:2109] [node 4] ICS09 handshake done sender: [1:7574435850839286241:2110] self: [1:7574435850839286240:0] peer: [4:7574435848918607857:0] socket: 496 qp: -1
2025-11-19T13:59:34.943404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435850839286260:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:34.943409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.944346Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435851759881739:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:34.943967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435852510533429:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.944475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435851759881661:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.944509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:34.944516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435851759881661:2049] [node 2] ICP22 created new session: [5:7574435851759881743:2049]
2025-11-19T13:59:34.944522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435851759881743:2049] [node 2] ICS09 handshake done sender: [5:7574435851759881739:2115] self: [5:7574435851759881741:0] peer: [2:7574435848839756233:0] socket: 518 qp: -1
2025-11-19T13:59:34.944529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435851759881743:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:34.944538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:34.945299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849423714697:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7748
2025-11-19T13:59:34.945173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435850839286173:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.945189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:34.945193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435850839286173:2049] [node 3] ICP22 created new session: [1:7574435850839286262:2049]
2025-11-19T13:59:34.945199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435850839286262:2049] [node 3] ICS09 handshake done sender: [1:7574435850839286235:2108] self: [1:7574435850839286234:0] peer: [3:7574435852510533418:0] socket: 494 qp: -1
2025-11-19T13:59:34.945204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435850839286262:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:34.945233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:34.945360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435850839286257:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41500
2025-11-19T13:59:34.945010Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435848918607841:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:34.945944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848918607863:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:878
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.949883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435852510533426:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47920
2025-11-19T13:59:34.960077Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435849423714614:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(23138113ns) {HandleHandshakeStatus:329(23136707ns) {DropIncomingHandshake:441(184ns) DropOutgoingHandshake:456(367ns) SwitchToState:178(136ns) ProcessPendingSessionEvents:495(14542981ns) {ForwardSessionEventToSession:612(14533520ns) ForwardSessionEventToSession:612(2437ns) ForwardSessionEventToSession:612(1110ns) ForwardSessionEventToSession:612(723ns)}}}
2025-11-19T13:59:34.960303Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435849423714684:2107] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:34.961835Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435850839286246:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:34.961930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435848918607839:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.961959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:34.961967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435848918607839:2104] [node 6] ICP22 created new session: [4:7574435848918607864:2104]
2025-11-19T13:59:34.961973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435848918607864:2104] [node 6] ICS09 handshake done sender: [4:7574435848918607841:2108] self: [4:7574435848918607840:0] peer: [6:7574435849423714693:0] socket: 506 qp: -1
2025-11-19T13:59:34.961980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435848918607864:2104] [node 6] ICS10 traffic start
2025-11-19T13:59:34.961989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:34.962552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435848839756230:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.962575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:34.962581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435848839756230:2108] [node 5] ICP22 created new session: [2:7574435848839756254:2108]
2025-11-19T13:59:34.962586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435848839756254:2108] [node 5] ICS09 handshake done sender: [2:7574435848839756234:2111] self: [2:7574435848839756233:0] peer: [5:7574435851759881741:0] socket: 514 qp: -1
2025-11-19T13:59:34.962591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435848839756254:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:34.962599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:34.962759Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435848839756244:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.965493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435848839756164:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7330) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435848839756244:2116] held: no
2025-11-19T13:59:34.965520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435848839756164:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.965535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:34.965541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435848839756164:2048] [node 1] ICP22 created new session: [2:7574435848839756256:2048]
2025-11-19T13:59:34.965546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435848839756256:2048] [node 1] ICS09 handshake done sender: [2:7574435848839756244:2116] self: [2:7574435848839756246:0] peer: [1:7574435850839286232:0] socket: 516 qp: -1
2025-11-19T13:59:34.965553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435848839756256:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:34.965559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:34.963996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849423714699:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7748
2025-11-19T13:59:34.964227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435849423714653:2082] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.964242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:34.964249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435849423714653:2082] [node 4] ICP22 created new session: [6:7574435849423714700:2082]
2025-11-19T13:59:34.964255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435849423714700:2082] [node 4] ICS09 handshake done sender: [6:7574435849423714684:2107] self: [6:7574435849423714693:0] peer: [4:7574435848918607840:0] socket: 511 qp: -1
2025-11-19T13:59:34.964261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435849423714700:2082] [node 4] ICS10 traffic start
2025-11-19T13:59:34.964270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.968046Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435852510533426:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.968187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435852510533433:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47920
2025-11-19T13:59:34.968186Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435848918607836:2105] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.968667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848918607866:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:878
2025-11-19T13:59:34.968698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848918607867:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:878
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.965762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435851759881746:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36944
2025-11-19T13:59:34.970062Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435852510533409:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:34.969947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435848918607778:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.969962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:34.969969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435848918607778:2048] [node 1] ICP22 created new session: [4:7574435848918607869:2048]
2025-11-19T13:59:34.969975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435848918607869:2048] [node 1] ICS09 handshake done sender: [4:7574435848918607836:2105] self: [4:7574435848918607857:0] peer: [1:7574435850839286240:0] socket: 503 qp: -1
2025-11-19T13:59:34.969980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435848918607869:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:34.969988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.970465Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848839756248:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.970500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435848839756253:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30837
2025-11-19T13:59:34.971056Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435848839756236:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:34.971166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435848839756231:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.971182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:34.971187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435848839756231:2109] [node 4] ICP22 created new session: [2:7574435848839756261:2109]
2025-11-19T13:59:34.971193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435848839756261:2109] [node 4] ICS09 handshake done sender: [2:7574435848839756236:2112] self: [2:7574435848839756235:0] peer: [4:7574435848918607858:0] socket: 515 qp: -1
2025-11-19T13:59:34.971197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435848839756261:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:34.971203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:34.971538Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435848918607855:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:34.971596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435848918607779:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.971606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:34.971611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435848918607779:2049] [node 2] ICP22 created new session: [4:7574435848918607871:2049]
2025-11-19T13:59:34.971615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435848918607871:2049] [node 2] ICS09 handshake done sender: [4:7574435848918607855:2116] self: [4:7574435848918607858:0] peer: [2:7574435848839756235:0] socket: 522 qp: -1
2025-11-19T13:59:34.971620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435848918607871:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:34.971626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:34.974607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435852510533349:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.974629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:34.974636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435852510533349:2048] [node 1] ICP22 created new session: [3:7574435852510533437:2048]
2025-11-19T13:59:34.974643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435852510533437:2048] [node 1] ICS09 handshake done sender: [3:7574435852510533409:2107] self: [3:7574435852510533418:0] peer: [1:7574435850839286234:0] socket: 498 qp: -1
2025-11-19T13:59:34.974649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435852510533437:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:34.974657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.975264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435852510533434:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47920
2025-11-19T13:59:34.975458Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435848918607866:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.975493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435848918607780:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.978349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848839756253:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.978889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574435850839286249:2114] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.978911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:34.978920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435850839286249:2114] [node 6] ICP22 created new session: [1:7574435850839286265:2114]
2025-11-19T13:59:34.978926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435850839286265:2114] [node 6] ICS09 handshake done sender: [1:7574435850839286246:2112] self: [1:7574435850839286250:0] peer: [6:7574435849423714663:0] socket: 507 qp: -1
2025-11-19T13:59:34.978934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435850839286265:2114] [node 6] ICS10 traffic start
2025-11-19T13:59:34.978943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:34.979102Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435850839286252:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.979278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435850839286173:2049] [node 3] ICP18 handshake (actor [1:7574435850839286252:2116]) is too old
2025-11-19T13:59:34.984973Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435849423714699:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.985016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435849423714654:2085] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.987443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435852510533436:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47920
2025-11-19T13:59:34.987572Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435852510533424:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.987577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435852510533424:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.987508Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435851759881727:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.987523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435851759881727:2105] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.988620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435851759881748:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36944
2025-11-19T13:59:34.989278Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435851759881727:2105] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:34.989614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435851759881724:2102] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.989651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:34.989660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435851759881724:2102] [node 6] ICP22 created new session: [5:7574435851759881749:2102]
2025-11-19T13:59:34.989668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435851759881749:2102] [node 6] ICS09 handshake done sender: [5:7574435851759881727:2105] self: [5:7574435851759881726:0] peer: [6:7574435849423714703:0] socket: 492 qp: -1
2025-11-19T13:59:34.989677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435851759881749:2102] [node 6] ICS10 traffic start
2025-11-19T13:59:34.989688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:34.988406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849423714704:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7748
2025-11-19T13:59:34.988470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435849423714705:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7748
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.992255Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435852510533434:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.992342Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435852510533436:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.992400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435852510533350:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.992118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435850839286267:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41500
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.993629Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435849423714704:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.993879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435848918607874:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:878
2025-11-19T13:59:34.994099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435849423714654:2085] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20624) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435849423714699:2117] held: no
2025-11-19T13:59:34.994132Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435849423714699:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:34.994374Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435850839286267:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.994256Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435851759881746:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.993283Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848839756240:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.993299Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435848839756240:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.994970Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435848839756240:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:34.995065Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435848839756238:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.995068Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435848839756238:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.995424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435848839756165:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.995440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:34.995446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435848839756165:2049] [node 3] ICP22 created new session: [2:7574435848839756265:2049]
2025-11-19T13:59:34.995451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435848839756265:2049] [node 3] ICS09 handshake done sender: [2:7574435848839756240:2114] self: [2:7574435848839756239:0] peer: [3:7574435852510533441:0] socket: 548 qp: -1
2025-11-19T13:59:34.995456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435848839756265:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:34.995464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:34.996275Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435851759881748:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.996316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435851759881660:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.996329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435851759881717:2099] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.996485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435850839286247:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.996495Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435850839286247:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.997351Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435850839286247:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:34.996205Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435852510533424:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:34.997504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435852510533443:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47920
2025-11-19T13:59:34.997469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435850839286244:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.997489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:34.997497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435850839286244:2111] [node 5] ICP22 created new session: [1:7574435850839286270:2111]
2025-11-19T13:59:34.997503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435850839286270:2111] [node 5] ICS09 handshake done sender: [1:7574435850839286247:2113] self: [1:7574435850839286245:0] peer: [5:7574435851759881752:0] socket: 541 qp: -1
2025-11-19T13:59:34.997509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435850839286270:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:34.997517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:34.994626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435849423714615:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:34.994644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435849423714654:2085] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.994654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:34.994659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435849423714654:2085] [node 5] ICP22 created new session: [6:7574435849423714707:2085]
2025-11-19T13:59:34.994665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435849423714707:2085] [node 5] ICS09 handshake done sender: [6:7574435849423714699:2117] self: [6:7574435849423714703:0] peer: [5:7574435851759881726:0] socket: 539 qp: -1
2025-11-19T13:59:34.994671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435849423714707:2085] [node 5] ICS10 traffic start
2025-11-19T13:59:34.994680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:34.998189Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435848839756238:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:34.999247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435852510533413:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:34.999270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:34.999278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435852510533413:2109] [node 4] ICP22 created new session: [3:7574435852510533444:2109]
2025-11-19T13:59:34.999283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435852510533444:2109] [node 4] ICS09 handshake done sender: [3:7574435852510533424:2114] self: [3:7574435852510533414:0] peer: [4:7574435848918607873:0] socket: 513 qp: -1
2025-11-19T13:59:34.999289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435852510533444:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:34.999297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:34.997997Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435848918607860:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:34.998011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435848918607860:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:34.998057Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435848918607866:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:34.998483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435848918607780:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:34.998496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:34.998501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435848918607780:2050] [node 3] ICP22 created new session: [4:7574435848918607875:2050]
2025-11-19T13:59:34.998509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435848918607875:2050] [node 3] ICS09 handshake done sender: [4:7574435848918607866:2119] self: [4:7574435848918607873:0] peer: [3:7574435852510533414:0] socket: 542 qp: -1
2025-11-19T13:59:34.998516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435848918607875:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:34.998523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:35.000085Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435848918607860:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:35.000180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435848918607851:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.000197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:35.000202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435848918607851:2112] [node 5] ICP22 created new session: [4:7574435853213575174:2112]
2025-11-19T13:59:35.000207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435853213575174:2112] [node 5] ICS09 handshake done sender: [4:7574435848918607860:2117] self: [4:7574435848918607852:0] peer: [5:7574435851759881753:0] socket: 546 qp: -1
2025-11-19T13:59:35.000211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435853213575174:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:35.000218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:35.000579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435848839756232:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.000597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:35.000602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435848839756232:2110] [node 6] ICP22 created new session: [2:7574435848839756268:2110]
2025-11-19T13:59:35.000606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435848839756268:2110] [node 6] ICS09 handshake done sender: [2:7574435848839756238:2113] self: [2:7574435848839756237:0] peer: [6:7574435849423714706:0] socket: 510 qp: -1
2025-11-19T13:59:35.000610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435848839756268:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:35.000615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:35.002168Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435852510533436:2120] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.002856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435849423714615:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30072) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435849423714704:2118] held: no
2025-11-19T13:59:35.002930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435853718682005:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7748
2025-11-19T13:59:35.004204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435852510533350:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30072) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435852510533436:2120] held: no
2025-11-19T13:59:35.004227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435852510533350:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.004236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:35.004242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435852510533350:2049] [node 2] ICP22 created new session: [3:7574435856805500743:2049]
2025-11-19T13:59:35.004247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435856805500743:2049] [node 2] ICS09 handshake done sender: [3:7574435852510533436:2120] self: [3:7574435852510533441:0] peer: [2:7574435848839756239:0] socket: 509 qp: -1
2025-11-19T13:59:35.004253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435856805500743:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:35.004261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:35.004788Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435849423714704:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.006119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435849423714615:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.006138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:35.006143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435849423714615:2049] [node 2] ICP22 created new session: [6:7574435853718682007:2049]
2025-11-19T13:59:35.006148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435853718682007:2049] [node 2] ICS09 handshake done sender: [6:7574435849423714704:2118] self: [6:7574435849423714706:0] peer: [2:7574435848839756237:0] socket: 512 qp: -1
2025-11-19T13:59:35.006153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435853718682007:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:35.006159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:35.006279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435853718682006:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7748
2025-11-19T13:59:35.006616Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435853718682006:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.006646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435849423714616:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.006748Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435852510533431:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.006757Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435852510533431:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.007563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435853718682010:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7748
2025-11-19T13:59:35.007663Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435852510533431:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:35.007978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435852510533429:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.007996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:35.008002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435852510533429:2113] [node 6] ICP22 created new session: [3:7574435856805500746:2113]
2025-11-19T13:59:35.008008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435856805500746:2113] [node 6] ICS09 handshake done sender: [3:7574435852510533431:2117] self: [3:7574435852510533430:0] peer: [6:7574435853718682009:0] socket: 508 qp: -1
2025-11-19T13:59:35.008013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435856805500746:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:35.008021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:35.008417Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435853718682006:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:35.008497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435849423714616:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.008510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:35.008515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435849423714616:2050] [node 3] ICP22 created new session: [6:7574435853718682011:2050]
2025-11-19T13:59:35.008519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435853718682011:2050] [node 3] ICS09 handshake done sender: [6:7574435853718682006:2121] self: [6:7574435853718682009:0] peer: [3:7574435852510533430:0] socket: 520 qp: -1
2025-11-19T13:59:35.008524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435853718682011:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:35.008529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:35.009571Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435851759881660:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7330) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435851759881746:2118] held: no
2025-11-19T13:59:35.009810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435851759881717:2099] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28163) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435851759881748:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.009869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435856054849050:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36944
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.009913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435856054849051:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36944
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.010058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435856054849052:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36944
2025-11-19T13:59:35.010169Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435851759881746:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:35.010220Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435851759881748:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:35.010269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435851759881660:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.010281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:35.010286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435851759881660:2048] [node 1] ICP22 created new session: [5:7574435856054849053:2048]
2025-11-19T13:59:35.010291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435856054849053:2048] [node 1] ICS09 handshake done sender: [5:7574435851759881746:2118] self: [5:7574435851759881752:0] peer: [1:7574435850839286245:0] socket: 545 qp: -1
2025-11-19T13:59:35.010299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435856054849053:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:35.010305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:35.010388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435851759881717:2099] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.010396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:35.010400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435851759881717:2099] [node 4] ICP22 created new session: [5:7574435856054849055:2099]
2025-11-19T13:59:35.010403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435856054849055:2099] [node 4] ICS09 handshake done sender: [5:7574435851759881748:2119] self: [5:7574435851759881753:0] peer: [4:7574435848918607852:0] socket: 547 qp: -1
2025-11-19T13:59:35.010407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435856054849055:2099] [node 4] ICS10 traffic start
2025-11-19T13:59:35.010411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:35.011238Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435856054849050:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.011269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435851759881662:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.011358Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435852510533427:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.011366Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435852510533427:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.012177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435856054849058:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36944
2025-11-19T13:59:35.012494Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435852510533427:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:35.012655Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435856054849050:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:35.012716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435851759881662:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.012729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:35.012734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435851759881662:2050] [node 3] ICP22 created new session: [5:7574435856054849059:2050]
2025-11-19T13:59:35.012739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435856054849059:2050] [node 3] ICS09 handshake done sender: [5:7574435856054849050:2120] self: [5:7574435856054849057:0] peer: [3:7574435852510533425:0] socket: 553 qp: -1
2025-11-19T13:59:35.012744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435856054849059:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:35.012750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:35.013617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435852510533422:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.013634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:35.013639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435852510533422:2112] [node 5] ICP22 created new session: [3:7574435856805500749:2112]
2025-11-19T13:59:35.013643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435856805500749:2112] [node 5] ICS09 handshake done sender: [3:7574435852510533427:2116] self: [3:7574435852510533425:0] peer: [5:7574435856054849057:0] socket: 549 qp: -1
2025-11-19T13:59:35.013647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435856805500749:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:35.013653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:35.032074Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:35.032170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435848839756256:2048] [node 1] ICS01 socket: 516 reason# EndOfStream
2025-11-19T13:59:35.032180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435848839756164:2048] [node 1] ICP30 unregister session Session# [2:7574435848839756256:2048] VirtualId# [2:7574435848839756246:0]
2025-11-19T13:59:35.032186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435848839756256:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.032231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:35.032254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:35.036949Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:35.037032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435852510533437:2048] [node 1] ICS01 socket: 498 reason# EndOfStream
2025-11-19T13:59:35.037040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435852510533349:2048] [node 1] ICP30 unregister session Session# [3:7574435852510533437:2048] VirtualId# [3:7574435852510533418:0]
2025-11-19T13:59:35.037048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435852510533437:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.037080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:35.037100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:35.042149Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:35.042255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435848918607869:2048] [node 1] ICS01 socket: 503 reason# EndOfStream
2025-11-19T13:59:35.042264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435848918607778:2048] [node 1] ICP30 unregister session Session# [4:7574435848918607869:2048] VirtualId# [4:7574435848918607857:0]
2025-11-19T13:59:35.042273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435848918607869:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.042343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:35.042372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:35.043261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:35.043358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435856054849053:2048] [node 1] ICS01 socket: 545 reason# EndOfStream
2025-11-19T13:59:35.043367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435851759881660:2048] [node 1] ICP30 unregister session Session# [5:7574435856054849053:2048] VirtualId# [5:7574435851759881752:0]
2025-11-19T13:59:35.043374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435856054849053:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.043429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:35.043457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:35.043492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:35.043554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435849423714695:2048] [node 1] ICS01 socket: 502 reason# EndOfStream
2025-11-19T13:59:35.043559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435849423714614:2048] [node 1] ICP30 unregister session Session# [6:7574435849423714695:2048] VirtualId# [6:7574435849423714663:0]
2025-11-19T13:59:35.043563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435849423714695:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.043590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:35.043620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:35.048466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:35.048578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435856805500743:2049] [node 2] ICS01 socket: 509 reason# EndOfStream
2025-11-19T13:59:35.048586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435852510533350:2049] [node 2] ICP30 unregister session Session# [3:7574435856805500743:2049] VirtualId# [3:7574435852510533441:0]
2025-11-19T13:59:35.048593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435856805500743:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.048624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:35.048642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:35.057215Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:35.057326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435848918607871:2049] [node 2] ICS01 socket: 522 reason# EndOfStream
2025-11-19T13:59:35.057335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435848918607779:2049] [node 2] ICP30 unregister session Session# [4:7574435848918607871:2049] VirtualId# [4:7574435848918607858:0]
2025-11-19T13:59:35.057343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435848918607871:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.057387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:35.057412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:35.057449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:35.057514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435853718682007:2049] [node 2] ICS01 socket: 512 reason# EndOfStream
2025-11-19T13:59:35.057519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435849423714615:2049] [node 2] ICP30 unregister session Session# [6:7574435853718682007:2049] VirtualId# [6:7574435849423714706:0]
2025-11-19T13:59:35.057524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435853718682007:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.057553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:35.057573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:35.057602Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:35.057673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435851759881743:2049] [node 2] ICS07 socket disconnect 518 reason# EndOfStream
2025-11-19T13:59:35.057677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435851759881743:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.057702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:35.057720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435851759881743:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:35.058278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435851759881661:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30072) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:35.058287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435851759881743:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:35.058291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435851759881661:2049] [node 2] ICP30 unregister session Session# [5:7574435851759881743:2049] VirtualId# [5:7574435851759881741:0]
2025-11-19T13:59:35.058306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:35.058313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435851759881661:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30072) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:35.058317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30072) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:35.067573Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:35.067665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435856054849059:2050] [node 3] ICS01 socket: 553 reason# EndOfStream
2025-11-19T13:59:35.067674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435851759881662:2050] [node 3] ICP30 unregister session Session# [5:7574435856054849059:2050] VirtualId# [5:7574435856054849057:0]
2025-11-19T13:59:35.067681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435856054849059:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.067741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:35.067781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:35.067820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:35.067885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435853718682011:2050] [node 3] ICS01 socket: 520 reason# EndOfStream
2025-11-19T13:59:35.067890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435849423714616:2050] [node 3] ICP30 unregister session Session# [6:7574435853718682011:2050] VirtualId# [6:7574435853718682009:0]
2025-11-19T13:59:35.067894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435853718682011:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.067920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:35.067941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:35.068897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435851759881661:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:35.087900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:35.088009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435849423714700:2082] [node 4] ICS07 socket disconnect 511 reason# EndOfStream
2025-11-19T13:59:35.088015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435849423714700:2082] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.088068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:35.088090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435849423714700:2082] [node 4] ICS15 start handshake
2025-11-19T13:59:35.093492Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435849423714653:2082] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28163) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:35.093514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435849423714700:2082] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:35.093519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435849423714653:2082] [node 4] ICP30 unregister session Session# [6:7574435849423714700:2082] VirtualId# [6:7574435849423714693:0]
2025-11-19T13:59:35.093538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:35.093545Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435849423714653:2082] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28163) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:35.093549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28163) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:35.103543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435849423714653:2082] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:35.216161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435853758653182:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:35.216176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435853758653183:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:35.275880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435853714693814:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:35.275901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435853714693815:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:35.323881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435856461393574:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:35.323899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435856461393575:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:35.351932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435853417225365:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:35.351946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435853417225366:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:35.351950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435853417225367:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:35.380751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435855178160604:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:35.380771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435855178160605:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:35.380776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435855178160606:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:35.403990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435855830424646:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:35.404010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435855830424647:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:35.404014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435855830424648:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:35.447718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435853758653243:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:35.448572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435853758653249:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.449453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435856461393634:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34146
2025-11-19T13:59:35.450735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435856461393637:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:35.450814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435856461393638:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:35.451793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435853417225428:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:35.450804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435853758653258:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.452337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435853417225429:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23859
2025-11-19T13:59:35.452007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435855178160665:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2551
2025-11-19T13:59:35.452085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435855830424688:2087] [node 4] ICP01 ready to work
2025-11-19T13:59:35.452491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435856461393643:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.453963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435853758653259:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47224
2025-11-19T13:59:35.453511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435853714693880:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:35.453593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435853714693881:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:35.453658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435853714693882:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.454993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435855178160666:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2551
2025-11-19T13:59:35.453581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435855830424695:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29048
2025-11-19T13:59:35.453599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435855830424696:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29048
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.471911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435853417225365:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(20589111ns) {Configure:104(16677708ns) {ProcessConfigured:135(16432617ns) {StartInitialHandshake:157(16431164ns) {DropHandshakes:469(323ns) {DropIncomingHandshake:441(35ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(139ns) PrepareNewSessionHandshake:476(2065ns) {SwitchToState:178(275ns)}}}}}
2025-11-19T13:59:35.474115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435853417225438:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23859
2025-11-19T13:59:35.474158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435853417225439:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23859
2025-11-19T13:59:35.476212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435855178160669:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.478869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435855830424648:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(25800449ns) {Configure:104(25799857ns) {ProcessConfigured:135(25561768ns) {StartInitialHandshake:157(25561057ns) {DropHandshakes:469(416ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(25ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:476(905ns) {SwitchToState:178(536ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.480277Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435856461393637:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(28856948ns) {Configure:104(23682117ns) {ProcessConfigured:135(23425124ns) {StartInitialHandshake:157(23424167ns) {DropHandshakes:469(378ns) {DropIncomingHandshake:441(25ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(1027ns) {SwitchToState:178(530ns)}}}}}
2025-11-19T13:59:35.480445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435856461393653:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34146
2025-11-19T13:59:35.480457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435856461393654:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34146
2025-11-19T13:59:35.480618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435853714693893:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51955
2025-11-19T13:59:35.480670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435853714693894:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51955
2025-11-19T13:59:35.482712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435855178160677:2113] [node 4] ICP01 ready to work
2025-11-19T13:59:35.482773Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435855178160665:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.484157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435855830424699:2085] [node 5] ICP01 ready to work
2025-11-19T13:59:35.485696Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435856461393634:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.485974Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435853714693893:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.486006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435853714693814:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.483930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435853417225434:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:35.485835Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435853417225438:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.488129Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435853758653259:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.491850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435855178160604:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(28009608ns) {Configure:104(28008668ns) {ProcessConfigured:135(27620348ns) {StartInitialHandshake:157(27619608ns) {DropHandshakes:469(843ns) {DropIncomingHandshake:441(187ns) DropOutgoingHandshake:456(29ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:476(848ns) {SwitchToState:178(341ns)}}}}}
2025-11-19T13:59:35.493099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435855178160604:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.493428Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435853417225439:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.493620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435853417225366:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.493467Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435856461393575:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(42613404ns) {Configure:104(42612401ns) {ProcessConfigured:135(42217117ns) {StartInitialHandshake:157(42215904ns) {DropHandshakes:469(924ns) {DropIncomingHandshake:441(311ns) DropOutgoingHandshake:456(154ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:476(966ns) {SwitchToState:178(266ns)}}}}}
2025-11-19T13:59:35.493498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435856461393575:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.493768Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435853714693894:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.493772Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435853758653248:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.493786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435853758653248:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.500150Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435855830424696:2093] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.500183Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435855830424695:2092] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.500501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435855830424648:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.500515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435855830424688:2087] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.500726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435855830424728:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29048
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.500055Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435853417225434:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16063930ns) {Configure:104(16063564ns) {ProcessConfigured:135(15765304ns) {StartInitialHandshake:157(15764741ns) {DropHandshakes:469(515ns) {DropIncomingHandshake:441(91ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:476(766ns) {SwitchToState:178(114ns)}}}}}
2025-11-19T13:59:35.501516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435853417225446:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23859
2025-11-19T13:59:35.502108Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435853714693875:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.502123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435853714693875:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:35.503838Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435853714693886:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.503847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435853714693886:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:35.505369Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435855830424728:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.505460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435855830424699:2085] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.503850Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435853758653251:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.503868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435853758653251:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:35.505619Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435855178160673:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.505631Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435855178160673:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:35.506142Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435856461393653:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.506174Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435856461393654:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.508408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435853758653263:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47224
2025-11-19T13:59:35.504223Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435853417225431:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.504233Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435853417225431:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:35.504648Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435853417225446:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.506084Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435853417225431:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:35.506274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435853417225428:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.506293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:35.506300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435853417225428:2109] [node 6] ICP22 created new session: [4:7574435853417225451:2109]
2025-11-19T13:59:35.506306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435853417225451:2109] [node 6] ICS09 handshake done sender: [4:7574435853417225431:2111] self: [4:7574435853417225430:0] peer: [6:7574435855830424727:0] socket: 506 qp: -1
2025-11-19T13:59:35.506314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435853417225451:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:35.506335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:35.509024Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435853758653248:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:35.509401Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435855178160666:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.509457Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435855178160673:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:35.511837Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435856461393642:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.511853Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435856461393642:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:35.511994Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435853417225429:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.512032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435853417225365:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.512354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435853758653182:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.512377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:35.512386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435853758653182:2048] [node 2] ICP22 created new session: [1:7574435853758653267:2048]
2025-11-19T13:59:35.512394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435853758653267:2048] [node 2] ICS09 handshake done sender: [1:7574435853758653248:2110] self: [1:7574435853758653247:0] peer: [2:7574435853714693895:0] socket: 500 qp: -1
2025-11-19T13:59:35.512400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435853758653267:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:35.512421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:35.512615Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435853758653245:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.512618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435853758653245:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:35.513123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435853758653265:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47224
2025-11-19T13:59:35.513172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435853758653266:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47224
2025-11-19T13:59:35.513248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435855830424688:2087] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13149) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435855830424695:2092] held: no
2025-11-19T13:59:35.515825Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435853758653266:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.515881Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435853758653245:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:35.516479Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435853714693875:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:35.517530Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435853714693886:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:35.517107Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435853758653263:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.517136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435853758653243:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.517171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:35.517179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435853758653243:2107] [node 4] ICP22 created new session: [1:7574435853758653272:2107]
2025-11-19T13:59:35.517185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435853758653272:2107] [node 4] ICS09 handshake done sender: [1:7574435853758653245:2108] self: [1:7574435853758653244:0] peer: [4:7574435853417225454:0] socket: 498 qp: -1
2025-11-19T13:59:35.517191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435853758653272:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:35.517212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.523852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435853714693815:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.523876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:35.523886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435853714693815:2049] [node 3] ICP22 created new session: [2:7574435853714693900:2049]
2025-11-19T13:59:35.523893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435853714693900:2049] [node 3] ICS09 handshake done sender: [2:7574435853714693875:2107] self: [2:7574435853714693874:0] peer: [3:7574435856461393656:0] socket: 499 qp: -1
2025-11-19T13:59:35.523900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435853714693900:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:35.523937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:35.524086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435853714693880:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.524099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:35.524118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435853714693880:2108] [node 4] ICP22 created new session: [2:7574435853714693902:2108]
2025-11-19T13:59:35.524124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435853714693902:2108] [node 4] ICS09 handshake done sender: [2:7574435853714693886:2112] self: [2:7574435853714693885:0] peer: [4:7574435853417225444:0] socket: 515 qp: -1
2025-11-19T13:59:35.524129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435853714693902:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:35.524134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:35.524475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435855830424730:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29048
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.524774Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435853758653265:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.526030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435853758653182:2048] [node 2] ICP18 handshake (actor [1:7574435853758653263:2116]) is too old
2025-11-19T13:59:35.527873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435855178160669:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.527893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:35.527903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435855178160669:2110] [node 6] ICP22 created new session: [5:7574435855178160685:2110]
2025-11-19T13:59:35.527910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435855178160685:2110] [node 6] ICS09 handshake done sender: [5:7574435855178160673:2112] self: [5:7574435855178160672:0] peer: [6:7574435855830424729:0] socket: 524 qp: -1
2025-11-19T13:59:35.527920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435855178160685:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:35.527930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:35.527992Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435855830424695:2092] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:35.528127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435855830424688:2087] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.528143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:35.528149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435855830424688:2087] [node 4] ICP22 created new session: [6:7574435855830424732:2087]
2025-11-19T13:59:35.528155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435855830424732:2087] [node 4] ICS09 handshake done sender: [6:7574435855830424695:2092] self: [6:7574435855830424727:0] peer: [4:7574435853417225430:0] socket: 511 qp: -1
2025-11-19T13:59:35.528161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435855830424732:2087] [node 4] ICS10 traffic start
2025-11-19T13:59:35.528169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.529303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435853714693899:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51955
2025-11-19T13:59:35.529618Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435853714693893:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:35.529698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435853714693814:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.529711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:35.529718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435853714693814:2048] [node 1] ICP22 created new session: [2:7574435853714693906:2048]
2025-11-19T13:59:35.529723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435853714693906:2048] [node 1] ICS09 handshake done sender: [2:7574435853714693893:2115] self: [2:7574435853714693895:0] peer: [1:7574435853758653247:0] socket: 503 qp: -1
2025-11-19T13:59:35.529728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435853714693906:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:35.529733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.532527Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435856461393642:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:35.532686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435856461393638:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.532704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:35.532709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435856461393638:2109] [node 6] ICP22 created new session: [3:7574435856461393660:2109]
2025-11-19T13:59:35.532715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435856461393660:2109] [node 6] ICS09 handshake done sender: [3:7574435856461393642:2111] self: [3:7574435856461393641:0] peer: [6:7574435855830424726:0] socket: 508 qp: -1
2025-11-19T13:59:35.532720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435856461393660:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:35.532739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.532389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435853417225366:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62410) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435853417225439:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.536017Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435853758653251:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:35.536114Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435855178160605:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(39216462ns) {Configure:104(39214417ns) {ProcessConfigured:135(38918591ns) {StartInitialHandshake:157(38917264ns) {DropHandshakes:469(713ns) {DropIncomingHandshake:441(91ns) DropOutgoingHandshake:456(205ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:476(735ns) {SwitchToState:178(410ns)}}}}}
2025-11-19T13:59:35.536208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435855178160606:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.536291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435855178160687:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2551
2025-11-19T13:59:35.539135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435853417225455:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23859
2025-11-19T13:59:35.539904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435853417225456:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23859
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.540290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435853758653249:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.540309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:35.540313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435853758653249:2109] [node 5] ICP22 created new session: [1:7574435853758653277:2109]
2025-11-19T13:59:35.540319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435853758653277:2109] [node 5] ICS09 handshake done sender: [1:7574435853758653251:2111] self: [1:7574435853758653250:0] peer: [5:7574435855178160682:0] socket: 501 qp: -1
2025-11-19T13:59:35.540324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435853758653277:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:35.540331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:35.540460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435853758653276:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47224
2025-11-19T13:59:35.539957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435855830424731:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29048
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.544617Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435853758653276:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.544654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435853758653249:2109] [node 5] ICP18 handshake (actor [1:7574435853758653276:2119]) is too old
2025-11-19T13:59:35.545188Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435853758653243:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(28015275ns) {HandleHandshakeStatus:329(28014081ns) {DropIncomingHandshake:441(183ns) DropOutgoingHandshake:456(264ns) SwitchToState:178(248ns) ProcessPendingSessionEvents:495(27700325ns) {ForwardSessionEventToSession:612(27688615ns) ForwardSessionEventToSession:612(1682ns) ForwardSessionEventToSession:612(870ns) ForwardSessionEventToSession:612(956ns) ForwardSessionEventToSession:612(553ns) ForwardSessionEventToSession:612(1333ns) ForwardSessionEventToSession:612(1088ns) ForwardSessionEventToSession:612(710ns) ForwardSessionEventToSession:612(870ns)}}}
2025-11-19T13:59:35.545219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435853758653243:2107] [node 4] ICP18 handshake (actor [1:7574435853758653266:2118]) is too old
2025-11-19T13:59:35.544009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435856461393663:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34146
2025-11-19T13:59:35.544078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435856461393664:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34146
2025-11-19T13:59:35.547303Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435856461393647:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.547314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435856461393647:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:35.548856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435856461393663:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.548892Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435856461393634:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:35.548643Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435855178160669:2110] [node 6] ICP35 event processing took too much time PendingConnection:278(20744709ns) {HandleHandshakeStatus:329(20744009ns) {DropIncomingHandshake:441(311ns) DropOutgoingHandshake:456(149ns) SwitchToState:178(401ns) ProcessPendingSessionEvents:495(8765030ns) {ForwardSessionEventToSession:612(8757821ns) ForwardSessionEventToSession:612(1972ns) ForwardSessionEventToSession:612(1140ns) ForwardSessionEventToSession:612(668ns) ForwardSessionEventToSession:612(582ns)}}}
2025-11-19T13:59:35.548685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435855178160604:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30904) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435855178160665:2108] held: no
2025-11-19T13:59:35.552250Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435853417225429:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:35.553576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435856461393575:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.553600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:35.553609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435856461393575:2049] [node 2] ICP22 created new session: [3:7574435856461393665:2049]
2025-11-19T13:59:35.553616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435856461393665:2049] [node 2] ICS09 handshake done sender: [3:7574435856461393634:2107] self: [3:7574435856461393656:0] peer: [2:7574435853714693874:0] socket: 502 qp: -1
2025-11-19T13:59:35.553623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435856461393665:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:35.553632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:35.553794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435856461393574:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.554173Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435853417225439:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.555917Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435855178160687:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.555957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435855178160677:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.556600Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435855830424648:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25221) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435855830424696:2093] held: no
2025-11-19T13:59:35.556653Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435855830424728:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:35.558035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435855830424699:2085] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.558045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:35.558050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435855830424699:2085] [node 5] ICP22 created new session: [6:7574435855830424735:2085]
2025-11-19T13:59:35.558056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435855830424735:2085] [node 5] ICS09 handshake done sender: [6:7574435855830424728:2117] self: [6:7574435855830424729:0] peer: [5:7574435855178160672:0] socket: 530 qp: -1
2025-11-19T13:59:35.558060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435855830424735:2085] [node 5] ICS10 traffic start
2025-11-19T13:59:35.558066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:35.558204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435855830424734:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29048
2025-11-19T13:59:35.558790Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435855830424734:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.558686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435853714693908:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51955
2025-11-19T13:59:35.554308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435853417225365:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.554321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:35.554326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435853417225365:2048] [node 1] ICP22 created new session: [4:7574435853417225458:2048]
2025-11-19T13:59:35.554331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435853417225458:2048] [node 1] ICS09 handshake done sender: [4:7574435853417225429:2110] self: [4:7574435853417225454:0] peer: [1:7574435853758653244:0] socket: 504 qp: -1
2025-11-19T13:59:35.554335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435853417225458:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:35.554341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:35.554450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435853417225366:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.554456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:35.554459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435853417225366:2049] [node 2] ICP22 created new session: [4:7574435853417225460:2049]
2025-11-19T13:59:35.554463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435853417225460:2049] [node 2] ICS09 handshake done sender: [4:7574435853417225439:2115] self: [4:7574435853417225444:0] peer: [2:7574435853714693885:0] socket: 520 qp: -1
2025-11-19T13:59:35.554467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435853417225460:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:35.554471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:35.554548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435853417225457:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23859
2025-11-19T13:59:35.559960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435855830424647:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.560147Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435856461393647:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:35.560097Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435853417225445:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.560104Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435853417225445:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:35.560265Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435853417225457:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.560748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435853417225367:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.561037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435856461393669:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34146
2025-11-19T13:59:35.563115Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435853417225445:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.561857Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435856461393652:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.561866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435856461393652:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:35.562364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435856461393643:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.562376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:35.562380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435856461393643:2110] [node 5] ICP22 created new session: [3:7574435856461393670:2110]
2025-11-19T13:59:35.562385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435856461393670:2110] [node 5] ICS09 handshake done sender: [3:7574435856461393647:2113] self: [3:7574435856461393646:0] peer: [5:7574435855178160689:0] socket: 510 qp: -1
2025-11-19T13:59:35.562389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435856461393670:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:35.562395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:35.562744Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435856461393652:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:35.562815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435856461393637:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.562823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:35.562829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435856461393637:2108] [node 4] ICP22 created new session: [3:7574435856461393673:2108]
2025-11-19T13:59:35.562834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435856461393673:2108] [node 4] ICS09 handshake done sender: [3:7574435856461393652:2114] self: [3:7574435856461393651:0] peer: [4:7574435853417225464:0] socket: 527 qp: -1
2025-11-19T13:59:35.562838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435856461393673:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:35.562843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.563902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435855830424740:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29048
2025-11-19T13:59:35.567057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435856461393669:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.567092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435856461393637:2108] [node 4] ICP18 handshake (actor [3:7574435856461393669:2120]) is too old
2025-11-19T13:59:35.567074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435853758653253:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.567090Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435853758653253:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:35.567226Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435853714693890:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.567238Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435853714693890:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:35.568099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435853417225434:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.568117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:35.568122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435853417225434:2108] [node 5] ICP22 created new session: [4:7574435853417225465:2108]
2025-11-19T13:59:35.568127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435853417225465:2108] [node 5] ICS09 handshake done sender: [4:7574435853417225445:2116] self: [4:7574435853417225442:0] peer: [5:7574435855178160691:0] socket: 535 qp: -1
2025-11-19T13:59:35.568133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435853417225465:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:35.568138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:35.567841Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435855830424696:2093] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.568506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435855830424648:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.568517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:35.568523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435855830424648:2050] [node 3] ICP22 created new session: [6:7574435855830424743:2050]
2025-11-19T13:59:35.568528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435855830424743:2050] [node 3] ICS09 handshake done sender: [6:7574435855830424696:2093] self: [6:7574435855830424726:0] peer: [3:7574435856461393641:0] socket: 512 qp: -1
2025-11-19T13:59:35.568532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435855830424743:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:35.568537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:35.568652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435855830424742:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29048
2025-11-19T13:59:35.568937Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435853714693890:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:35.569080Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435855830424734:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:35.569136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435855830424647:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.569147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:35.569151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435855830424647:2049] [node 2] ICP22 created new session: [6:7574435855830424745:2049]
2025-11-19T13:59:35.569155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435855830424745:2049] [node 2] ICS09 handshake done sender: [6:7574435855830424734:2120] self: [6:7574435855830424739:0] peer: [2:7574435853714693889:0] socket: 514 qp: -1
2025-11-19T13:59:35.569159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435855830424745:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:35.569163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:35.569337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435853714693882:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.569348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:35.569352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435853714693882:2110] [node 6] ICP22 created new session: [2:7574435853714693910:2110]
2025-11-19T13:59:35.569356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435853714693910:2110] [node 6] ICS09 handshake done sender: [2:7574435853714693890:2114] self: [2:7574435853714693889:0] peer: [6:7574435855830424739:0] socket: 548 qp: -1
2025-11-19T13:59:35.569360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435853714693910:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:35.569365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:35.573435Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435853714693882:2110] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.579894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435853714693912:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51955
2025-11-19T13:59:35.583771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435855178160693:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2551
2025-11-19T13:59:35.583782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435855830424741:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29048
2025-11-19T13:59:35.584293Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435855830424741:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.584337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435855830424646:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.585027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435853417225367:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25221) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435853417225457:2121] held: no
2025-11-19T13:59:35.584837Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435853758653261:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.584856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435853758653261:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.585838Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435855178160606:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25221) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435855178160666:2109] held: no
2025-11-19T13:59:35.585928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435855178160677:2113] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13149) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435855178160687:2116] held: no
2025-11-19T13:59:35.585960Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435855178160665:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.586394Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435853758653261:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:35.586538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435855178160694:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2551
2025-11-19T13:59:35.586590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435855178160695:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2551
2025-11-19T13:59:35.586940Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435853758653253:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:35.587171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435853758653183:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.587191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:35.587198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435853758653183:2049] [node 3] ICP22 created new session: [1:7574435853758653281:2049]
2025-11-19T13:59:35.587204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435853758653281:2049] [node 3] ICS09 handshake done sender: [1:7574435853758653253:2112] self: [1:7574435853758653252:0] peer: [3:7574435856461393667:0] socket: 540 qp: -1
2025-11-19T13:59:35.587210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435853758653281:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:35.587219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:35.586038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435853417225467:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23859
2025-11-19T13:59:35.585701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435855830424748:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29048
2025-11-19T13:59:35.587069Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435855830424741:2122] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:35.588939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435855830424646:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.588952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:35.588959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435855830424646:2048] [node 1] ICP22 created new session: [6:7574435855830424749:2048]
2025-11-19T13:59:35.588965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435855830424749:2048] [node 1] ICS09 handshake done sender: [6:7574435855830424741:2122] self: [6:7574435855830424747:0] peer: [1:7574435853758653260:0] socket: 526 qp: -1
2025-11-19T13:59:35.588972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435855830424749:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:35.588978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:35.589302Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435853714693912:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.588905Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435853417225457:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:35.590167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435853417225367:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.590182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:35.590189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435853417225367:2050] [node 3] ICP22 created new session: [4:7574435853417225468:2050]
2025-11-19T13:59:35.590194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435853417225468:2050] [node 3] ICS09 handshake done sender: [4:7574435853417225457:2121] self: [4:7574435853417225464:0] peer: [3:7574435856461393651:0] socket: 507 qp: -1
2025-11-19T13:59:35.590200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435853417225468:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:35.590206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:35.589840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435856461393675:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34146
2025-11-19T13:59:35.590598Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435856461393663:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:35.591134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435856461393574:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.591148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:35.591155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435856461393574:2048] [node 1] ICP22 created new session: [3:7574435856461393676:2048]
2025-11-19T13:59:35.591160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435856461393676:2048] [node 1] ICS09 handshake done sender: [3:7574435856461393663:2118] self: [3:7574435856461393667:0] peer: [1:7574435853758653252:0] socket: 545 qp: -1
2025-11-19T13:59:35.591166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435856461393676:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:35.591172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.595832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435853758653258:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.595859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:35.595868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435853758653258:2113] [node 6] ICP22 created new session: [1:7574435853758653283:2113]
2025-11-19T13:59:35.595878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435853758653283:2113] [node 6] ICS09 handshake done sender: [1:7574435853758653261:2115] self: [1:7574435853758653260:0] peer: [6:7574435855830424747:0] socket: 552 qp: -1
2025-11-19T13:59:35.595885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435853758653283:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:35.595892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:35.596143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435855178160696:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2551
2025-11-19T13:59:35.596283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435855178160604:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.596295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:35.596300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435855178160604:2048] [node 1] ICP22 created new session: [5:7574435855178160697:2048]
2025-11-19T13:59:35.596306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435855178160697:2048] [node 1] ICS09 handshake done sender: [5:7574435855178160665:2108] self: [5:7574435855178160682:0] peer: [1:7574435853758653250:0] socket: 505 qp: -1
2025-11-19T13:59:35.596312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435855178160697:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:35.596322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:35.596473Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435855178160694:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.596482Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435855178160666:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:35.597257Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435855178160687:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:35.597725Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435853714693888:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:35.597762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435853714693888:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:35.597480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435855178160605:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:35.597504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435855178160606:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.597513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:35.597518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435855178160606:2050] [node 3] ICP22 created new session: [5:7574435855178160700:2050]
2025-11-19T13:59:35.597522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435855178160700:2050] [node 3] ICS09 handshake done sender: [5:7574435855178160666:2109] self: [5:7574435855178160689:0] peer: [3:7574435856461393646:0] socket: 516 qp: -1
2025-11-19T13:59:35.597527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435855178160700:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:35.597533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:35.598802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435855178160677:2113] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.598813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:35.598818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435855178160677:2113] [node 4] ICP22 created new session: [5:7574435855178160703:2113]
2025-11-19T13:59:35.598824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435855178160703:2113] [node 4] ICS09 handshake done sender: [5:7574435855178160687:2116] self: [5:7574435855178160691:0] peer: [4:7574435853417225442:0] socket: 542 qp: -1
2025-11-19T13:59:35.598829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435855178160703:2113] [node 4] ICS10 traffic start
2025-11-19T13:59:35.598871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:35.598984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435855178160702:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2551
2025-11-19T13:59:35.599137Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435853714693888:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:35.599287Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435855178160694:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:35.599364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435855178160605:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:35.599377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:35.599381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435855178160605:2049] [node 2] ICP22 created new session: [5:7574435855178160705:2049]
2025-11-19T13:59:35.599385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435855178160705:2049] [node 2] ICS09 handshake done sender: [5:7574435855178160694:2119] self: [5:7574435855178160699:0] peer: [2:7574435853714693887:0] socket: 559 qp: -1
2025-11-19T13:59:35.599389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435855178160705:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:35.599394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:35.600083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435853714693881:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:35.600098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:35.600102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435853714693881:2109] [node 5] ICP22 created new session: [2:7574435853714693915:2109]
2025-11-19T13:59:35.600108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435853714693915:2109] [node 5] ICS09 handshake done sender: [2:7574435853714693888:2113] self: [2:7574435853714693887:0] peer: [5:7574435855178160699:0] socket: 528 qp: -1
2025-11-19T13:59:35.600111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435853714693915:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:35.600116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:35.656854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:35.656922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435856461393676:2048] [node 1] ICS01 socket: 545 reason# EndOfStream
2025-11-19T13:59:35.656928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435856461393574:2048] [node 1] ICP30 unregister session Session# [3:7574435856461393676:2048] VirtualId# [3:7574435856461393667:0]
2025-11-19T13:59:35.656933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435856461393676:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.656960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:35.656978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:35.656297Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:35.656407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435853714693906:2048] [node 1] ICS01 socket: 503 reason# EndOfStream
2025-11-19T13:59:35.656420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435853714693814:2048] [node 1] ICP30 unregister session Session# [2:7574435853714693906:2048] VirtualId# [2:7574435853714693895:0]
2025-11-19T13:59:35.656429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435853714693906:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.656472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:35.656495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:35.670562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:35.670663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435853417225458:2048] [node 1] ICS01 socket: 504 reason# EndOfStream
2025-11-19T13:59:35.670674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435853417225365:2048] [node 1] ICP30 unregister session Session# [4:7574435853417225458:2048] VirtualId# [4:7574435853417225454:0]
2025-11-19T13:59:35.670681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435853417225458:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.670730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:35.670756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:35.670977Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:35.671038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435855830424749:2048] [node 1] ICS01 socket: 526 reason# EndOfStream
2025-11-19T13:59:35.671043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435855830424646:2048] [node 1] ICP30 unregister session Session# [6:7574435855830424749:2048] VirtualId# [6:7574435855830424747:0]
2025-11-19T13:59:35.671047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435855830424749:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.671075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:35.671096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:35.670792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:35.670881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435855178160697:2048] [node 1] ICS01 socket: 505 reason# EndOfStream
2025-11-19T13:59:35.670886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435855178160604:2048] [node 1] ICP30 unregister session Session# [5:7574435855178160697:2048] VirtualId# [5:7574435855178160682:0]
2025-11-19T13:59:35.670891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435855178160697:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.670922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:35.670946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:35.680559Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:35.680653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435856461393665:2049] [node 2] ICS01 socket: 502 reason# EndOfStream
2025-11-19T13:59:35.680660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435856461393575:2049] [node 2] ICP30 unregister session Session# [3:7574435856461393665:2049] VirtualId# [3:7574435856461393656:0]
2025-11-19T13:59:35.680668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435856461393665:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.680703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:35.680724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:35.695954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:35.696057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435853417225460:2049] [node 2] ICS01 socket: 520 reason# EndOfStream
2025-11-19T13:59:35.696067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435853417225366:2049] [node 2] ICP30 unregister session Session# [4:7574435853417225460:2049] VirtualId# [4:7574435853417225444:0]
2025-11-19T13:59:35.696074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435853417225460:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.696116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:35.696143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:35.696326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:35.696389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435855830424745:2049] [node 2] ICS01 socket: 514 reason# EndOfStream
2025-11-19T13:59:35.696393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435855830424647:2049] [node 2] ICP30 unregister session Session# [6:7574435855830424745:2049] VirtualId# [6:7574435855830424739:0]
2025-11-19T13:59:35.696397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435855830424745:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.696423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:35.696443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:35.696176Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:35.696242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435855178160705:2049] [node 2] ICS01 socket: 559 reason# EndOfStream
2025-11-19T13:59:35.696246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435855178160605:2049] [node 2] ICP30 unregister session Session# [5:7574435855178160705:2049] VirtualId# [5:7574435855178160699:0]
2025-11-19T13:59:35.696251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435855178160705:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.696279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:35.696299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:35.706337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:35.706436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435855830424743:2050] [node 3] ICS01 socket: 512 reason# EndOfStream
2025-11-19T13:59:35.706444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435855830424648:2050] [node 3] ICP30 unregister session Session# [6:7574435855830424743:2050] VirtualId# [6:7574435855830424726:0]
2025-11-19T13:59:35.706450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435855830424743:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.706493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:35.706520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:35.706548Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:35.706610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435855178160700:2050] [node 3] ICS01 socket: 516 reason# EndOfStream
2025-11-19T13:59:35.706615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435855178160606:2050] [node 3] ICP30 unregister session Session# [5:7574435855178160700:2050] VirtualId# [5:7574435855178160689:0]
2025-11-19T13:59:35.706619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435855178160700:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.706643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:35.706663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:35.726208Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:35.726311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435855830424732:2087] [node 4] ICS01 socket: 511 reason# EndOfStream
2025-11-19T13:59:35.726322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435855830424688:2087] [node 4] ICP30 unregister session Session# [6:7574435855830424732:2087] VirtualId# [6:7574435855830424727:0]
2025-11-19T13:59:35.726330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435855830424732:2087] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:35.726385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:35.726411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:35.795889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435855157157476:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:35.795908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435855157157477:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:35.823886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435853193940720:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:35.823898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435853193940721:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:35.864698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435855537954907:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:35.864711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435855537954908:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:35.900289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435855733552264:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:35.900305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435855733552265:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:35.900311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435855733552266:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:35.947906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435855457476023:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:35.947926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435855457476024:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:35.947931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435855457476025:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:35.991870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435855958673389:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:35.991888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435855958673390:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:35.991893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435855958673391:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:36.076404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435859752443358:2085] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.077749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435857488908077:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:36.077843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435857488908079:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:36.077567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860028519621:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41752
2025-11-19T13:59:36.080139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435859452124840:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:36.084063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435860253640710:2068] [node 4] ICP01 ready to work
2025-11-19T13:59:36.089352Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435860028519621:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.089531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435860028519623:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:36.089904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435860028519623:2109] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.090401Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435859752443362:2087] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.090412Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435859752443362:2087] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.091163Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435859752443362:2087] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:36.091247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435859752443358:2085] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.091264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:36.091270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435859752443358:2085] [node 4] ICP22 created new session: [5:7574435859752443379:2085]
2025-11-19T13:59:36.091276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435859752443379:2085] [node 4] ICS09 handshake done sender: [5:7574435859752443362:2087] self: [5:7574435859752443361:0] peer: [4:7574435860028519624:0] socket: 506 qp: -1
2025-11-19T13:59:36.091284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435859752443379:2085] [node 4] ICS10 traffic start
2025-11-19T13:59:36.091292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.093654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435859452124842:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9488
2025-11-19T13:59:36.093678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435859452124843:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9488
2025-11-19T13:59:36.095292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435859832922264:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:36.095825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435859452124839:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:36.096541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435859452124837:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.102364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435859832922268:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50953
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.102437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435859832922269:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50953
2025-11-19T13:59:36.102474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435859832922270:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50953
2025-11-19T13:59:36.102485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435859832922271:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50953
2025-11-19T13:59:36.102904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435855958673390:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(18788358ns) {Configure:104(18787924ns) {ProcessConfigured:135(18498158ns) {StartInitialHandshake:157(18497707ns) {DropHandshakes:469(565ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(306ns) PrepareNewSessionHandshake:476(523ns) {SwitchToState:178(336ns)}}}}}
2025-11-19T13:59:36.103233Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435859832922269:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.104063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435857488908092:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:36.108673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435859832922279:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:36.108760Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435859832922268:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.109108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435855537954907:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.111387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435859752443393:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51473
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.112034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860253640758:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6193
2025-11-19T13:59:36.112193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435859832922264:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16832940ns) {Configure:104(16832216ns) {ProcessConfigured:135(16432337ns) {StartInitialHandshake:157(16431661ns) {DropHandshakes:469(482ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(88ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:476(698ns) {SwitchToState:178(487ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.112560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435859452124839:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16679410ns) {Configure:104(16678938ns) {ProcessConfigured:135(16391470ns) {StartInitialHandshake:157(16390908ns) {DropHandshakes:469(432ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(141ns) PrepareNewSessionHandshake:476(1106ns) {SwitchToState:178(630ns)}}}}}
2025-11-19T13:59:36.112575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860028519629:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41752
2025-11-19T13:59:36.112595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860028519630:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41752
2025-11-19T13:59:36.112609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860028519631:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41752
2025-11-19T13:59:36.114091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435859452124837:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17475965ns) {Configure:104(17475339ns) {ProcessConfigured:135(17228345ns) {StartInitialHandshake:157(17227729ns) {DropHandshakes:469(547ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(159ns) PrepareNewSessionHandshake:476(619ns) {SwitchToState:178(273ns)}}}}}
2025-11-19T13:59:36.114155Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859452124833:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.114159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435859452124833:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.113619Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435860028519621:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:36.113651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435855733552265:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17640553ns) {Configure:104(17639811ns) {ProcessConfigured:135(17370308ns) {StartInitialHandshake:157(17370067ns) {DropHandshakes:469(431ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(589ns) {SwitchToState:178(357ns)}}}}}
2025-11-19T13:59:36.115086Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435859452124833:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:36.115974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435859752443387:2091] [node 6] ICP01 ready to work
2025-11-19T13:59:36.116352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435860253640743:2065] [node 5] ICP01 ready to work
2025-11-19T13:59:36.125262Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435857488908092:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21135063ns) {Configure:104(21134479ns) {ProcessConfigured:135(20849698ns) {StartInitialHandshake:157(20849290ns) {DropHandshakes:469(424ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(47ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(1274ns) {SwitchToState:178(287ns)}}}}}
2025-11-19T13:59:36.125361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435855157157476:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(45175467ns) {Configure:104(45175050ns) {ProcessConfigured:135(44869280ns) {StartInitialHandshake:157(44869070ns) {DropHandshakes:469(490ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(136ns) PrepareNewSessionHandshake:476(1054ns) {SwitchToState:178(181ns)}}}}}
2025-11-19T13:59:36.125391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435855157157477:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.125407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:36.125414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435855157157477:2049] [node 3] ICP22 created new session: [1:7574435859452124854:2049]
2025-11-19T13:59:36.125421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435859452124854:2049] [node 3] ICS09 handshake done sender: [1:7574435859452124833:2107] self: [1:7574435859452124832:0] peer: [3:7574435859832922283:0] socket: 504 qp: -1
2025-11-19T13:59:36.125428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435859452124854:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:36.125450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.127341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435859832922286:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50953
2025-11-19T13:59:36.127367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435859832922287:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50953
2025-11-19T13:59:36.127543Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435859832922268:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:36.127618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435855537954907:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.127631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:36.127638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435855537954907:2048] [node 1] ICP22 created new session: [3:7574435859832922288:2048]
2025-11-19T13:59:36.127644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435859832922288:2048] [node 1] ICS09 handshake done sender: [3:7574435859832922268:2110] self: [3:7574435859832922283:0] peer: [1:7574435859452124832:0] socket: 512 qp: -1
2025-11-19T13:59:36.127650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435859832922288:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:36.127667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:36.127451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435859752443399:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51473
2025-11-19T13:59:36.127525Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435855733552264:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(30809383ns) {Configure:104(30808721ns) {ProcessConfigured:135(30562780ns) {StartInitialHandshake:157(30562308ns) {DropHandshakes:469(399ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:476(805ns) {SwitchToState:178(182ns)}}}}}
2025-11-19T13:59:36.127575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435860028519623:2109] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.127583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:36.127589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435860028519623:2109] [node 5] ICP22 created new session: [4:7574435860028519639:2109]
2025-11-19T13:59:36.127595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435860028519639:2109] [node 5] ICS09 handshake done sender: [4:7574435860028519621:2108] self: [4:7574435860028519624:0] peer: [5:7574435859752443361:0] socket: 507 qp: -1
2025-11-19T13:59:36.127599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435860028519639:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:36.127620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:36.127878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860028519638:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41752
2025-11-19T13:59:36.129636Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435860253640758:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.134088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435859832922278:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:36.134537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435860028519628:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:36.134607Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435860028519638:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.142282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435859752443387:2091] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26215560ns) {Configure:104(26214994ns) {ProcessConfigured:135(25907430ns) {StartInitialHandshake:157(25906306ns) {DropHandshakes:469(508ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(255ns) PrepareNewSessionHandshake:476(1585ns) {SwitchToState:178(865ns)}}}}}
2025-11-19T13:59:36.142572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435855457476024:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23286249ns) {Configure:104(23284832ns) {ProcessConfigured:135(23032647ns) {StartInitialHandshake:157(23032263ns) {DropHandshakes:469(365ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(24ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(1208ns) {SwitchToState:178(465ns)}}}}}
2025-11-19T13:59:36.143014Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435859832922270:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.143042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435855537954908:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.141470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435857488908095:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29951
2025-11-19T13:59:36.141529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435857488908096:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29951
2025-11-19T13:59:36.142872Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857488908095:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.143211Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857488908081:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.143214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435857488908081:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.143845Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435857488908081:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:36.150204Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435859752443393:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.150267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435855457476025:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.150504Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859452124849:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.150520Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435859452124849:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.151831Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859452124845:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.151849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435859452124845:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.153727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860028519644:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41752
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.155014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435860028519628:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20370906ns) {Configure:104(20370314ns) {ProcessConfigured:135(20102063ns) {StartInitialHandshake:157(20101596ns) {DropHandshakes:469(272ns) {DropIncomingHandshake:441(33ns) DropOutgoingHandshake:456(43ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:476(538ns) {SwitchToState:178(43ns)}}}}}
2025-11-19T13:59:36.155054Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435860028519644:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.155078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435855733552266:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.154703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435859832922278:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20525695ns) {Configure:104(20524633ns) {ProcessConfigured:135(20145007ns) {StartInitialHandshake:157(20143898ns) {DropHandshakes:469(798ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:476(863ns) {SwitchToState:178(326ns)}}}}}
2025-11-19T13:59:36.154753Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435859832922286:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.155348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435859832922295:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50953
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.156330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435859752443404:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51473
2025-11-19T13:59:36.156438Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435860028519630:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.156462Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435860028519629:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.156494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435855733552265:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.156619Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435859832922285:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.156626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435859832922285:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.156694Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435859832922270:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:36.157377Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435859832922285:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:36.157469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435855537954908:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.157490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:36.157497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435855537954908:2049] [node 2] ICP22 created new session: [3:7574435859832922297:2049]
2025-11-19T13:59:36.157503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435859832922297:2049] [node 2] ICS09 handshake done sender: [3:7574435859832922270:2112] self: [3:7574435859832922292:0] peer: [2:7574435857488908080:0] socket: 522 qp: -1
2025-11-19T13:59:36.157509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435859832922297:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:36.157517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:36.157637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435859832922264:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.157646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:36.157651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435859832922264:2107] [node 4] ICP22 created new session: [3:7574435859832922299:2107]
2025-11-19T13:59:36.157655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435859832922299:2107] [node 4] ICS09 handshake done sender: [3:7574435859832922285:2117] self: [3:7574435859832922265:0] peer: [4:7574435860028519646:0] socket: 543 qp: -1
2025-11-19T13:59:36.157658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435859832922299:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:36.157661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.158817Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435859752443404:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.158870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435855457476024:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.159008Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857488908087:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.159018Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435857488908087:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.159730Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435857488908087:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:36.154377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435859452124857:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9488
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.159911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435859752443406:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51473
2025-11-19T13:59:36.160080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435857488908079:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.160095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:36.160101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435857488908079:2108] [node 5] ICP22 created new session: [2:7574435857488908101:2108]
2025-11-19T13:59:36.160106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435857488908101:2108] [node 5] ICS09 handshake done sender: [2:7574435857488908087:2112] self: [2:7574435857488908086:0] peer: [5:7574435859752443405:0] socket: 549 qp: -1
2025-11-19T13:59:36.160112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435857488908101:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:36.160119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:36.160528Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435859752443404:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:36.160646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435855457476024:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.160661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:36.160668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435855457476024:2049] [node 2] ICP22 created new session: [5:7574435859752443407:2049]
2025-11-19T13:59:36.160674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435859752443407:2049] [node 2] ICS09 handshake done sender: [5:7574435859752443404:2117] self: [5:7574435859752443405:0] peer: [2:7574435857488908086:0] socket: 550 qp: -1
2025-11-19T13:59:36.160679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435859752443407:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:36.160686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:36.162508Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435859832922271:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.204199Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857488908096:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.204597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435853193940721:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.204642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:36.204656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435853193940721:2049] [node 3] ICP22 created new session: [2:7574435857488908105:2049]
2025-11-19T13:59:36.204664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435857488908105:2049] [node 3] ICS09 handshake done sender: [2:7574435857488908081:2109] self: [2:7574435857488908080:0] peer: [3:7574435859832922292:0] socket: 508 qp: -1
2025-11-19T13:59:36.204677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435857488908105:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:36.204704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:36.204856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857488908085:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.204864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435857488908085:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.205063Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435859832922281:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.205081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435859832922281:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.204964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435855733552264:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.205007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435855733552264:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(69519560ns) {IncomingHandshake:261(69519238ns) {DropIncomingHandshake:441(95ns) PrepareNewSessionHandshake:476(594ns) {SwitchToState:178(366ns)} GenerateSessionVirtualId:988(200ns) IssueIncomingHandshakeReply:193(69512285ns) {SwitchToState:178(576ns)}}}
2025-11-19T13:59:36.204917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435859452124858:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9488
2025-11-19T13:59:36.207788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435855958673389:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.207835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435855958673389:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(78135049ns) {IncomingHandshake:261(78134357ns) {DropIncomingHandshake:441(393ns) PrepareNewSessionHandshake:476(1081ns) {SwitchToState:178(391ns)} GenerateSessionVirtualId:988(184ns) IssueIncomingHandshakeReply:193(78126623ns) {SwitchToState:178(1487ns)}}}
2025-11-19T13:59:36.211031Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435855733552266:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2503) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435860028519644:2118] held: no
2025-11-19T13:59:36.209014Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859452124842:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.209042Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859452124843:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.209070Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859452124858:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.212593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860253640764:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6193
2025-11-19T13:59:36.212637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860253640765:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6193
2025-11-19T13:59:36.212667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860253640766:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6193
2025-11-19T13:59:36.212698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860253640767:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6193
2025-11-19T13:59:36.212780Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435860253640743:2065] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(96358148ns) {Configure:104(96357392ns) {ProcessConfigured:135(96078570ns) {StartInitialHandshake:157(96077802ns) {DropHandshakes:469(162ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(139ns) PrepareNewSessionHandshake:476(678ns) {SwitchToState:178(105ns)}}}}}
2025-11-19T13:59:36.213240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435859452124859:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9488
2025-11-19T13:59:36.214781Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435860253640765:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.214804Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435860253640766:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.214873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435855958673390:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.214884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435855958673391:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.215202Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435859832922281:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.215280Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435859832922293:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.215285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435859832922293:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.216187Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435859832922293:2120] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:36.216266Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435859452124849:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:36.216325Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435859452124845:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:36.216736Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435860253640764:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.216400Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859452124859:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.216415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435859452124839:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.216428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:36.216436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435859452124839:2110] [node 4] ICP22 created new session: [1:7574435859452124865:2110]
2025-11-19T13:59:36.216444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435859452124865:2110] [node 4] ICS09 handshake done sender: [1:7574435859452124849:2114] self: [1:7574435859452124847:0] peer: [4:7574435860028519643:0] socket: 537 qp: -1
2025-11-19T13:59:36.216451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435859452124865:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:36.216462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:36.216597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435859452124839:2110] [node 4] ICP18 handshake (actor [1:7574435859452124859:2119]) is too old
2025-11-19T13:59:36.216993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435859452124840:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.217009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:36.217013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435859452124840:2109] [node 6] ICP22 created new session: [1:7574435859452124868:2109]
2025-11-19T13:59:36.217019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435859452124868:2109] [node 6] ICS09 handshake done sender: [1:7574435859452124845:2113] self: [1:7574435859452124844:0] peer: [6:7574435860253640760:0] socket: 519 qp: -1
2025-11-19T13:59:36.217023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435859452124868:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:36.217029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.217304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435860253640743:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.217558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435855733552264:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4133) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435860028519638:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.215785Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435859752443399:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.215983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435855457476023:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.216004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435859752443409:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51473
2025-11-19T13:59:36.217501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435859832922279:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.217522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:36.217530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435859832922279:2108] [node 5] ICP22 created new session: [3:7574435859832922305:2108]
2025-11-19T13:59:36.217535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435859832922305:2108] [node 5] ICS09 handshake done sender: [3:7574435859832922281:2116] self: [3:7574435859832922280:0] peer: [5:7574435859752443403:0] socket: 528 qp: -1
2025-11-19T13:59:36.217541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435859832922305:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:36.217547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:36.217559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435859832922278:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.217571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:36.217577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435859832922278:2109] [node 6] ICP22 created new session: [3:7574435859832922307:2109]
2025-11-19T13:59:36.217582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435859832922307:2109] [node 6] ICS09 handshake done sender: [3:7574435859832922293:2120] self: [3:7574435859832922291:0] peer: [6:7574435860253640770:0] socket: 524 qp: -1
2025-11-19T13:59:36.217586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435859832922307:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:36.217590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:36.217835Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435859752443393:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:36.217914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435855457476025:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.217924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:36.217929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435855457476025:2050] [node 3] ICP22 created new session: [5:7574435859752443411:2050]
2025-11-19T13:59:36.217934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435859752443411:2050] [node 3] ICS09 handshake done sender: [5:7574435859752443393:2112] self: [5:7574435859752443403:0] peer: [3:7574435859832922280:0] socket: 529 qp: -1
2025-11-19T13:59:36.217938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435859752443411:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:36.217942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:36.217997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860253640771:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6193
2025-11-19T13:59:36.218558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435855958673389:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4133) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435860253640758:2111] held: no
2025-11-19T13:59:36.218593Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435860253640758:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:36.218675Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435860253640766:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:36.218742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435855958673389:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.218752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:36.218758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435855958673389:2048] [node 1] ICP22 created new session: [6:7574435860253640774:2048]
2025-11-19T13:59:36.218762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435860253640774:2048] [node 1] ICS09 handshake done sender: [6:7574435860253640758:2111] self: [6:7574435860253640760:0] peer: [1:7574435859452124844:0] socket: 520 qp: -1
2025-11-19T13:59:36.218767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435860253640774:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:36.218781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:36.218870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435855958673391:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.218886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:36.218891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435855958673391:2050] [node 3] ICP22 created new session: [6:7574435860253640776:2050]
2025-11-19T13:59:36.218895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435860253640776:2050] [node 3] ICS09 handshake done sender: [6:7574435860253640766:2117] self: [6:7574435860253640770:0] peer: [3:7574435859832922291:0] socket: 559 qp: -1
2025-11-19T13:59:36.218899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435860253640776:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:36.218903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:36.219059Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859452124850:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.219067Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435859452124850:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.214130Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435857488908085:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:36.218089Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435853193940721:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(74146706ns) {HandleHandshakeStatus:329(74145656ns) {DropIncomingHandshake:441(11936ns) DropOutgoingHandshake:456(424ns) SwitchToState:178(292ns) ProcessPendingSessionEvents:495(4848461ns) {ForwardSessionEventToSession:612(4839226ns) ForwardSessionEventToSession:612(1542ns) ForwardSessionEventToSession:612(677ns) ForwardSessionEventToSession:612(503ns) ForwardSessionEventToSession:612(670ns) ForwardSessionEventToSession:612(606ns) ForwardSessionEventToSession:612(522ns) ForwardSessionEventToSession:612(592ns) ForwardSessionEventToSession:612(523ns) ForwardSessionEventToSession:612(748ns)}}}
2025-11-19T13:59:36.219781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860253640772:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6193
2025-11-19T13:59:36.219825Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435859752443400:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.219836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435859752443400:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.220213Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435859452124850:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:36.220112Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435860253640772:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.220140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435860253640710:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.220280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435859452124837:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.220291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:36.220297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435859452124837:2108] [node 5] ICP22 created new session: [1:7574435859452124871:2108]
2025-11-19T13:59:36.220302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435859452124871:2108] [node 5] ICS09 handshake done sender: [1:7574435859452124850:2115] self: [1:7574435859452124848:0] peer: [5:7574435859752443410:0] socket: 538 qp: -1
2025-11-19T13:59:36.220305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435859452124871:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:36.220310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:36.223860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435857488908077:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.223887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:36.223895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435857488908077:2107] [node 4] ICP22 created new session: [2:7574435857488908108:2107]
2025-11-19T13:59:36.223901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435857488908108:2107] [node 4] ICS09 handshake done sender: [2:7574435857488908085:2111] self: [2:7574435857488908084:0] peer: [4:7574435860028519648:0] socket: 510 qp: -1
2025-11-19T13:59:36.223907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435857488908108:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:36.223913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.228945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435859752443413:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51473
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.229751Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435859752443399:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:36.227955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860028519650:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41752
2025-11-19T13:59:36.228720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860028519651:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41752
2025-11-19T13:59:36.229390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860028519652:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41752
2025-11-19T13:59:36.229513Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435860028519638:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:36.229622Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435860028519629:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:36.230889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435855457476023:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.230905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:36.230910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435855457476023:2048] [node 1] ICP22 created new session: [5:7574435859752443414:2048]
2025-11-19T13:59:36.230915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435859752443414:2048] [node 1] ICS09 handshake done sender: [5:7574435859752443399:2114] self: [5:7574435859752443410:0] peer: [1:7574435859452124848:0] socket: 540 qp: -1
2025-11-19T13:59:36.230920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435859752443414:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:36.230926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:36.229814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435855733552264:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.229825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:36.229831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435855733552264:2048] [node 1] ICP22 created new session: [4:7574435860028519653:2048]
2025-11-19T13:59:36.229838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435860028519653:2048] [node 1] ICS09 handshake done sender: [4:7574435860028519638:2117] self: [4:7574435860028519643:0] peer: [1:7574435859452124847:0] socket: 541 qp: -1
2025-11-19T13:59:36.229844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435860028519653:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:36.229851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:36.229937Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435860028519644:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:36.230020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435855733552265:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.230028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:36.230032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435855733552265:2049] [node 2] ICP22 created new session: [4:7574435860028519655:2049]
2025-11-19T13:59:36.230036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435860028519655:2049] [node 2] ICS09 handshake done sender: [4:7574435860028519629:2111] self: [4:7574435860028519648:0] peer: [2:7574435857488908084:0] socket: 511 qp: -1
2025-11-19T13:59:36.230041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435860028519655:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:36.230045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:36.231968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435855733552266:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.231980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:36.231986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435855733552266:2050] [node 3] ICP22 created new session: [4:7574435860028519657:2050]
2025-11-19T13:59:36.231991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435860028519657:2050] [node 3] ICS09 handshake done sender: [4:7574435860028519644:2118] self: [4:7574435860028519646:0] peer: [3:7574435859832922265:0] socket: 525 qp: -1
2025-11-19T13:59:36.231996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435860028519657:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:36.232000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:36.233089Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435859752443400:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:36.233180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435859752443387:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.233192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:36.233197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435859752443387:2091] [node 6] ICP22 created new session: [5:7574435859752443417:2091]
2025-11-19T13:59:36.233203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435859752443417:2091] [node 6] ICS09 handshake done sender: [5:7574435859752443400:2115] self: [5:7574435859752443397:0] peer: [6:7574435860253640773:0] socket: 545 qp: -1
2025-11-19T13:59:36.233208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435859752443417:2091] [node 6] ICS10 traffic start
2025-11-19T13:59:36.233212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:36.236248Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857488908094:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.236265Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435857488908094:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.236302Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435860028519645:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.236311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435860028519645:2119] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.237155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860253640780:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6193
2025-11-19T13:59:36.236943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435857488908110:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29951
2025-11-19T13:59:36.236986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435857488908111:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29951
2025-11-19T13:59:36.237419Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435860253640764:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:36.237505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435860253640743:2065] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.237519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:36.237524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435860253640743:2065] [node 5] ICP22 created new session: [6:7574435860253640781:2065]
2025-11-19T13:59:36.237530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435860253640781:2065] [node 5] ICS09 handshake done sender: [6:7574435860253640764:2115] self: [6:7574435860253640773:0] peer: [5:7574435859752443397:0] socket: 546 qp: -1
2025-11-19T13:59:36.237536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435860253640781:2065] [node 5] ICS10 traffic start
2025-11-19T13:59:36.237542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:36.238058Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857488908110:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.238082Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857488908111:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.238111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435853193940720:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.238225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859452124853:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.238236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435859452124853:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.239513Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435859452124853:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:36.239605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435855157157476:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.239624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:36.239631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435855157157476:2048] [node 2] ICP22 created new session: [1:7574435859452124874:2048]
2025-11-19T13:59:36.239637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435859452124874:2048] [node 2] ICS09 handshake done sender: [1:7574435859452124853:2116] self: [1:7574435859452124841:0] peer: [2:7574435857488908112:0] socket: 566 qp: -1
2025-11-19T13:59:36.239642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435859452124874:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:36.239649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:36.240908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435859452124837:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(20576569ns) {HandleHandshakeStatus:329(20575988ns) {DropIncomingHandshake:441(2117ns) DropOutgoingHandshake:456(86ns) SwitchToState:178(78ns) ProcessPendingSessionEvents:495(20521045ns) {ForwardSessionEventToSession:612(20510639ns) ForwardSessionEventToSession:612(1530ns) ForwardSessionEventToSession:612(929ns) ForwardSessionEventToSession:612(956ns) ForwardSessionEventToSession:612(566ns) ForwardSessionEventToSession:612(1304ns) ForwardSessionEventToSession:612(658ns) ForwardSessionEventToSession:612(1034ns) ForwardSessionEventToSession:612(631ns) ForwardSessionEventToSession:612(538ns)}}}
2025-11-19T13:59:36.241512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435857488908077:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(17624088ns) {HandleHandshakeStatus:329(17623302ns) {DropIncomingHandshake:441(5743ns) DropOutgoingHandshake:456(86ns) SwitchToState:178(68ns) ProcessPendingSessionEvents:495(17532278ns) {ForwardSessionEventToSession:612(17519618ns) ForwardSessionEventToSession:612(1461ns) ForwardSessionEventToSession:612(645ns) ForwardSessionEventToSession:612(1388ns) ForwardSessionEventToSession:612(944ns) ForwardSessionEventToSession:612(1112ns) ForwardSessionEventToSession:612(691ns) ForwardSessionEventToSession:612(640ns) ForwardSessionEventToSession:612(522ns) ForwardSessionEventToSession:612(794ns)}}}
2025-11-19T13:59:36.241538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435853193940720:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4133) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435857488908110:2117] held: no
2025-11-19T13:59:36.246085Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435857488908094:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.246386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860253640783:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6193
2025-11-19T13:59:36.246436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860253640784:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6193
2025-11-19T13:59:36.246420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435857488908115:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29951
2025-11-19T13:59:36.246577Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435860028519645:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:36.246764Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435860253640772:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:36.246955Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435860253640765:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:36.247301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435860253640710:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.247315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:36.247320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435860253640710:2068] [node 4] ICP22 created new session: [6:7574435860253640785:2068]
2025-11-19T13:59:36.247326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435860253640785:2068] [node 4] ICS09 handshake done sender: [6:7574435860253640772:2121] self: [6:7574435860253640778:0] peer: [4:7574435860028519642:0] socket: 565 qp: -1
2025-11-19T13:59:36.247331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435860253640785:2068] [node 4] ICS10 traffic start
2025-11-19T13:59:36.247336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:36.247082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435857488908092:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.247088Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435857488908110:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:36.247109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:36.247116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435857488908092:2113] [node 6] ICP22 created new session: [2:7574435857488908116:2113]
2025-11-19T13:59:36.247122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435857488908116:2113] [node 6] ICS09 handshake done sender: [2:7574435857488908094:2114] self: [2:7574435857488908093:0] peer: [6:7574435860253640769:0] socket: 556 qp: -1
2025-11-19T13:59:36.247130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435857488908116:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:36.247138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:36.247275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435853193940720:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.247288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:36.247294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435853193940720:2048] [node 1] ICP22 created new session: [2:7574435857488908118:2048]
2025-11-19T13:59:36.247298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435857488908118:2048] [node 1] ICS09 handshake done sender: [2:7574435857488908110:2117] self: [2:7574435857488908112:0] peer: [1:7574435859452124841:0] socket: 552 qp: -1
2025-11-19T13:59:36.247303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435857488908118:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:36.247308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:36.248530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435855958673390:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.248551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:36.248557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435855958673390:2049] [node 2] ICP22 created new session: [6:7574435860253640787:2049]
2025-11-19T13:59:36.248562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435860253640787:2049] [node 2] ICS09 handshake done sender: [6:7574435860253640765:2116] self: [6:7574435860253640769:0] peer: [2:7574435857488908093:0] socket: 558 qp: -1
2025-11-19T13:59:36.248569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435860253640787:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:36.248592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:36.250314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435860028519628:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.250336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:36.250342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435860028519628:2110] [node 6] ICP22 created new session: [4:7574435860028519660:2110]
2025-11-19T13:59:36.250346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435860028519660:2110] [node 6] ICS09 handshake done sender: [4:7574435860028519645:2119] self: [4:7574435860028519642:0] peer: [6:7574435860253640778:0] socket: 533 qp: -1
2025-11-19T13:59:36.250351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435860028519660:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:36.250367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:36.260012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:36.260111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435857488908118:2048] [node 1] ICS01 socket: 552 reason# EndOfStream
2025-11-19T13:59:36.260123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435853193940720:2048] [node 1] ICP30 unregister session Session# [2:7574435857488908118:2048] VirtualId# [2:7574435857488908112:0]
2025-11-19T13:59:36.260130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435857488908118:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:36.260170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:36.260191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:36.260561Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:36.260632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435859832922288:2048] [node 1] ICS01 socket: 512 reason# EndOfStream
2025-11-19T13:59:36.260637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435855537954907:2048] [node 1] ICP30 unregister session Session# [3:7574435859832922288:2048] VirtualId# [3:7574435859832922283:0]
2025-11-19T13:59:36.260642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435859832922288:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:36.260668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:36.260688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:36.267861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:36.267936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435860253640774:2048] [node 1] ICS01 socket: 520 reason# EndOfStream
2025-11-19T13:59:36.267944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435855958673389:2048] [node 1] ICP30 unregister session Session# [6:7574435860253640774:2048] VirtualId# [6:7574435860253640760:0]
2025-11-19T13:59:36.267950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435860253640774:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:36.267986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:36.268009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:36.268037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:36.268091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435860028519653:2048] [node 1] ICS01 socket: 541 reason# EndOfStream
2025-11-19T13:59:36.268095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435855733552264:2048] [node 1] ICP30 unregister session Session# [4:7574435860028519653:2048] VirtualId# [4:7574435860028519643:0]
2025-11-19T13:59:36.268099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435860028519653:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:36.268130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:36.268149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:36.271385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:36.271473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435859752443414:2048] [node 1] ICS01 socket: 540 reason# EndOfStream
2025-11-19T13:59:36.271480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435855457476023:2048] [node 1] ICP30 unregister session Session# [5:7574435859752443414:2048] VirtualId# [5:7574435859752443410:0]
2025-11-19T13:59:36.271486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435859752443414:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:36.271528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:36.271549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:36.272451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:36.272519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435859832922297:2049] [node 2] ICS01 socket: 522 reason# EndOfStream
2025-11-19T13:59:36.272526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435855537954908:2049] [node 2] ICP30 unregister session Session# [3:7574435859832922297:2049] VirtualId# [3:7574435859832922292:0]
2025-11-19T13:59:36.272530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435859832922297:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:36.272556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:36.272572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:36.277740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:36.277818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435860253640787:2049] [node 2] ICS01 socket: 558 reason# EndOfStream
2025-11-19T13:59:36.277826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435855958673390:2049] [node 2] ICP30 unregister session Session# [6:7574435860253640787:2049] VirtualId# [6:7574435860253640769:0]
2025-11-19T13:59:36.277831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435860253640787:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:36.277868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:36.277887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:36.279231Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:36.279306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435859752443407:2049] [node 2] ICS01 socket: 550 reason# EndOfStream
2025-11-19T13:59:36.279313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435855457476024:2049] [node 2] ICP30 unregister session Session# [5:7574435859752443407:2049] VirtualId# [5:7574435859752443405:0]
2025-11-19T13:59:36.279319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435859752443407:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:36.279361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:36.279381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:36.286720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:36.286809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435860028519655:2049] [node 2] ICS01 socket: 511 reason# EndOfStream
2025-11-19T13:59:36.286817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435855733552265:2049] [node 2] ICP30 unregister session Session# [4:7574435860028519655:2049] VirtualId# [4:7574435860028519648:0]
2025-11-19T13:59:36.286824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435860028519655:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:36.286875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:36.286900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:36.309842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T13:59:36.309955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435860028519657:2050] [node 3] ICS01 socket: 525 reason# EndOfStream
2025-11-19T13:59:36.309967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435855733552266:2050] [node 3] ICP30 unregister session Session# [4:7574435860028519657:2050] VirtualId# [4:7574435860028519646:0]
2025-11-19T13:59:36.309975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435860028519657:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:36.310029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:36.310055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:36.310103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:36.310165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435859752443411:2050] [node 3] ICS01 socket: 529 reason# EndOfStream
2025-11-19T13:59:36.310170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435855457476025:2050] [node 3] ICP30 unregister session Session# [5:7574435859752443411:2050] VirtualId# [5:7574435859752443403:0]
2025-11-19T13:59:36.310176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435859752443411:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:36.310207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:36.310230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:36.314840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:36.314943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435860253640776:2050] [node 3] ICS01 socket: 559 reason# EndOfStream
2025-11-19T13:59:36.314953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435855958673391:2050] [node 3] ICP30 unregister session Session# [6:7574435860253640776:2050] VirtualId# [6:7574435860253640770:0]
2025-11-19T13:59:36.314960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435860253640776:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:36.315010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:36.315033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:36.356927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:36.357037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435860253640785:2068] [node 4] ICS01 socket: 565 reason# EndOfStream
2025-11-19T13:59:36.357047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435860253640710:2068] [node 4] ICP30 unregister session Session# [6:7574435860253640785:2068] VirtualId# [6:7574435860253640778:0]
2025-11-19T13:59:36.357055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435860253640785:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:36.357092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:36.357118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:36.672145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435859477549377:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:36.672160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435859477549378:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:36.696270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435857931455967:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:36.696292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435857931455968:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:36.708173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435858323879701:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:36.708188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435858323879702:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:36.740274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435860842577301:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:36.740289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435860842577302:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:36.740293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435860842577303:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:36.768107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435860109670716:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:36.768131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435860109670717:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:36.768136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435860109670718:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:36.791602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435860854864268:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:36.791629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435860854864269:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:36.791634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435860854864270:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:36.831328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435857931456028:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:36.831117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435858323879739:2084] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.831965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435857931456029:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26418
2025-11-19T13:59:36.831984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435860842577326:2068] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.832204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860842577328:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57075
2025-11-19T13:59:36.832308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435859477549438:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:36.833762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435859477549444:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:36.832655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435858323879749:2089] [node 5] ICP01 ready to work
2025-11-19T13:59:36.833869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435860109670748:2070] [node 4] ICP01 ready to work
2025-11-19T13:59:36.833973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435860109670749:2074] [node 6] ICP01 ready to work
2025-11-19T13:59:36.834403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435860842577337:2075] [node 6] ICP01 ready to work
2025-11-19T13:59:36.840570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435860854864288:2063] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.842518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435860109670757:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31823
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.855855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435857931456028:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24457772ns) {Configure:104(24457127ns) {ProcessConfigured:135(22510842ns) {StartInitialHandshake:157(22510269ns) {DropHandshakes:469(304ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(15ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:476(1509ns) {SwitchToState:178(1009ns)}}}}}
2025-11-19T13:59:36.856076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435857931456038:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26418
2025-11-19T13:59:36.856087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435857931456039:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26418
2025-11-19T13:59:36.857662Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857931456029:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.860408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435858323879702:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(26011534ns) {Configure:104(26010493ns) {ProcessConfigured:135(25611527ns) {StartInitialHandshake:157(25611027ns) {DropHandshakes:469(505ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(740ns) {SwitchToState:178(241ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.862422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435859477549448:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54106
2025-11-19T13:59:36.862627Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435860842577337:2075] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(28166283ns) {Configure:104(28165882ns) {ProcessConfigured:135(25365664ns) {StartInitialHandshake:157(25364957ns) {DropHandshakes:469(319ns) {DropIncomingHandshake:441(14ns) DropOutgoingHandshake:456(28ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:476(900ns) {SwitchToState:178(105ns)}}}}}
2025-11-19T13:59:36.862794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860842577372:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57075
2025-11-19T13:59:36.863051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435860842577326:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(30915438ns) {Configure:104(30914693ns) {ProcessConfigured:135(25647524ns) {StartInitialHandshake:157(25646654ns) {DropHandshakes:469(441ns) {DropIncomingHandshake:441(22ns) DropOutgoingHandshake:456(263ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:476(486ns) {SwitchToState:178(48ns)}}}}}
2025-11-19T13:59:36.860175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435860109670749:2074] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25734355ns) {Configure:104(25732979ns) {ProcessConfigured:135(25503381ns) {StartInitialHandshake:157(25502981ns) {DropHandshakes:469(251ns) {DropIncomingHandshake:441(45ns) DropOutgoingHandshake:456(52ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(1010ns) {SwitchToState:178(725ns)}}}}}
2025-11-19T13:59:36.859271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435857931455968:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(25921092ns) {Configure:104(25919404ns) {ProcessConfigured:135(25667804ns) {StartInitialHandshake:157(25667301ns) {DropHandshakes:469(395ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(1366ns) PrepareNewSessionHandshake:476(1058ns) {SwitchToState:178(507ns)}}}}}
2025-11-19T13:59:36.864180Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857931456039:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.864187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435857931456034:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:36.864722Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435860842577372:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.864757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435860842577302:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.869609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860854864342:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16440
2025-11-19T13:59:36.869633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860854864343:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16440
2025-11-19T13:59:36.869694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860854864344:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16440
2025-11-19T13:59:36.870540Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435860854864343:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.870573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435860854864288:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.867780Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435860842577328:2073] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.872365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435858323879777:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24703
2025-11-19T13:59:36.872422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435858323879778:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24703
2025-11-19T13:59:36.876216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435858323879749:2089] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(43512118ns) {Configure:104(43510571ns) {ProcessConfigured:135(43212995ns) {StartInitialHandshake:157(43212500ns) {DropHandshakes:469(570ns) {DropIncomingHandshake:441(144ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(528ns) {SwitchToState:178(361ns)}}}}}
2025-11-19T13:59:36.875953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435860854864268:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(29221598ns) {Configure:104(29220251ns) {ProcessConfigured:135(28895671ns) {StartInitialHandshake:157(28894828ns) {DropHandshakes:469(642ns) {DropIncomingHandshake:441(124ns) DropOutgoingHandshake:456(127ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:476(1919ns) {SwitchToState:178(1125ns)}}}}}
2025-11-19T13:59:36.876418Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435858323879777:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.876599Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435860854864344:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.876954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435860109670716:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25278888ns) {Configure:104(25277241ns) {ProcessConfigured:135(24904697ns) {StartInitialHandshake:157(24904042ns) {DropHandshakes:469(1023ns) {DropIncomingHandshake:441(255ns) DropOutgoingHandshake:456(160ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:476(1248ns) {SwitchToState:178(515ns)}}}}}
2025-11-19T13:59:36.877261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435859477549452:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54106
2025-11-19T13:59:36.878206Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435860109670757:2082] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.879368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435860109670748:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(45344701ns) {Configure:104(45344080ns) {ProcessConfigured:135(45050173ns) {StartInitialHandshake:157(45049725ns) {DropHandshakes:469(491ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(116ns) PrepareNewSessionHandshake:476(841ns) {SwitchToState:178(563ns)}}}}}
2025-11-19T13:59:36.879383Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435860109670718:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(27289001ns) {Configure:104(27288181ns) {ProcessConfigured:135(27065056ns) {StartInitialHandshake:157(27064606ns) {DropHandshakes:469(281ns) {DropIncomingHandshake:441(28ns) DropOutgoingHandshake:456(27ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:476(961ns) {SwitchToState:178(763ns)}}}}}
2025-11-19T13:59:36.879636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435858323879761:2091] [node 6] ICP01 ready to work
2025-11-19T13:59:36.880206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435859477549447:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:36.880286Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859477549448:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.880742Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859477549452:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.880504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435857931456036:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:36.884146Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435860109670784:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.884175Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435860109670784:2107] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.884522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435860109670796:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31823
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.884692Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435859477549444:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(50880666ns) {Configure:104(50879682ns) {ProcessConfigured:135(50619231ns) {StartInitialHandshake:157(50618762ns) {DropHandshakes:469(468ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(77ns)} GenerateSessionVirtualId:988(53ns) PrepareNewSessionHandshake:476(975ns) {SwitchToState:178(246ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.885624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860854864348:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16440
2025-11-19T13:59:36.884721Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435857931456034:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20347373ns) {Configure:104(20346398ns) {ProcessConfigured:135(19974688ns) {StartInitialHandshake:157(19974206ns) {DropHandshakes:469(920ns) {DropIncomingHandshake:441(182ns) DropOutgoingHandshake:456(114ns)} GenerateSessionVirtualId:988(254ns) PrepareNewSessionHandshake:476(1068ns) {SwitchToState:178(655ns)}}}}}
2025-11-19T13:59:36.884825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435857931456044:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26418
2025-11-19T13:59:36.884879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435857931456045:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26418
2025-11-19T13:59:36.885172Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435857931455967:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(54002464ns) {Configure:104(54000808ns) {ProcessConfigured:135(51977698ns) {StartInitialHandshake:157(51976910ns) {DropHandshakes:469(532ns) {DropIncomingHandshake:441(98ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:476(856ns) {SwitchToState:178(30ns)}}}}}
2025-11-19T13:59:36.885355Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435860842577301:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(53326600ns) {Configure:104(53325867ns) {ProcessConfigured:135(47886930ns) {StartInitialHandshake:157(47886378ns) {DropHandshakes:469(396ns) {DropIncomingHandshake:441(14ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:476(828ns) {SwitchToState:178(203ns)}}}}}
2025-11-19T13:59:36.885526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860842577382:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57075
2025-11-19T13:59:36.888081Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857931456038:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.888112Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857931456044:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.890044Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435859477549378:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(55679933ns) {Configure:104(55679120ns) {ProcessConfigured:135(55421936ns) {StartInitialHandshake:157(55421558ns) {DropHandshakes:469(493ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(279ns) PrepareNewSessionHandshake:476(917ns) {SwitchToState:178(733ns)}}}}}
2025-11-19T13:59:36.890485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435860854864330:2069] [node 4] ICP01 ready to work
2025-11-19T13:59:36.890349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857931456037:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.890368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435857931456037:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.899964Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435860854864348:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.900034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435860854864269:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.912004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435858323879761:2091] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(32264495ns) {Configure:104(32263967ns) {ProcessConfigured:135(31871577ns) {StartInitialHandshake:157(31842237ns) {DropHandshakes:469(671ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(163ns)} GenerateSessionVirtualId:988(376ns) PrepareNewSessionHandshake:476(1958ns) {SwitchToState:178(283ns)}} IncomingHandshake:261(15025ns) {DropIncomingHandshake:441(204ns) PrepareNewSessionHandshake:476(185ns) {SwitchToState:178(135ns)} GenerateSessionVirtualId:988(570ns) IssueIncomingHandshakeReply:193(374ns)}}}}
2025-11-19T13:59:36.912678Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435860109670796:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.912732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435860109670748:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.912964Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435860842577374:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.912981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435860842577374:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.913690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435860109670798:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31823
2025-11-19T13:59:36.914062Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435860109670796:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.914611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435858323879783:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24703
2025-11-19T13:59:36.916366Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435858323879783:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.917238Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857931456046:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.917255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435857931456046:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.917538Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435860109670784:2107] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:36.917648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435860109670749:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.917666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:36.917672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435860109670749:2074] [node 6] ICP22 created new session: [5:7574435860109670800:2074]
2025-11-19T13:59:36.917678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435860109670800:2074] [node 6] ICS09 handshake done sender: [5:7574435860109670784:2107] self: [5:7574435860109670751:0] peer: [6:7574435860854864345:0] socket: 528 qp: -1
2025-11-19T13:59:36.917684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435860109670800:2074] [node 6] ICS10 traffic start
2025-11-19T13:59:36.917703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:36.917856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435858323879778:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.913863Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435860842577374:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:36.914221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435860842577326:2068] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.914233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:36.914239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435860842577326:2068] [node 5] ICP22 created new session: [4:7574435860842577384:2068]
2025-11-19T13:59:36.914245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435860842577384:2068] [node 5] ICS09 handshake done sender: [4:7574435860842577374:2111] self: [4:7574435860842577340:0] peer: [5:7574435860109670797:0] socket: 533 qp: -1
2025-11-19T13:59:36.914253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435860842577384:2068] [node 5] ICS10 traffic start
2025-11-19T13:59:36.914271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:36.917065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435860842577303:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(48130681ns) {Configure:104(48130032ns) {ProcessConfigured:135(42923096ns) {StartInitialHandshake:157(42922448ns) {DropHandshakes:469(919ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(122ns)} GenerateSessionVirtualId:988(314ns) PrepareNewSessionHandshake:476(792ns) {SwitchToState:178(508ns)}}}}}
2025-11-19T13:59:36.917107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435860842577303:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.919808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435860109670748:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.919837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:36.919847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435860109670748:2070] [node 4] ICP22 created new session: [5:7574435860109670802:2070]
2025-11-19T13:59:36.919853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435860109670802:2070] [node 4] ICS09 handshake done sender: [5:7574435860109670796:2116] self: [5:7574435860109670797:0] peer: [4:7574435860842577340:0] socket: 534 qp: -1
2025-11-19T13:59:36.919860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435860109670802:2070] [node 4] ICS10 traffic start
2025-11-19T13:59:36.919868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.922756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435858323879786:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24703
2025-11-19T13:59:36.925162Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435858323879786:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.920570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435860854864330:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.920630Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435860854864330:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(30054288ns) {Configure:104(30053520ns) {ProcessConfigured:135(29667220ns) {StartInitialHandshake:157(29625148ns) {DropHandshakes:469(593ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(147ns)} GenerateSessionVirtualId:988(396ns) PrepareNewSessionHandshake:476(1390ns) {SwitchToState:178(167ns)}} IncomingHandshake:261(31805ns) {DropIncomingHandshake:441(153ns) PrepareNewSessionHandshake:476(168ns) {SwitchToState:178(122ns)} GenerateSessionVirtualId:988(532ns) IssueIncomingHandshakeReply:193(25244ns) {SwitchToState:178(61ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.929226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860854864353:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16440
2025-11-19T13:59:36.929525Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435858323879741:2085] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.929540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435858323879741:2085] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.929585Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435859477549447:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(35547031ns) {Configure:104(35545405ns) {ProcessConfigured:135(35229987ns) {StartInitialHandshake:157(35229142ns) {DropHandshakes:469(377ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(111ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:476(1158ns) {SwitchToState:178(697ns)}}}}}
2025-11-19T13:59:36.929712Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857931456041:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.929724Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435857931456041:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.930261Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435857931456037:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:36.930307Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857931456045:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.930333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435857931455967:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.931041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435857931456028:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.931059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:36.931065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435857931456028:2107] [node 4] ICP22 created new session: [2:7574435857931456057:2107]
2025-11-19T13:59:36.931071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435857931456057:2107] [node 4] ICS09 handshake done sender: [2:7574435857931456037:2111] self: [2:7574435857931456031:0] peer: [4:7574435860842577379:0] socket: 526 qp: -1
2025-11-19T13:59:36.931077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435857931456057:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:36.931111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:36.930037Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435860854864342:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.930068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435860854864268:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.931162Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435860854864288:2063] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20348) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435860854864343:2113] held: no
2025-11-19T13:59:36.931264Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435860854864343:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:36.931320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435860854864288:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.931326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:36.931331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435860854864288:2063] [node 5] ICP22 created new session: [6:7574435860854864355:2063]
2025-11-19T13:59:36.931337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435860854864355:2063] [node 5] ICS09 handshake done sender: [6:7574435860854864343:2113] self: [6:7574435860854864345:0] peer: [5:7574435860109670751:0] socket: 536 qp: -1
2025-11-19T13:59:36.931340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435860854864355:2063] [node 5] ICS10 traffic start
2025-11-19T13:59:36.931346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:36.932255Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435860842577371:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.932277Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435860842577371:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.933149Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435860842577371:2109] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.933252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860854864357:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16440
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.933731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435859477549460:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54106
2025-11-19T13:59:36.933341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435860842577337:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.933356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:36.933364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435860842577337:2075] [node 6] ICP22 created new session: [4:7574435860842577389:2075]
2025-11-19T13:59:36.933370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435860842577389:2075] [node 6] ICS09 handshake done sender: [4:7574435860842577371:2109] self: [4:7574435860842577339:0] peer: [6:7574435860854864352:0] socket: 532 qp: -1
2025-11-19T13:59:36.933376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435860842577389:2075] [node 6] ICS10 traffic start
2025-11-19T13:59:36.933383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:36.935130Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435860854864344:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:36.935206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435860854864330:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.935224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:36.935230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435860854864330:2069] [node 4] ICP22 created new session: [6:7574435860854864358:2069]
2025-11-19T13:59:36.935236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435860854864358:2069] [node 4] ICS09 handshake done sender: [6:7574435860854864344:2114] self: [6:7574435860854864352:0] peer: [4:7574435860842577339:0] socket: 537 qp: -1
2025-11-19T13:59:36.935243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435860854864358:2069] [node 4] ICS10 traffic start
2025-11-19T13:59:36.935263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:36.935348Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859477549460:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.935731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435859477549462:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54106
2025-11-19T13:59:36.940292Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435860842577382:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.940343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435860842577301:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.940541Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859477549454:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.940551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435859477549454:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.941227Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435859477549454:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:36.941341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435859477549444:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.941361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:36.941367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435859477549444:2110] [node 4] ICP22 created new session: [1:7574435859477549464:2110]
2025-11-19T13:59:36.941372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435859477549464:2110] [node 4] ICS09 handshake done sender: [1:7574435859477549454:2114] self: [1:7574435859477549445:0] peer: [4:7574435860842577392:0] socket: 542 qp: -1
2025-11-19T13:59:36.941379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435859477549464:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:36.941385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.943949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435858323879788:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24703
2025-11-19T13:59:36.944270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435860109670805:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31823
2025-11-19T13:59:36.944116Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859477549462:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.944497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435860842577301:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23251) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435860842577382:2117] held: no
2025-11-19T13:59:36.944419Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859477549450:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.944431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435859477549450:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.944349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435858323879702:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.944378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435858323879702:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(26468727ns) {IncomingHandshake:261(26468441ns) {DropIncomingHandshake:441(16ns) PrepareNewSessionHandshake:476(311ns) {SwitchToState:178(127ns)} GenerateSessionVirtualId:988(214ns) IssueIncomingHandshakeReply:193(26465631ns) {SwitchToState:178(276ns)}}}
2025-11-19T13:59:36.945001Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435858323879788:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.945052Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435858323879741:2085] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:36.945145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435858323879701:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.947327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435860842577302:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12903) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435860842577372:2110] held: no
2025-11-19T13:59:36.947521Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435857931456046:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.947858Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859477549440:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.947879Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435859477549440:2108] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.949039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435857931456034:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.949060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:36.949067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435857931456034:2109] [node 6] ICP22 created new session: [2:7574435857931456061:2109]
2025-11-19T13:59:36.949073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435857931456061:2109] [node 6] ICS09 handshake done sender: [2:7574435857931456046:2117] self: [2:7574435857931456043:0] peer: [6:7574435860854864350:0] socket: 543 qp: -1
2025-11-19T13:59:36.949080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435857931456061:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:36.949088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:36.950839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435860109670749:2074] [node 6] ICP35 event processing took too much time PendingConnection:278(33162158ns) {HandleHandshakeStatus:329(33161400ns) {DropIncomingHandshake:441(3973ns) DropOutgoingHandshake:456(146ns) SwitchToState:178(83ns) ProcessPendingSessionEvents:495(21116595ns) {ForwardSessionEventToSession:612(21106236ns) ForwardSessionEventToSession:612(1462ns) ForwardSessionEventToSession:612(747ns) ForwardSessionEventToSession:612(539ns) ForwardSessionEventToSession:612(579ns) ForwardSessionEventToSession:612(654ns) ForwardSessionEventToSession:612(863ns) ForwardSessionEventToSession:612(538ns) ForwardSessionEventToSession:612(629ns) ForwardSessionEventToSession:612(852ns)}}}
2025-11-19T13:59:36.951367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435860109670806:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31823
2025-11-19T13:59:36.951323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860842577393:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57075
2025-11-19T13:59:36.952293Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435860842577372:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:36.952504Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435857931456041:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:36.952366Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859477549456:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.952380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435859477549456:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.952888Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435859477549440:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:36.954011Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435859477549456:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:36.954079Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435860854864269:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12903) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435860854864348:2117] held: no
2025-11-19T13:59:36.954185Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435859477549450:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:36.954244Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435860109670805:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.954307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435860109670717:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.954444Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435857931456028:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(23368470ns) {HandleHandshakeStatus:329(23367619ns) {DropIncomingHandshake:441(4231ns) DropOutgoingHandshake:456(171ns) SwitchToState:178(58ns) ProcessPendingSessionEvents:495(9794057ns) {ForwardSessionEventToSession:612(9781176ns) ForwardSessionEventToSession:612(2072ns) ForwardSessionEventToSession:612(1005ns) ForwardSessionEventToSession:612(601ns) ForwardSessionEventToSession:612(801ns) ForwardSessionEventToSession:612(968ns) ForwardSessionEventToSession:612(863ns) ForwardSessionEventToSession:612(930ns) ForwardSessionEventToSession:612(518ns) ForwardSessionEventToSession:612(746ns)}}}
2025-11-19T13:59:36.954548Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435857931456050:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.954551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435857931456050:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.955213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435857931456065:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26418
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.955416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435860842577302:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.955435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:36.955443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435860842577302:2049] [node 2] ICP22 created new session: [4:7574435860842577396:2049]
2025-11-19T13:59:36.955449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435860842577396:2049] [node 2] ICS09 handshake done sender: [4:7574435860842577372:2110] self: [4:7574435860842577379:0] peer: [2:7574435857931456031:0] socket: 531 qp: -1
2025-11-19T13:59:36.955454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435860842577396:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:36.955462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:36.955627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860842577394:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57075
2025-11-19T13:59:36.955164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435859477549378:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.955185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:36.955193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435859477549378:2049] [node 3] ICP22 created new session: [1:7574435859477549471:2049]
2025-11-19T13:59:36.955198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435859477549471:2049] [node 3] ICS09 handshake done sender: [1:7574435859477549456:2115] self: [1:7574435859477549446:0] peer: [3:7574435858323879790:0] socket: 562 qp: -1
2025-11-19T13:59:36.955205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435859477549471:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:36.955223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.956422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435860842577395:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57075
2025-11-19T13:59:36.956465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860854864361:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16440
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.956527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860854864362:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16440
2025-11-19T13:59:36.956571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435859477549377:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.956583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:36.956591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435859477549377:2048] [node 2] ICP22 created new session: [1:7574435859477549473:2048]
2025-11-19T13:59:36.956597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435859477549473:2048] [node 2] ICS09 handshake done sender: [1:7574435859477549450:2112] self: [1:7574435859477549449:0] peer: [2:7574435857931456056:0] socket: 530 qp: -1
2025-11-19T13:59:36.956602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435859477549473:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:36.956608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:36.957005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435858323879739:2084] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.957036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:36.957042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435858323879739:2084] [node 4] ICP22 created new session: [3:7574435858323879792:2084]
2025-11-19T13:59:36.957049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435858323879792:2084] [node 4] ICS09 handshake done sender: [3:7574435858323879741:2085] self: [3:7574435858323879740:0] peer: [4:7574435860842577387:0] socket: 511 qp: -1
2025-11-19T13:59:36.957054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435858323879792:2084] [node 4] ICS10 traffic start
2025-11-19T13:59:36.957075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:36.955977Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435857931456050:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:36.956205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435857931456036:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.956219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:36.956225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435857931456036:2110] [node 5] ICP22 created new session: [2:7574435857931456067:2110]
2025-11-19T13:59:36.956230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435857931456067:2110] [node 5] ICS09 handshake done sender: [2:7574435857931456050:2118] self: [2:7574435857931456049:0] peer: [5:7574435860109670807:0] socket: 564 qp: -1
2025-11-19T13:59:36.956236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435857931456067:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:36.956243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:36.956990Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435857931456045:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:36.957839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435859477549438:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.957855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:36.957859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435859477549438:2107] [node 6] ICP22 created new session: [1:7574435859477549475:2107]
2025-11-19T13:59:36.957864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435859477549475:2107] [node 6] ICS09 handshake done sender: [1:7574435859477549440:2108] self: [1:7574435859477549439:0] peer: [6:7574435860854864354:0] socket: 515 qp: -1
2025-11-19T13:59:36.957868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435859477549475:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:36.957875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:36.957985Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435860854864348:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:36.958059Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435860854864342:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.958443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435858323879794:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24703
2025-11-19T13:59:36.958560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435858323879795:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24703
2025-11-19T13:59:36.958973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435860109670717:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12903) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435860109670805:2118] held: no
2025-11-19T13:59:36.961522Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435860109670806:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.961555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435860109670718:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.961815Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435858323879701:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23251) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435858323879788:2119] held: no
2025-11-19T13:59:36.961870Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435858323879779:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.961872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435858323879779:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:36.962320Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435858323879788:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:36.963150Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435858323879778:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:36.963224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435858323879701:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.963238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:36.963245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435858323879701:2048] [node 1] ICP22 created new session: [3:7574435858323879797:2048]
2025-11-19T13:59:36.963251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435858323879797:2048] [node 1] ICS09 handshake done sender: [3:7574435858323879788:2119] self: [3:7574435858323879790:0] peer: [1:7574435859477549446:0] socket: 565 qp: -1
2025-11-19T13:59:36.963258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435858323879797:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:36.963291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:36.963466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435858323879702:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.963474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:36.963479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435858323879702:2049] [node 2] ICP22 created new session: [3:7574435858323879799:2049]
2025-11-19T13:59:36.963483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435858323879799:2049] [node 2] ICS09 handshake done sender: [3:7574435858323879778:2114] self: [3:7574435858323879785:0] peer: [2:7574435857931456033:0] socket: 538 qp: -1
2025-11-19T13:59:36.963491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435858323879799:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:36.963495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:36.964606Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435860842577382:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.965025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435859477549477:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54106
2025-11-19T13:59:36.964809Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435858323879779:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:36.965103Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435860842577328:2073] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:36.965175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435858323879749:2089] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.965194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:36.965199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435858323879749:2089] [node 5] ICP22 created new session: [3:7574435858323879802:2089]
2025-11-19T13:59:36.965205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435858323879802:2089] [node 5] ICS09 handshake done sender: [3:7574435858323879779:2115] self: [3:7574435858323879752:0] peer: [5:7574435860109670808:0] socket: 558 qp: -1
2025-11-19T13:59:36.965209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435858323879802:2089] [node 5] ICS10 traffic start
2025-11-19T13:59:36.965226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:36.965246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435860842577301:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.965259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:36.965264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435860842577301:2048] [node 1] ICP22 created new session: [4:7574435860842577398:2048]
2025-11-19T13:59:36.965270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435860842577398:2048] [node 1] ICS09 handshake done sender: [4:7574435860842577382:2117] self: [4:7574435860842577392:0] peer: [1:7574435859477549445:0] socket: 544 qp: -1
2025-11-19T13:59:36.965275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435860842577398:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:36.965282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:36.965363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435860842577303:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.965370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:36.965374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435860842577303:2050] [node 3] ICP22 created new session: [4:7574435860842577400:2050]
2025-11-19T13:59:36.965378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435860842577400:2050] [node 3] ICS09 handshake done sender: [4:7574435860842577328:2073] self: [4:7574435860842577387:0] peer: [3:7574435858323879740:0] socket: 514 qp: -1
2025-11-19T13:59:36.965381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435860842577400:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:36.965386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:36.966857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435860854864269:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.966895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:36.966903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435860854864269:2049] [node 2] ICP22 created new session: [6:7574435860854864363:2049]
2025-11-19T13:59:36.966908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435860854864363:2049] [node 2] ICS09 handshake done sender: [6:7574435860854864348:2117] self: [6:7574435860854864350:0] peer: [2:7574435857931456043:0] socket: 545 qp: -1
2025-11-19T13:59:36.966915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435860854864363:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:36.966921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:36.967012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435860854864268:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.967020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:36.967023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435860854864268:2048] [node 1] ICP22 created new session: [6:7574435860854864365:2048]
2025-11-19T13:59:36.967028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435860854864365:2048] [node 1] ICS09 handshake done sender: [6:7574435860854864342:2112] self: [6:7574435860854864354:0] peer: [1:7574435859477549439:0] socket: 516 qp: -1
2025-11-19T13:59:36.967032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435860854864365:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:36.967037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:36.969436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435860109670718:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32608) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435860109670806:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.970221Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435858323879781:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.970229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435858323879781:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.971182Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435858323879781:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:36.971290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435858323879761:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.971307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:36.971314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435858323879761:2091] [node 6] ICP22 created new session: [3:7574435858323879805:2091]
2025-11-19T13:59:36.971320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435858323879805:2091] [node 6] ICS09 handshake done sender: [3:7574435858323879781:2116] self: [3:7574435858323879780:0] peer: [6:7574435860854864368:0] socket: 517 qp: -1
2025-11-19T13:59:36.971324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435858323879805:2091] [node 6] ICS10 traffic start
2025-11-19T13:59:36.971332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:36.969883Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435860854864268:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:36.969939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860854864367:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16440
2025-11-19T13:59:36.970122Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435860854864367:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.970148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435860854864270:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.971105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435860854864369:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16440
2025-11-19T13:59:36.971631Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435860854864367:2123] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:36.971861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435860854864270:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.971871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:36.971877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435860854864270:2050] [node 3] ICP22 created new session: [6:7574435860854864370:2050]
2025-11-19T13:59:36.971881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435860854864370:2050] [node 3] ICS09 handshake done sender: [6:7574435860854864367:2123] self: [6:7574435860854864368:0] peer: [3:7574435858323879780:0] socket: 518 qp: -1
2025-11-19T13:59:36.971887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435860854864370:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:36.971895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:36.974047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435857931455967:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.974071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:36.974078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435857931455967:2048] [node 1] ICP22 created new session: [2:7574435857931456069:2048]
2025-11-19T13:59:36.974084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435857931456069:2048] [node 1] ICS09 handshake done sender: [2:7574435857931456045:2116] self: [2:7574435857931456056:0] peer: [1:7574435859477549449:0] socket: 541 qp: -1
2025-11-19T13:59:36.974090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435857931456069:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:36.974096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:36.974203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435857931455968:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.974213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:36.974218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435857931455968:2049] [node 3] ICP22 created new session: [2:7574435857931456071:2049]
2025-11-19T13:59:36.974222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435857931456071:2049] [node 3] ICS09 handshake done sender: [2:7574435857931456041:2114] self: [2:7574435857931456033:0] peer: [3:7574435858323879785:0] socket: 527 qp: -1
2025-11-19T13:59:36.974227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435857931456071:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:36.974231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:36.976048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435859477549378:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(20841923ns) {HandleHandshakeStatus:329(20841358ns) {DropIncomingHandshake:441(4146ns) DropOutgoingHandshake:456(554ns) SwitchToState:178(47ns) ProcessPendingSessionEvents:495(20732151ns) {ForwardSessionEventToSession:612(20719978ns) ForwardSessionEventToSession:612(955ns) ForwardSessionEventToSession:612(1089ns) ForwardSessionEventToSession:612(845ns) ForwardSessionEventToSession:612(485ns) ForwardSessionEventToSession:612(1189ns) ForwardSessionEventToSession:612(817ns) ForwardSessionEventToSession:612(574ns) ForwardSessionEventToSession:612(554ns) ForwardSessionEventToSession:612(898ns)}}}
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.979369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435860109670811:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31823
::ffff:127.1.0.0
2025-11-19T13:59:36.979474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435860109670809:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31823
2025-11-19T13:59:36.979591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435859477549377:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:36.979690Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435859477549438:2107] [node 6] ICP35 event processing took too much time PendingConnection:278(21827012ns) {HandleHandshakeStatus:329(21826448ns) {DropIncomingHandshake:441(3213ns) DropOutgoingHandshake:456(81ns) SwitchToState:178(43ns) ProcessPendingSessionEvents:495(21751522ns) {ForwardSessionEventToSession:612(21739531ns) ForwardSessionEventToSession:612(1404ns) ForwardSessionEventToSession:612(530ns) ForwardSessionEventToSession:612(688ns) ForwardSessionEventToSession:612(537ns) ForwardSessionEventToSession:612(1113ns) ForwardSessionEventToSession:612(736ns) ForwardSessionEventToSession:612(527ns) ForwardSessionEventToSession:612(590ns) ForwardSessionEventToSession:612(617ns)}}}
2025-11-19T13:59:36.979767Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435860109670805:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:36.980000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435860109670717:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.980016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:36.980022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435860109670717:2049] [node 2] ICP22 created new session: [5:7574435860109670812:2049]
2025-11-19T13:59:36.980028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435860109670812:2049] [node 2] ICS09 handshake done sender: [5:7574435860109670805:2118] self: [5:7574435860109670807:0] peer: [2:7574435857931456049:0] socket: 567 qp: -1
2025-11-19T13:59:36.980034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435860109670812:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:36.980045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:36.984217Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435857931456036:2110] [node 5] ICP35 event processing took too much time PendingConnection:278(27975486ns) {HandleHandshakeStatus:329(27974288ns) {DropIncomingHandshake:441(3414ns) DropOutgoingHandshake:456(101ns) SwitchToState:178(75ns) ProcessPendingSessionEvents:495(27898208ns) {ForwardSessionEventToSession:612(27883869ns) ForwardSessionEventToSession:612(2509ns) ForwardSessionEventToSession:612(1191ns) ForwardSessionEventToSession:612(500ns) ForwardSessionEventToSession:612(821ns) ForwardSessionEventToSession:612(976ns) ForwardSessionEventToSession:612(609ns) ForwardSessionEventToSession:612(525ns) ForwardSessionEventToSession:612(618ns) ForwardSessionEventToSession:612(1068ns)}}}
2025-11-19T13:59:36.984442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435860109670810:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31823
2025-11-19T13:59:36.984614Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435858323879739:2084] [node 4] ICP35 event processing took too much time PendingConnection:278(38515012ns) {HandleHandshakeStatus:329(38514567ns) {DropIncomingHandshake:441(12314ns) DropOutgoingHandshake:456(74ns) SwitchToState:178(157ns) ProcessPendingSessionEvents:495(8243687ns) {ForwardSessionEventToSession:612(8231565ns) ForwardSessionEventToSession:612(1880ns) ForwardSessionEventToSession:612(824ns) ForwardSessionEventToSession:612(570ns) ForwardSessionEventToSession:612(647ns) ForwardSessionEventToSession:612(901ns) ForwardSessionEventToSession:612(622ns) ForwardSessionEventToSession:612(536ns) ForwardSessionEventToSession:612(597ns) ForwardSessionEventToSession:612(1312ns)}}}
2025-11-19T13:59:36.984860Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435860109670811:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.984889Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435860109670806:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:36.984952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435860109670716:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:36.985092Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435859477549458:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:36.985104Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435859477549458:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:36.986056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435860109670815:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31823
2025-11-19T13:59:36.986292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435860109670718:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.986308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:36.986314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435860109670718:2050] [node 3] ICP22 created new session: [5:7574435860109670816:2050]
2025-11-19T13:59:36.986319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435860109670816:2050] [node 3] ICS09 handshake done sender: [5:7574435860109670806:2119] self: [5:7574435860109670808:0] peer: [3:7574435858323879752:0] socket: 569 qp: -1
2025-11-19T13:59:36.986325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435860109670816:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:36.986335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:36.986757Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435859477549458:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:36.986979Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435860109670811:2122] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:36.987051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435860109670716:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:36.987067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:36.987072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435860109670716:2048] [node 1] ICP22 created new session: [5:7574435860109670818:2048]
2025-11-19T13:59:36.987077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435860109670818:2048] [node 1] ICS09 handshake done sender: [5:7574435860109670811:2122] self: [5:7574435860109670814:0] peer: [1:7574435859477549453:0] socket: 523 qp: -1
2025-11-19T13:59:36.987081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435860109670818:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:36.987089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:36.988036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435859477549447:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:36.988060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:36.988065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435859477549447:2109] [node 5] ICP22 created new session: [1:7574435859477549480:2109]
2025-11-19T13:59:36.988070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435859477549480:2109] [node 5] ICS09 handshake done sender: [1:7574435859477549458:2116] self: [1:7574435859477549453:0] peer: [5:7574435860109670814:0] socket: 560 qp: -1
2025-11-19T13:59:36.988075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435859477549480:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:36.988085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:37.004381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:37.004476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435857931456069:2048] [node 1] ICS01 socket: 541 reason# EndOfStream
2025-11-19T13:59:37.004486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435857931455967:2048] [node 1] ICP30 unregister session Session# [2:7574435857931456069:2048] VirtualId# [2:7574435857931456056:0]
2025-11-19T13:59:37.004494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435857931456069:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.004534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:37.004554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:37.004999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:37.005065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435858323879797:2048] [node 1] ICS01 socket: 565 reason# EndOfStream
2025-11-19T13:59:37.005069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435858323879701:2048] [node 1] ICP30 unregister session Session# [3:7574435858323879797:2048] VirtualId# [3:7574435858323879790:0]
2025-11-19T13:59:37.005074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435858323879797:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.005102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:37.005121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:37.018380Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:37.018519Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:37.018575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435860842577398:2048] [node 1] ICS01 socket: 544 reason# EndOfStream
2025-11-19T13:59:37.018579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435860842577301:2048] [node 1] ICP30 unregister session Session# [4:7574435860842577398:2048] VirtualId# [4:7574435860842577392:0]
2025-11-19T13:59:37.018583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435860842577398:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.018441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435860109670818:2048] [node 1] ICS01 socket: 523 reason# EndOfStream
2025-11-19T13:59:37.018614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:37.018634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:37.018168Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:37.018264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435860854864365:2048] [node 1] ICS01 socket: 516 reason# EndOfStream
2025-11-19T13:59:37.018272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435860854864268:2048] [node 1] ICP30 unregister session Session# [6:7574435860854864365:2048] VirtualId# [6:7574435860854864354:0]
2025-11-19T13:59:37.018279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435860854864365:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.018325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:37.018348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:37.018444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435860109670716:2048] [node 1] ICP30 unregister session Session# [5:7574435860109670818:2048] VirtualId# [5:7574435860109670814:0]
2025-11-19T13:59:37.018448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435860109670818:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.018472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:37.018490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:37.029477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:37.029562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435858323879799:2049] [node 2] ICS01 socket: 538 reason# EndOfStream
2025-11-19T13:59:37.029570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435858323879702:2049] [node 2] ICP30 unregister session Session# [3:7574435858323879799:2049] VirtualId# [3:7574435858323879785:0]
2025-11-19T13:59:37.029576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435858323879799:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.029608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:37.029629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:37.032878Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:37.032948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435860109670812:2049] [node 2] ICS01 socket: 567 reason# EndOfStream
2025-11-19T13:59:37.032955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435860109670717:2049] [node 2] ICP30 unregister session Session# [5:7574435860109670812:2049] VirtualId# [5:7574435860109670807:0]
2025-11-19T13:59:37.032961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435860109670812:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.033001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:37.033026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:37.033113Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:37.033193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435860842577396:2049] [node 2] ICS01 socket: 531 reason# EndOfStream
2025-11-19T13:59:37.033198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435860842577302:2049] [node 2] ICP30 unregister session Session# [4:7574435860842577396:2049] VirtualId# [4:7574435860842577379:0]
2025-11-19T13:59:37.033202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435860842577396:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.033233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:37.033252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:37.033283Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:37.033344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435860854864363:2049] [node 2] ICS01 socket: 545 reason# EndOfStream
2025-11-19T13:59:37.033348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435860854864269:2049] [node 2] ICP30 unregister session Session# [6:7574435860854864363:2049] VirtualId# [6:7574435860854864350:0]
2025-11-19T13:59:37.033352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435860854864363:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.033376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:37.033394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:37.049052Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:37.049159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435860854864370:2050] [node 3] ICS01 socket: 518 reason# EndOfStream
2025-11-19T13:59:37.049169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435860854864270:2050] [node 3] ICP30 unregister session Session# [6:7574435860854864370:2050] VirtualId# [6:7574435860854864368:0]
2025-11-19T13:59:37.049177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435860854864370:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.049244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:37.049272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:37.049365Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:37.049432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435860109670816:2050] [node 3] ICS01 socket: 569 reason# EndOfStream
2025-11-19T13:59:37.049438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435860109670718:2050] [node 3] ICP30 unregister session Session# [5:7574435860109670816:2050] VirtualId# [5:7574435860109670808:0]
2025-11-19T13:59:37.049443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435860109670816:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.049477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:37.049502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:37.074861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:37.074978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435860854864358:2069] [node 4] ICS01 socket: 537 reason# EndOfStream
2025-11-19T13:59:37.074988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435860854864330:2069] [node 4] ICP30 unregister session Session# [6:7574435860854864358:2069] VirtualId# [6:7574435860854864352:0]
2025-11-19T13:59:37.074996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435860854864358:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.075072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:37.075102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:37.085547Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:59:37.085643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435860854864355:2063] [node 5] ICS01 socket: 536 reason# EndOfStream
2025-11-19T13:59:37.085653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435860854864288:2063] [node 5] ICP30 unregister session Session# [6:7574435860854864355:2063] VirtualId# [6:7574435860854864345:0]
2025-11-19T13:59:37.085661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435860854864355:2063] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.085712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:59:37.085734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:37.484144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435865337027216:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:37.484182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435865337027217:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:37.538692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435865168942791:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:37.538713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435865168942792:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:37.563725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435864559633704:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:37.563737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435864559633705:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:37.596300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435865772236088:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:37.596314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435865772236089:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:37.596318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435865772236090:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:37.639914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435863356639605:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:37.639930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435863356639606:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:37.639934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435863356639607:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:37.689125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435865678939363:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:37.689139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435865678939364:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:37.689152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435865678939365:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.722119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435863356639655:2096] [node 6] ICP01 ready to work
2025-11-19T13:59:37.721349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435865678939388:2071] [node 4] ICP01 ready to work
2025-11-19T13:59:37.722483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435865772236146:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57104
2025-11-19T13:59:37.724066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435863356639660:2098] [node 4] ICP01 ready to work
2025-11-19T13:59:37.728662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435865337027281:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:37.728757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435865337027282:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:37.728827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435865337027283:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.729469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435865168942854:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:37.729581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435865168942855:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:37.729660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435865168942856:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:37.730297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435865772236147:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57104
2025-11-19T13:59:37.731732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435865772236152:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:37.731193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435865678939409:2076] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.747192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435865678939437:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6944
2025-11-19T13:59:37.747216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435865678939438:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6944
2025-11-19T13:59:37.744209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435865772236155:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57104
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.757503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435865772236158:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.758701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435863356639683:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-19T13:59:37.758730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435863356639684:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-19T13:59:37.752342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435865168942855:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(21733841ns) {Configure:104(21733345ns) {ProcessConfigured:135(21526428ns) {StartInitialHandshake:157(21525837ns) {DropHandshakes:469(104ns) {DropIncomingHandshake:441(16ns) DropOutgoingHandshake:456(23ns)} GenerateSessionVirtualId:988(41ns) PrepareNewSessionHandshake:476(1857ns) {SwitchToState:178(874ns)}}}}}
2025-11-19T13:59:37.753005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435865168942867:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21314
2025-11-19T13:59:37.753017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435865168942868:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21314
2025-11-19T13:59:37.753027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435865168942869:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21314
2025-11-19T13:59:37.759254Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435864559633705:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(35200281ns) {Configure:104(35199135ns) {ProcessConfigured:135(34753313ns) {StartInitialHandshake:157(34752892ns) {DropHandshakes:469(565ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(124ns)} GenerateSessionVirtualId:988(373ns) PrepareNewSessionHandshake:476(1178ns) {SwitchToState:178(457ns)}}}}}
2025-11-19T13:59:37.759416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435864559633769:2106] [node 4] ICP01 ready to work
2025-11-19T13:59:37.759503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435864559633770:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.773762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435863356639685:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.772224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435864559633774:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:37.773359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435864559633771:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2918
2025-11-19T13:59:37.773372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435864559633772:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2918
2025-11-19T13:59:37.773381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435864559633773:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2918
2025-11-19T13:59:37.778910Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435865168942868:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.778596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435865337027295:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4964
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.781963Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435865772236155:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.782198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435865772236169:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57104
2025-11-19T13:59:37.783481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435865168942873:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21314
2025-11-19T13:59:37.787459Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435865772236147:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.787501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435865772236088:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:37.787661Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435865337027287:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.787672Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435865337027287:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.788897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435865772236172:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57104
2025-11-19T13:59:37.789394Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435865337027287:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:37.789566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435865168942874:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21314
2025-11-19T13:59:37.789646Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435865168942867:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.792550Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435864559633771:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.792587Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435864559633773:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.794814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435864559633704:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:37.799861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435865337027281:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.799892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:37.799904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435865337027281:2107] [node 4] ICP22 created new session: [1:7574435865337027297:2107]
2025-11-19T13:59:37.799914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435865337027297:2107] [node 4] ICS09 handshake done sender: [1:7574435865337027287:2112] self: [1:7574435865337027286:0] peer: [4:7574435865772236171:0] socket: 520 qp: -1
2025-11-19T13:59:37.799927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435865337027297:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:37.799939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:37.800171Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435865337027285:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.800176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435865337027285:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:37.802601Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435865337027285:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.804664Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435865772236146:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.804739Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435865772236147:2106] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:37.808317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435865337027217:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.808342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:37.808353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435865337027217:2049] [node 3] ICP22 created new session: [1:7574435865337027301:2049]
2025-11-19T13:59:37.808361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435865337027301:2049] [node 3] ICS09 handshake done sender: [1:7574435865337027285:2111] self: [1:7574435865337027284:0] peer: [3:7574435864559633786:0] socket: 519 qp: -1
2025-11-19T13:59:37.808368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435865337027301:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:37.808377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:37.804991Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435865678939438:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.805033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435865678939388:2071] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.810112Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435863356639685:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.810251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435863356639686:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-19T13:59:37.810282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435863356639607:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:37.810608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435864559633788:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2918
2025-11-19T13:59:37.816103Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435864559633778:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.816120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435864559633778:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:37.816776Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435865772236169:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.816818Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435865772236154:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.816821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435865772236154:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:37.817333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435865772236088:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.817354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:37.817362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435865772236088:2048] [node 1] ICP22 created new session: [4:7574435865772236173:2048]
2025-11-19T13:59:37.817368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435865772236173:2048] [node 1] ICS09 handshake done sender: [4:7574435865772236147:2106] self: [4:7574435865772236171:0] peer: [1:7574435865337027286:0] socket: 521 qp: -1
2025-11-19T13:59:37.817379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435865772236173:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:37.817391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:37.817862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435865772236090:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:37.816654Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435864559633771:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:37.820074Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435865772236154:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:37.820209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435865772236152:2108] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-11-19T13:59:37.820391Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435865678939437:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.820228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:37.820238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435865772236152:2108] [node 6] ICP22 created new session: [4:7574435865772236179:2108]
2025-11-19T13:59:37.820243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435865772236179:2108] [node 6] ICS09 handshake done sender: [4:7574435865772236154:2111] self: [4:7574435865772236153:0] peer: [6:7574435865678939443:0] socket: 526 qp: -1
2025-11-19T13:59:37.820249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435865772236179:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:37.820258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
::ffff:127.1.0.0
2025-11-19T13:59:37.820736Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435865337027280:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.820748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435865337027280:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.822734Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435863356639683:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.822764Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435863356639686:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.822814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435863356639688:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-19T13:59:37.822856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435863356639605:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:37.823161Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435864559633776:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.823176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435864559633776:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:37.823093Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435863356639688:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.823134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435863356639660:2098] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.823465Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435865337027280:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:37.819821Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435865168942873:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.819907Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435865168942869:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.819984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435865168942791:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:37.823277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435865772236164:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.823287Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435865772236164:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:37.824209Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435865772236164:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:37.823224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435865678939444:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6944
2025-11-19T13:59:37.824416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435865772236158:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.824436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:37.824443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435865772236158:2113] [node 5] ICP22 created new session: [4:7574435865772236182:2113]
2025-11-19T13:59:37.824449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435865772236182:2113] [node 5] ICS09 handshake done sender: [4:7574435865772236164:2116] self: [4:7574435865772236163:0] peer: [5:7574435863356639691:0] socket: 561 qp: -1
2025-11-19T13:59:37.824455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435865772236182:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:37.824465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:37.828050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435865678939445:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6944
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.828114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435863356639692:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-19T13:59:37.829297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435865678939409:2076] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:37.829086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435865168942879:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21314
2025-11-19T13:59:37.830192Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435865168942867:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:37.830249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435865168942791:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.830266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:37.830272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435865168942791:2048] [node 1] ICP22 created new session: [2:7574435865168942880:2048]
2025-11-19T13:59:37.830278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435865168942880:2048] [node 1] ICS09 handshake done sender: [2:7574435865168942867:2115] self: [2:7574435865168942876:0] peer: [1:7574435865337027277:0] socket: 531 qp: -1
2025-11-19T13:59:37.830286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435865168942880:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:37.830294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:37.831376Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435865337027289:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.831388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435865337027289:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:37.832068Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435865337027295:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.832078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435865337027216:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.832090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:37.832096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435865337027216:2048] [node 2] ICP22 created new session: [1:7574435865337027305:2048]
2025-11-19T13:59:37.832102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435865337027305:2048] [node 2] ICS09 handshake done sender: [1:7574435865337027280:2110] self: [1:7574435865337027277:0] peer: [2:7574435865168942876:0] socket: 518 qp: -1
2025-11-19T13:59:37.832107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435865337027305:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:37.832117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:37.832420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435865337027216:2048] [node 2] ICP18 handshake (actor [1:7574435865337027295:2115]) is too old
2025-11-19T13:59:37.832949Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435864559633772:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.832966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435864559633704:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.832978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:37.832984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435864559633704:2048] [node 1] ICP22 created new session: [3:7574435864559633790:2048]
2025-11-19T13:59:37.832990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435864559633790:2048] [node 1] ICS09 handshake done sender: [3:7574435864559633771:2111] self: [3:7574435864559633786:0] peer: [1:7574435865337027284:0] socket: 542 qp: -1
2025-11-19T13:59:37.832995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435864559633790:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:37.833002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:37.833367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435864559633705:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.833497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435865772236184:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57104
2025-11-19T13:59:37.833663Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435863356639660:2098] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4319) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435863356639688:2119] held: no
2025-11-19T13:59:37.833729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435863356639684:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.833834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435863356639693:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-19T13:59:37.833965Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435865678939388:2071] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4319) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435865678939438:2114] held: no
2025-11-19T13:59:37.834057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435865678939444:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.834112Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435865678939438:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:37.834336Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435863356639659:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.834347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435863356639659:2099] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.835700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435863356639606:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:37.835764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435863356639694:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-19T13:59:37.835874Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435863356639688:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.836130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435865678939388:2071] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.836147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:37.836152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435865678939388:2071] [node 4] ICP22 created new session: [6:7574435865678939448:2071]
2025-11-19T13:59:37.836157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435865678939448:2071] [node 4] ICS09 handshake done sender: [6:7574435865678939438:2114] self: [6:7574435865678939443:0] peer: [4:7574435865772236153:0] socket: 528 qp: -1
2025-11-19T13:59:37.836163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435865678939448:2071] [node 4] ICS10 traffic start
2025-11-19T13:59:37.836170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:37.836316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435865678939363:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:37.836338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435865678939447:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6944
2025-11-19T13:59:37.836002Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435864559633778:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:37.836688Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435864559633776:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:37.837880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435864559633770:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.837890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:37.837895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435864559633770:2107] [node 5] ICP22 created new session: [3:7574435864559633798:2107]
2025-11-19T13:59:37.837901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435864559633798:2107] [node 5] ICS09 handshake done sender: [3:7574435864559633778:2115] self: [3:7574435864559633777:0] peer: [5:7574435863356639687:0] socket: 548 qp: -1
2025-11-19T13:59:37.837906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435864559633798:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:37.837912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:37.838020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435864559633769:2106] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.838029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:37.838032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435864559633769:2106] [node 4] ICP22 created new session: [3:7574435864559633800:2106]
2025-11-19T13:59:37.838035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435864559633800:2106] [node 4] ICS09 handshake done sender: [3:7574435864559633776:2114] self: [3:7574435864559633775:0] peer: [4:7574435865772236177:0] socket: 547 qp: -1
2025-11-19T13:59:37.838039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435864559633800:2106] [node 4] ICS10 traffic start
2025-11-19T13:59:37.838045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:37.838504Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435865772236169:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.836824Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435865168942874:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.836868Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435865168942858:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.836872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435865168942858:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:37.839884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435865772236090:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.839901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:37.839907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435865772236090:2050] [node 3] ICP22 created new session: [4:7574435865772236186:2050]
2025-11-19T13:59:37.839912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435865772236186:2050] [node 3] ICS09 handshake done sender: [4:7574435865772236169:2118] self: [4:7574435865772236177:0] peer: [3:7574435864559633775:0] socket: 551 qp: -1
2025-11-19T13:59:37.839918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435865772236186:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:37.839924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:37.840062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435865772236185:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57104
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.841862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435864559633795:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2918
2025-11-19T13:59:37.841892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435864559633796:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2918
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.842103Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435865337027289:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:37.841917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435864559633797:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2918
2025-11-19T13:59:37.842334Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435865337027291:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.842340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435865337027291:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:37.842748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435865337027282:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.842758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:37.842763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435865337027282:2108] [node 5] ICP22 created new session: [1:7574435865337027308:2108]
2025-11-19T13:59:37.842768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435865337027308:2108] [node 5] ICS09 handshake done sender: [1:7574435865337027289:2113] self: [1:7574435865337027288:0] peer: [5:7574435863356639690:0] socket: 555 qp: -1
2025-11-19T13:59:37.842773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435865337027308:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:37.842780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:37.851879Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435865168942858:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:37.852057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435865168942792:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.852081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:37.852089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435865168942792:2049] [node 3] ICP22 created new session: [2:7574435865168942885:2049]
2025-11-19T13:59:37.852096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435865168942885:2049] [node 3] ICS09 handshake done sender: [2:7574435865168942858:2110] self: [2:7574435865168942857:0] peer: [3:7574435864559633793:0] socket: 534 qp: -1
2025-11-19T13:59:37.852102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435865168942885:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:37.852113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.854430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435865678939452:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6944
2025-11-19T13:59:37.854452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435865678939453:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6944
2025-11-19T13:59:37.854826Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435865337027291:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:37.855108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435865337027283:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.855124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:37.855130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435865337027283:2109] [node 6] ICP22 created new session: [1:7574435865337027311:2109]
2025-11-19T13:59:37.855137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435865337027311:2109] [node 6] ICS09 handshake done sender: [1:7574435865337027291:2114] self: [1:7574435865337027290:0] peer: [6:7574435865678939451:0] socket: 558 qp: -1
2025-11-19T13:59:37.855144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435865337027311:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:37.855153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:37.855602Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435865678939444:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:37.855730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435863356639660:2098] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.855769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:37.855774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435863356639660:2098] [node 4] ICP22 created new session: [5:7574435863356639698:2098]
2025-11-19T13:59:37.855779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435863356639698:2098] [node 4] ICS09 handshake done sender: [5:7574435863356639688:2119] self: [5:7574435863356639691:0] peer: [4:7574435865772236163:0] socket: 562 qp: -1
2025-11-19T13:59:37.855785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435863356639698:2098] [node 4] ICS10 traffic start
2025-11-19T13:59:37.855792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.856788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435865678939454:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6944
2025-11-19T13:59:37.856803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435865678939363:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.856816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:37.856821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435865678939363:2048] [node 1] ICP22 created new session: [6:7574435865678939455:2048]
2025-11-19T13:59:37.856827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435865678939455:2048] [node 1] ICS09 handshake done sender: [6:7574435865678939444:2117] self: [6:7574435865678939451:0] peer: [1:7574435865337027290:0] socket: 559 qp: -1
2025-11-19T13:59:37.856832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435865678939455:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:37.856839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:37.858705Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435865678939437:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:37.858861Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435864559633796:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.858945Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435864559633705:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16979) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435864559633772:2112] held: no
2025-11-19T13:59:37.858993Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435864559633772:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:37.859109Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435865678939447:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.859122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435865678939409:2076] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.859132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:37.859136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435865678939409:2076] [node 5] ICP22 created new session: [6:7574435865678939457:2076]
2025-11-19T13:59:37.859140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435865678939457:2076] [node 5] ICS09 handshake done sender: [6:7574435865678939437:2113] self: [6:7574435865678939446:0] peer: [5:7574435863356639656:0] socket: 527 qp: -1
2025-11-19T13:59:37.859146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435865678939457:2076] [node 5] ICS10 traffic start
2025-11-19T13:59:37.859152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:37.859843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435864559633705:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.859861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:37.859866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435864559633705:2049] [node 2] ICP22 created new session: [3:7574435864559633802:2049]
2025-11-19T13:59:37.859871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435864559633802:2049] [node 2] ICS09 handshake done sender: [3:7574435864559633772:2112] self: [3:7574435864559633793:0] peer: [2:7574435865168942857:0] socket: 544 qp: -1
2025-11-19T13:59:37.859876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435864559633802:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:37.859883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:37.855964Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435863356639685:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:37.857340Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435863356639686:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:37.858515Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435863356639659:2099] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:37.860182Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435865168942860:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.860197Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435865168942860:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:37.862487Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435865772236185:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.862521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435865772236089:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:37.862879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435865678939365:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:37.863010Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435864559633782:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.863018Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435864559633782:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.863651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435865678939460:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6944
2025-11-19T13:59:37.863864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435863356639607:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.863911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:37.863917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435863356639607:2050] [node 3] ICP22 created new session: [5:7574435863356639700:2050]
2025-11-19T13:59:37.863923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435863356639700:2050] [node 3] ICS09 handshake done sender: [5:7574435863356639685:2117] self: [5:7574435863356639687:0] peer: [3:7574435864559633777:0] socket: 549 qp: -1
2025-11-19T13:59:37.863929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435863356639700:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:37.863936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:37.864072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435863356639605:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.864079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:37.864085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435863356639605:2048] [node 1] ICP22 created new session: [5:7574435863356639702:2048]
2025-11-19T13:59:37.864091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435863356639702:2048] [node 1] ICS09 handshake done sender: [5:7574435863356639686:2118] self: [5:7574435863356639690:0] peer: [1:7574435865337027288:0] socket: 557 qp: -1
2025-11-19T13:59:37.864095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435863356639702:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:37.864101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:37.865240Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435864559633782:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:37.865416Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435865678939447:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:37.865480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435865678939365:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.865494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:37.865499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435865678939365:2050] [node 3] ICP22 created new session: [6:7574435865678939461:2050]
2025-11-19T13:59:37.865504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435865678939461:2050] [node 3] ICS09 handshake done sender: [6:7574435865678939447:2119] self: [6:7574435865678939459:0] peer: [3:7574435864559633781:0] socket: 516 qp: -1
2025-11-19T13:59:37.865509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435865678939461:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:37.865515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:37.866449Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435864559633795:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.866463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435864559633774:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.866474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:37.866479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435864559633774:2109] [node 6] ICP22 created new session: [3:7574435864559633806:2109]
2025-11-19T13:59:37.866483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435864559633806:2109] [node 6] ICS09 handshake done sender: [3:7574435864559633782:2117] self: [3:7574435864559633781:0] peer: [6:7574435865678939459:0] socket: 572 qp: -1
2025-11-19T13:59:37.866487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435864559633806:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:37.866494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:37.866823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435864559633770:2107] [node 5] ICP18 handshake (actor [3:7574435864559633795:2119]) is too old
2025-11-19T13:59:37.872353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435863356639655:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.872381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:37.872388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435863356639655:2096] [node 6] ICP22 created new session: [5:7574435863356639705:2096]
2025-11-19T13:59:37.872394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435863356639705:2096] [node 6] ICS09 handshake done sender: [5:7574435863356639659:2099] self: [5:7574435863356639656:0] peer: [6:7574435865678939446:0] socket: 522 qp: -1
2025-11-19T13:59:37.872400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435863356639705:2096] [node 6] ICS10 traffic start
2025-11-19T13:59:37.872409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:37.873015Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435865168942860:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.873242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435863356639704:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20332
2025-11-19T13:59:37.873375Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435865168942864:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.873384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435865168942864:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:37.874152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435865168942854:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.874169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:37.874175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435865168942854:2107] [node 5] ICP22 created new session: [2:7574435865168942889:2107]
2025-11-19T13:59:37.874180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435865168942889:2107] [node 5] ICS09 handshake done sender: [2:7574435865168942860:2111] self: [2:7574435865168942859:0] peer: [5:7574435863356639696:0] socket: 535 qp: -1
2025-11-19T13:59:37.874185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435865168942889:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:37.874192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:37.874486Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435865678939453:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.874519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435865678939364:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:37.874703Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435863356639684:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:37.874778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435863356639606:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.874791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:37.874795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435863356639606:2049] [node 2] ICP22 created new session: [5:7574435863356639707:2049]
2025-11-19T13:59:37.874800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435863356639707:2049] [node 2] ICS09 handshake done sender: [5:7574435863356639684:2116] self: [5:7574435863356639696:0] peer: [2:7574435865168942859:0] socket: 538 qp: -1
2025-11-19T13:59:37.874805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435863356639707:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:37.874812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:37.876000Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435865168942866:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:37.876011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435865168942866:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:37.876971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435865772236189:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57104
2025-11-19T13:59:37.877052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435865678939464:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6944
2025-11-19T13:59:37.877314Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435865168942864:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:37.877478Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435865772236185:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:37.877538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435865772236089:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.877551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:37.877556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435865772236089:2049] [node 2] ICP22 created new session: [4:7574435865772236190:2049]
2025-11-19T13:59:37.877561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435865772236190:2049] [node 2] ICS09 handshake done sender: [4:7574435865772236185:2121] self: [4:7574435865772236188:0] peer: [2:7574435865168942863:0] socket: 578 qp: -1
2025-11-19T13:59:37.877566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435865772236190:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:37.877574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:37.878560Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435865168942866:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:37.878717Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435865678939453:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:37.878782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435865678939364:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:37.878796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:37.878800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435865678939364:2049] [node 2] ICP22 created new session: [6:7574435865678939465:2049]
2025-11-19T13:59:37.878805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435865678939465:2049] [node 2] ICS09 handshake done sender: [6:7574435865678939453:2121] self: [6:7574435865678939463:0] peer: [2:7574435865168942865:0] socket: 550 qp: -1
2025-11-19T13:59:37.878810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435865678939465:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:37.878817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:37.879731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435865168942855:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.879790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:37.879795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435865168942855:2108] [node 4] ICP22 created new session: [2:7574435865168942893:2108]
2025-11-19T13:59:37.879800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435865168942893:2108] [node 4] ICS09 handshake done sender: [2:7574435865168942864:2113] self: [2:7574435865168942863:0] peer: [4:7574435865772236188:0] socket: 574 qp: -1
2025-11-19T13:59:37.879805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435865168942893:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:37.879811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:37.879933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435865168942856:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:37.879941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:37.879943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435865168942856:2109] [node 6] ICP22 created new session: [2:7574435865168942895:2109]
2025-11-19T13:59:37.879947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435865168942895:2109] [node 6] ICS09 handshake done sender: [2:7574435865168942866:2114] self: [2:7574435865168942865:0] peer: [6:7574435865678939463:0] socket: 541 qp: -1
2025-11-19T13:59:37.879951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435865168942895:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:37.879957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:37.888211Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:37.888299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435865168942880:2048] [node 1] ICS01 socket: 531 reason# EndOfStream
2025-11-19T13:59:37.888307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435865168942791:2048] [node 1] ICP30 unregister session Session# [2:7574435865168942880:2048] VirtualId# [2:7574435865168942876:0]
2025-11-19T13:59:37.888314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435865168942880:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.888352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:37.888374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:37.888820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:37.888891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435864559633790:2048] [node 1] ICS01 socket: 542 reason# EndOfStream
2025-11-19T13:59:37.888897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435864559633704:2048] [node 1] ICP30 unregister session Session# [3:7574435864559633790:2048] VirtualId# [3:7574435864559633786:0]
2025-11-19T13:59:37.888901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435864559633790:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.888930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:37.888948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:37.930103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:37.930197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435865772236173:2048] [node 1] ICS01 socket: 521 reason# EndOfStream
2025-11-19T13:59:37.930206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435865772236088:2048] [node 1] ICP30 unregister session Session# [4:7574435865772236173:2048] VirtualId# [4:7574435865772236171:0]
2025-11-19T13:59:37.930214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435865772236173:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.930292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:37.930325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:37.930384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:37.930468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435863356639702:2048] [node 1] ICS01 socket: 557 reason# EndOfStream
2025-11-19T13:59:37.930476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435863356639605:2048] [node 1] ICP30 unregister session Session# [5:7574435863356639702:2048] VirtualId# [5:7574435863356639690:0]
2025-11-19T13:59:37.930485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435863356639702:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.930533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:37.930557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:37.930594Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:37.930659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435865678939455:2048] [node 1] ICS01 socket: 559 reason# EndOfStream
2025-11-19T13:59:37.930664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435865678939363:2048] [node 1] ICP30 unregister session Session# [6:7574435865678939455:2048] VirtualId# [6:7574435865678939451:0]
2025-11-19T13:59:37.930669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435865678939455:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.930701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:37.930723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:37.932262Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:37.932335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435864559633802:2049] [node 2] ICS01 socket: 544 reason# EndOfStream
2025-11-19T13:59:37.932341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435864559633705:2049] [node 2] ICP30 unregister session Session# [3:7574435864559633802:2049] VirtualId# [3:7574435864559633793:0]
2025-11-19T13:59:37.932347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435864559633802:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.932376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:37.932393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:37.942019Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:37.942111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435863356639707:2049] [node 2] ICS01 socket: 538 reason# EndOfStream
2025-11-19T13:59:37.942119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435863356639606:2049] [node 2] ICP30 unregister session Session# [5:7574435863356639707:2049] VirtualId# [5:7574435863356639696:0]
2025-11-19T13:59:37.942127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435863356639707:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.942188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:37.942212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:37.942253Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:37.942314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435865772236190:2049] [node 2] ICS01 socket: 578 reason# EndOfStream
2025-11-19T13:59:37.942319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435865772236089:2049] [node 2] ICP30 unregister session Session# [4:7574435865772236190:2049] VirtualId# [4:7574435865772236188:0]
2025-11-19T13:59:37.942323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435865772236190:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.942348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:37.942380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:37.942409Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:37.942467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435865678939465:2049] [node 2] ICS01 socket: 550 reason# EndOfStream
2025-11-19T13:59:37.942471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435865678939364:2049] [node 2] ICP30 unregister session Session# [6:7574435865678939465:2049] VirtualId# [6:7574435865678939463:0]
2025-11-19T13:59:37.942475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435865678939465:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.942497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:37.942517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:37.947833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:37.947937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435863356639700:2050] [node 3] ICS01 socket: 549 reason# EndOfStream
2025-11-19T13:59:37.947950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435863356639607:2050] [node 3] ICP30 unregister session Session# [5:7574435863356639700:2050] VirtualId# [5:7574435863356639687:0]
2025-11-19T13:59:37.947959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435863356639700:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.948037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:37.948061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:37.948097Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T13:59:37.948168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435865772236186:2050] [node 3] ICS01 socket: 551 reason# EndOfStream
2025-11-19T13:59:37.948174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435865772236090:2050] [node 3] ICP30 unregister session Session# [4:7574435865772236186:2050] VirtualId# [4:7574435865772236177:0]
2025-11-19T13:59:37.948178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435865772236186:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.948212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:37.948234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:37.948395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:37.948464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435865678939461:2050] [node 3] ICS01 socket: 516 reason# EndOfStream
2025-11-19T13:59:37.948469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435865678939365:2050] [node 3] ICP30 unregister session Session# [6:7574435865678939461:2050] VirtualId# [6:7574435865678939459:0]
2025-11-19T13:59:37.948474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435865678939461:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.948509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:37.948530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:37.970459Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:37.970560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435865678939448:2071] [node 4] ICS01 socket: 528 reason# EndOfStream
2025-11-19T13:59:37.970569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435865678939388:2071] [node 4] ICP30 unregister session Session# [6:7574435865678939448:2071] VirtualId# [6:7574435865678939443:0]
2025-11-19T13:59:37.970577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435865678939448:2071] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.970628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:37.970651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:37.986303Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:59:37.986408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435865678939457:2076] [node 5] ICS01 socket: 527 reason# EndOfStream
2025-11-19T13:59:37.986418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435865678939409:2076] [node 5] ICP30 unregister session Session# [6:7574435865678939457:2076] VirtualId# [6:7574435865678939446:0]
2025-11-19T13:59:37.986425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435865678939457:2076] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:37.986471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:59:37.986496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:38.106840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435868532422047:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:38.106853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435868532422048:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:38.156054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435869097543912:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:38.156068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435869097543913:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:38.212079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435867797484016:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:38.212095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435867797484017:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:38.252098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435868625748306:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:38.252114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435868625748307:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:38.252120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435868625748308:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:38.321292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435867562319009:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:38.321306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435867562319010:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:38.321312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435867562319011:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:38.388064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435869224769457:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:38.388082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435869224769458:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:38.388086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435869224769459:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.461372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435869224769492:2081] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.462250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435867562319070:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:835
2025-11-19T13:59:38.465606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435868625748367:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10759
2025-11-19T13:59:38.464955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435867797484076:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19730
2025-11-19T13:59:38.464250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435868532422114:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:38.461960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435869097543972:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15475
2025-11-19T13:59:38.468109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435869224769508:2086] [node 4] ICP01 ready to work
2025-11-19T13:59:38.468733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435868532422119:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:38.469690Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435869097543972:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.470112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435869097543912:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.472748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435868625748368:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10759
2025-11-19T13:59:38.472907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435869097543975:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.473837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435868625748369:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.472941Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435868532422108:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.472953Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435868532422108:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:38.473742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435868532422124:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:38.473970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435868532422123:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15135
2025-11-19T13:59:38.474043Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435868625748367:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.474469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435868625748306:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:38.474754Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435868532422108:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:38.474070Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435867797484076:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.474496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435867797484016:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:38.474939Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435868532422116:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.474948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435868532422116:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:38.475454Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435868532422111:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.475458Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435868532422111:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:38.476044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435868532422047:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.476059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:38.476066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435868532422047:2048] [node 2] ICP22 created new session: [1:7574435868532422128:2048]
2025-11-19T13:59:38.476072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435868532422128:2048] [node 2] ICS09 handshake done sender: [1:7574435868532422108:2107] self: [1:7574435868532422107:0] peer: [2:7574435869097543973:0] socket: 522 qp: -1
2025-11-19T13:59:38.476081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435868532422128:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:38.476093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:38.476700Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435868532422116:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.476840Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435868532422111:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:38.476844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435867562319071:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:835
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.477249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435868625748371:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10759
2025-11-19T13:59:38.477469Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435868625748367:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:38.477557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435868625748306:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.477573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:38.477580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435868625748306:2048] [node 1] ICP22 created new session: [4:7574435868625748372:2048]
2025-11-19T13:59:38.477586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435868625748372:2048] [node 1] ICS09 handshake done sender: [4:7574435868625748367:2108] self: [4:7574435868625748370:0] peer: [1:7574435868532422115:0] socket: 529 qp: -1
2025-11-19T13:59:38.477592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435868625748372:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:38.477613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:38.480562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435868532422114:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.480582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:38.480589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435868532422114:2109] [node 4] ICP22 created new session: [1:7574435868532422133:2109]
2025-11-19T13:59:38.480595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435868532422133:2109] [node 4] ICS09 handshake done sender: [1:7574435868532422116:2110] self: [1:7574435868532422115:0] peer: [4:7574435868625748370:0] socket: 527 qp: -1
2025-11-19T13:59:38.480602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435868532422133:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:38.480614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:38.480785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435868532422048:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.480793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:38.480797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435868532422048:2049] [node 3] ICP22 created new session: [1:7574435868532422135:2049]
2025-11-19T13:59:38.480801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435868532422135:2049] [node 3] ICS09 handshake done sender: [1:7574435868532422111:2108] self: [1:7574435868532422110:0] peer: [3:7574435867797484077:0] socket: 526 qp: -1
2025-11-19T13:59:38.480805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435868532422135:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:38.480809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.482686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435867797484080:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:38.483153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435867797484079:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19730
2025-11-19T13:59:38.483319Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435867797484076:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:38.485357Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435868532422123:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.485438Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435867562319071:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.490771Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435868532422121:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.490798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435868532422121:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.500058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435867562319073:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:835
2025-11-19T13:59:38.500443Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435867562319070:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.500597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435867562319075:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:38.500978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435867562319075:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:38.501248Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435869224769494:2082] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.501260Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435869224769494:2082] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:38.498043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435867797484085:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:38.498130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435867797484086:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.502095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435867562319077:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:835
2025-11-19T13:59:38.502207Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435869224769494:2082] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:38.502418Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435867562319070:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:38.502485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435867562319075:2111] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.502500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:38.502505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435867562319075:2111] [node 6] ICP22 created new session: [5:7574435867562319078:2111]
2025-11-19T13:59:38.502511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435867562319078:2111] [node 6] ICS09 handshake done sender: [5:7574435867562319070:2108] self: [5:7574435867562319076:0] peer: [6:7574435869224769493:0] socket: 525 qp: -1
2025-11-19T13:59:38.502517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435867562319078:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:38.502537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.499656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435869097543980:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15475
2025-11-19T13:59:38.502428Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435869097543913:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(29463807ns) {Configure:104(29463128ns) {ProcessConfigured:135(29174613ns) {StartInitialHandshake:157(29173778ns) {DropHandshakes:469(694ns) {DropIncomingHandshake:441(98ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(75ns) PrepareNewSessionHandshake:476(970ns) {SwitchToState:178(678ns)}}}}}
2025-11-19T13:59:38.502589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435869097543984:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:38.502670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435869097543985:2112] [node 6] ICP01 ready to work
2025-11-19T13:59:38.510050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435867562319009:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:38.510122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435867562319009:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24620505ns) {Configure:104(24619581ns) {ProcessConfigured:135(24170207ns) {IncomingHandshake:261(24146269ns) {DropIncomingHandshake:441(33ns) PrepareNewSessionHandshake:476(209ns) {SwitchToState:178(124ns)} GenerateSessionVirtualId:988(249ns) IssueIncomingHandshakeReply:193(24142506ns) {SwitchToState:178(743ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.513671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435867562319080:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:835
2025-11-19T13:59:38.513691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435867562319081:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:835
2025-11-19T13:59:38.514339Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435868532422121:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:38.514467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435868532422119:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.514501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:38.514507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435868532422119:2111] [node 5] ICP22 created new session: [1:7574435868532422140:2111]
2025-11-19T13:59:38.514514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435868532422140:2111] [node 5] ICS09 handshake done sender: [1:7574435868532422121:2112] self: [1:7574435868532422120:0] peer: [5:7574435867562319072:0] socket: 540 qp: -1
2025-11-19T13:59:38.514520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435868532422140:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:38.514529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:38.515259Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435867562319071:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:38.515961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435868625748375:2112] [node 6] ICP01 ready to work
2025-11-19T13:59:38.511818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435869097543986:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15475
2025-11-19T13:59:38.512103Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435869097543972:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:38.519800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435867797484016:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.519835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:38.519848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435867797484016:2048] [node 1] ICP22 created new session: [3:7574435867797484096:2048]
2025-11-19T13:59:38.519855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435867797484096:2048] [node 1] ICS09 handshake done sender: [3:7574435867797484076:2107] self: [3:7574435867797484077:0] peer: [1:7574435868532422110:0] socket: 528 qp: -1
2025-11-19T13:59:38.519865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435867797484096:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:38.519889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:38.519609Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435868625748369:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.519804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435867562319009:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.519839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:38.519847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435867562319009:2048] [node 1] ICP22 created new session: [5:7574435867562319086:2048]
2025-11-19T13:59:38.519854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435867562319086:2048] [node 1] ICS09 handshake done sender: [5:7574435867562319071:2109] self: [5:7574435867562319072:0] peer: [1:7574435868532422120:0] socket: 541 qp: -1
2025-11-19T13:59:38.519864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435867562319086:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:38.519888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.526671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435867797484098:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19730
2025-11-19T13:59:38.526717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435867797484099:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19730
2025-11-19T13:59:38.526859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435869224769492:2081] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.526882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:38.526889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435869224769492:2081] [node 5] ICP22 created new session: [6:7574435869224769534:2081]
2025-11-19T13:59:38.526907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435869224769534:2081] [node 5] ICS09 handshake done sender: [6:7574435869224769494:2082] self: [6:7574435869224769493:0] peer: [5:7574435867562319076:0] socket: 524 qp: -1
2025-11-19T13:59:38.526916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435869224769534:2081] [node 5] ICS10 traffic start
2025-11-19T13:59:38.526924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.532325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435869097543912:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.532351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:38.532361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435869097543912:2048] [node 1] ICP22 created new session: [2:7574435869097543993:2048]
2025-11-19T13:59:38.532368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435869097543993:2048] [node 1] ICS09 handshake done sender: [2:7574435869097543972:2107] self: [2:7574435869097543973:0] peer: [1:7574435868532422107:0] socket: 523 qp: -1
2025-11-19T13:59:38.532375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435869097543993:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:38.532397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:38.529131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435867797484016:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(37288980ns) {HandleHandshakeStatus:329(37287822ns) {DropIncomingHandshake:441(1470ns) DropOutgoingHandshake:456(158ns) SwitchToState:178(210ns) ProcessPendingSessionEvents:495(33012ns) {ForwardSessionEventToSession:612(30886ns) ForwardSessionEventToSession:612(822ns)}}}
2025-11-19T13:59:38.532855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435869224769539:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47187
2025-11-19T13:59:38.532900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435869224769540:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47187
2025-11-19T13:59:38.532926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435869224769541:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47187
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.537456Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435867797484098:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.537583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435867797484017:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:38.537983Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435867797484091:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.537987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435867797484091:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:38.539696Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435867797484091:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:38.539889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435867797484085:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.539905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:38.539913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435867797484085:2111] [node 6] ICP22 created new session: [3:7574435867797484102:2111]
2025-11-19T13:59:38.539918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435867797484102:2111] [node 6] ICS09 handshake done sender: [3:7574435867797484091:2114] self: [3:7574435867797484090:0] peer: [6:7574435869224769543:0] socket: 548 qp: -1
2025-11-19T13:59:38.539928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435867797484102:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:38.539936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:38.537772Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435869224769539:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.537817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435869224769457:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:38.537843Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435869224769540:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.537891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435869224769459:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:38.538986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435869224769544:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47187
2025-11-19T13:59:38.541053Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435869224769540:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:38.539831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435868625748307:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:38.540043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435868625748383:2114] [node 5] ICP01 ready to work
2025-11-19T13:59:38.540088Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435868625748368:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.540952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435868625748388:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.541364Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435869097543986:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.541427Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435869097543978:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.541431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435869097543978:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:38.542131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435869097543995:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15475
2025-11-19T13:59:38.542195Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435868532422127:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.542207Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435868532422127:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:38.543527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435867562319088:2116] [node 4] ICP01 ready to work
2025-11-19T13:59:38.543398Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435869097543978:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:38.543729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435869097543983:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.543737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435869097543983:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:38.543850Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435868532422127:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:38.543930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435868532422124:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.543947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:38.543953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435868532422124:2113] [node 6] ICP22 created new session: [1:7574435868532422143:2113]
2025-11-19T13:59:38.543959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435868532422143:2113] [node 6] ICS09 handshake done sender: [1:7574435868532422127:2115] self: [1:7574435868532422126:0] peer: [6:7574435869224769542:0] socket: 543 qp: -1
2025-11-19T13:59:38.543964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435868532422143:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:38.543973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:38.546075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435869224769459:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.546097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:38.546104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435869224769459:2050] [node 3] ICP22 created new session: [6:7574435869224769545:2050]
2025-11-19T13:59:38.546111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435869224769545:2050] [node 3] ICS09 handshake done sender: [6:7574435869224769540:2116] self: [6:7574435869224769543:0] peer: [3:7574435867797484090:0] socket: 564 qp: -1
2025-11-19T13:59:38.546118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435869224769545:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:38.546128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.552307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435869224769547:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47187
2025-11-19T13:59:38.553251Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435867562319073:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.553297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435867562319011:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:38.554356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435869097543975:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.554377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:38.554386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435869097543975:2108] [node 4] ICP22 created new session: [2:7574435869097543998:2108]
2025-11-19T13:59:38.554393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435869097543998:2108] [node 4] ICS09 handshake done sender: [2:7574435869097543978:2109] self: [2:7574435869097543977:0] peer: [4:7574435868625748382:0] socket: 534 qp: -1
2025-11-19T13:59:38.554402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435869097543998:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:38.554418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:38.555649Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435869097543983:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:38.555726Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435867562319080:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.555769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435867562319010:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.569020Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435867797484082:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.569047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435867797484082:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:38.569742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435867797484105:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19730
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.570144Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435869097543988:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.570156Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435869097543988:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:38.570777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435869097543913:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.570796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:38.570804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435869097543913:2049] [node 3] ICP22 created new session: [2:7574435869097544003:2049]
2025-11-19T13:59:38.570810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435869097544003:2049] [node 3] ICS09 handshake done sender: [2:7574435869097543983:2113] self: [2:7574435869097543976:0] peer: [3:7574435867797484100:0] socket: 553 qp: -1
2025-11-19T13:59:38.570818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435869097544003:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:38.570827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:38.572623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435869097544002:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15475
2025-11-19T13:59:38.573816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435869224769457:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435869224769539:2115] held: no
2025-11-19T13:59:38.573949Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435869224769539:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:38.574130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435867797484085:2111] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:38.574295Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435867797484082:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.574415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435869224769457:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.574431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:38.574442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435869224769457:2048] [node 1] ICP22 created new session: [6:7574435869224769550:2048]
2025-11-19T13:59:38.574449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435869224769550:2048] [node 1] ICS09 handshake done sender: [6:7574435869224769539:2115] self: [6:7574435869224769542:0] peer: [1:7574435868532422126:0] socket: 545 qp: -1
2025-11-19T13:59:38.574456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435869224769550:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:38.574462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:38.574573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435869224769549:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47187
2025-11-19T13:59:38.574480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435867797484080:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.574498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:38.574509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435867797484080:2108] [node 5] ICP22 created new session: [3:7574435867797484107:2108]
2025-11-19T13:59:38.574515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435867797484107:2108] [node 5] ICS09 handshake done sender: [3:7574435867797484082:2110] self: [3:7574435867797484081:0] peer: [5:7574435867562319093:0] socket: 546 qp: -1
2025-11-19T13:59:38.574524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435867797484107:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:38.574535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:38.575037Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435869097543988:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:38.575076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435867797484017:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29500) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435867797484098:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-11-19T13:59:38.575161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435867797484109:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19730
2025-11-19T13:59:38.575492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435869097543984:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.575506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:38.575512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435869097543984:2111] [node 5] ICP22 created new session: [2:7574435869097544006:2111]
2025-11-19T13:59:38.575517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435869097544006:2111] [node 5] ICS09 handshake done sender: [2:7574435869097543988:2115] self: [2:7574435869097543987:0] peer: [5:7574435867562319096:0] socket: 554 qp: -1
2025-11-19T13:59:38.575521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435869097544006:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:38.575530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.581868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435867562319097:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:835
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.584050Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435868625748388:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.585163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435867797484110:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19730
2025-11-19T13:59:38.585498Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435869224769549:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.585539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435869224769458:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:38.585352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435867562319098:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:835
2025-11-19T13:59:38.586123Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435867562319073:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:38.586444Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435867797484098:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:38.586624Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435867562319080:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:38.586707Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435867797484099:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.586719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435867797484017:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.586733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:38.586740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435867797484017:2049] [node 2] ICP22 created new session: [3:7574435867797484111:2049]
2025-11-19T13:59:38.586746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435867797484111:2049] [node 2] ICS09 handshake done sender: [3:7574435867797484098:2116] self: [3:7574435867797484100:0] peer: [2:7574435869097543976:0] socket: 560 qp: -1
2025-11-19T13:59:38.586753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435867797484111:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:38.586787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:38.586945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435867797484080:2108] [node 5] ICP18 handshake (actor [3:7574435867797484099:2117]) is too old
2025-11-19T13:59:38.587142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435867562319010:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.587155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:38.587160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435867562319010:2049] [node 2] ICP22 created new session: [5:7574435867562319099:2049]
2025-11-19T13:59:38.587164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435867562319099:2049] [node 2] ICS09 handshake done sender: [5:7574435867562319080:2113] self: [5:7574435867562319096:0] peer: [2:7574435869097543987:0] socket: 556 qp: -1
2025-11-19T13:59:38.587169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435867562319099:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:38.587175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:38.587875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435867562319011:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.587886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:38.587891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435867562319011:2050] [node 3] ICP22 created new session: [5:7574435867562319101:2050]
2025-11-19T13:59:38.587907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435867562319101:2050] [node 3] ICS09 handshake done sender: [5:7574435867562319073:2110] self: [5:7574435867562319093:0] peer: [3:7574435867797484081:0] socket: 549 qp: -1
2025-11-19T13:59:38.587914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435867562319101:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:38.587920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:38.589781Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435869097543990:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.589794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435869097543990:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.593589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435868625748392:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.594294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435869224769553:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47187
2025-11-19T13:59:38.593825Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435869097543984:2111] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:38.593994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435869097544008:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15475
2025-11-19T13:59:38.594624Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435869097544008:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.594676Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435869097543990:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:38.594876Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435869224769549:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:38.594961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435869224769458:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.594980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:38.594989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435869224769458:2049] [node 2] ICP22 created new session: [6:7574435869224769554:2049]
2025-11-19T13:59:38.594996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435869224769554:2049] [node 2] ICS09 handshake done sender: [6:7574435869224769549:2120] self: [6:7574435869224769552:0] peer: [2:7574435869097543989:0] socket: 583 qp: -1
2025-11-19T13:59:38.595003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435869224769554:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:38.595010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:38.596122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435869097543985:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.596138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:38.596144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435869097543985:2112] [node 6] ICP22 created new session: [2:7574435869097544010:2112]
2025-11-19T13:59:38.596149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435869097544010:2112] [node 6] ICS09 handshake done sender: [2:7574435869097543990:2116] self: [2:7574435869097543989:0] peer: [6:7574435869224769552:0] socket: 580 qp: -1
2025-11-19T13:59:38.596154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435869097544010:2112] [node 6] ICS10 traffic start
2025-11-19T13:59:38.596161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:38.596304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435869097543985:2112] [node 6] ICP18 handshake (actor [2:7574435869097544008:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.597313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435868625748393:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10759
2025-11-19T13:59:38.597363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435868625748308:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:38.597644Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435867797484109:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.597678Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435867797484093:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.597681Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435867797484093:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:38.598486Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435867797484093:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:38.598565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435867797484086:2112] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.598595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:38.598601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435867797484086:2112] [node 4] ICP22 created new session: [3:7574435867797484115:2112]
2025-11-19T13:59:38.598606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435867797484115:2112] [node 4] ICS09 handshake done sender: [3:7574435867797484093:2115] self: [3:7574435867797484092:0] peer: [4:7574435868625748394:0] socket: 568 qp: -1
2025-11-19T13:59:38.598611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435867797484115:2112] [node 4] ICS10 traffic start
2025-11-19T13:59:38.598618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:38.599008Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435869224769541:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.599034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435869224769508:2086] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:38.600518Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435868625748377:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.600529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435868625748377:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:38.601178Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435868625748393:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.601256Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435868625748308:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4685) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435868625748388:2118] held: no
2025-11-19T13:59:38.601395Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435869097543995:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.601427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435869097543975:2108] [node 4] ICP18 handshake (actor [2:7574435869097543995:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.609801Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435868625748369:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:38.610017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435868625748395:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.610191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435869224769557:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47187
2025-11-19T13:59:38.611505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435868625748307:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.611525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:38.611534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435868625748307:2049] [node 2] ICP22 created new session: [4:7574435868625748398:2049]
2025-11-19T13:59:38.611541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435868625748398:2049] [node 2] ICS09 handshake done sender: [4:7574435868625748369:2110] self: [4:7574435868625748382:0] peer: [2:7574435869097543977:0] socket: 535 qp: -1
2025-11-19T13:59:38.611548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435868625748398:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:38.611556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:38.611636Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435868625748307:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:38.611996Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435868625748388:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:38.612126Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435868625748377:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:38.612298Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435869224769541:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:38.612411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435869224769508:2086] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.612423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:38.612428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435869224769508:2086] [node 4] ICP22 created new session: [6:7574435869224769558:2086]
2025-11-19T13:59:38.612433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435869224769558:2086] [node 4] ICS09 handshake done sender: [6:7574435869224769541:2117] self: [6:7574435869224769556:0] peer: [4:7574435868625748376:0] socket: 565 qp: -1
2025-11-19T13:59:38.612438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435869224769558:2086] [node 4] ICS10 traffic start
2025-11-19T13:59:38.612445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.614254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435867562319103:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:835
2025-11-19T13:59:38.613639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435868625748308:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.613650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:38.613655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435868625748308:2050] [node 3] ICP22 created new session: [4:7574435868625748401:2050]
2025-11-19T13:59:38.613661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435868625748401:2050] [node 3] ICS09 handshake done sender: [4:7574435868625748388:2118] self: [4:7574435868625748394:0] peer: [3:7574435867797484092:0] socket: 569 qp: -1
2025-11-19T13:59:38.613666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435868625748401:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:38.613673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:38.613835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435868625748375:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.613845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:38.613849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435868625748375:2112] [node 6] ICP22 created new session: [4:7574435868625748403:2112]
2025-11-19T13:59:38.613852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435868625748403:2112] [node 6] ICS09 handshake done sender: [4:7574435868625748377:2113] self: [4:7574435868625748376:0] peer: [6:7574435869224769556:0] socket: 559 qp: -1
2025-11-19T13:59:38.613856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435868625748403:2112] [node 6] ICS10 traffic start
2025-11-19T13:59:38.613861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:38.616147Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435867562319103:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.616182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435867562319088:2116] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:38.616305Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435868625748387:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:38.616312Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435868625748387:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:38.616882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435867562319105:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:835
2025-11-19T13:59:38.617003Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435868625748387:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:38.617142Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435867562319103:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:38.617217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435867562319088:2116] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:38.617229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:38.617233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435867562319088:2116] [node 4] ICP22 created new session: [5:7574435867562319106:2116]
2025-11-19T13:59:38.617238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435867562319106:2116] [node 4] ICS09 handshake done sender: [5:7574435867562319103:2121] self: [5:7574435867562319104:0] peer: [4:7574435868625748386:0] socket: 571 qp: -1
2025-11-19T13:59:38.617242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435867562319106:2116] [node 4] ICS10 traffic start
2025-11-19T13:59:38.617250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:38.618193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435868625748383:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:38.618209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:38.618213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435868625748383:2114] [node 5] ICP22 created new session: [4:7574435868625748406:2114]
2025-11-19T13:59:38.618218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435868625748406:2114] [node 5] ICS09 handshake done sender: [4:7574435868625748387:2117] self: [4:7574435868625748386:0] peer: [5:7574435867562319104:0] socket: 563 qp: -1
2025-11-19T13:59:38.618222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435868625748406:2114] [node 5] ICS10 traffic start
2025-11-19T13:59:38.618229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:38.630950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:38.631038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435869097543993:2048] [node 1] ICS01 socket: 523 reason# EndOfStream
2025-11-19T13:59:38.631048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435869097543912:2048] [node 1] ICP30 unregister session Session# [2:7574435869097543993:2048] VirtualId# [2:7574435869097543973:0]
2025-11-19T13:59:38.631055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435869097543993:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:38.631095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:38.631118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:38.631604Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:38.631673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435867797484096:2048] [node 1] ICS01 socket: 528 reason# EndOfStream
2025-11-19T13:59:38.631678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435867797484016:2048] [node 1] ICP30 unregister session Session# [3:7574435867797484096:2048] VirtualId# [3:7574435867797484077:0]
2025-11-19T13:59:38.631683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435867797484096:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:38.631713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:38.631729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:38.639920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:38.640018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435869224769550:2048] [node 1] ICS01 socket: 545 reason# EndOfStream
2025-11-19T13:59:38.640027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435869224769457:2048] [node 1] ICP30 unregister session Session# [6:7574435869224769550:2048] VirtualId# [6:7574435869224769542:0]
2025-11-19T13:59:38.640034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435869224769550:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:38.640077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:38.640100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:38.642038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:38.642108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435868625748372:2048] [node 1] ICS07 socket disconnect 529 reason# EndOfStream
2025-11-19T13:59:38.642114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435868625748372:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:38.642151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:38.642172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435868625748372:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:38.641531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:38.641612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435867562319086:2048] [node 1] ICS07 socket disconnect 541 reason# EndOfStream
2025-11-19T13:59:38.641618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435867562319086:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:38.641666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:38.641691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435867562319086:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:38.645683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435868625748306:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7995) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:38.645701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435868625748372:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:38.645707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435868625748306:2048] [node 1] ICP30 unregister session Session# [4:7574435868625748372:2048] VirtualId# [4:7574435868625748370:0]
2025-11-19T13:59:38.645726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:38.645731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435868625748306:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:38.645735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:38.645932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435867562319009:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7995) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:38.645945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435867562319086:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:38.645950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435867562319009:2048] [node 1] ICP30 unregister session Session# [5:7574435867562319086:2048] VirtualId# [5:7574435867562319072:0]
2025-11-19T13:59:38.645963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:38.645967Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435867562319009:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:38.645969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:38.654457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435868625748306:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:38.654715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:38.654826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435867797484111:2049] [node 2] ICS01 socket: 560 reason# EndOfStream
2025-11-19T13:59:38.654836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435867797484017:2049] [node 2] ICP30 unregister session Session# [3:7574435867797484111:2049] VirtualId# [3:7574435867797484100:0]
2025-11-19T13:59:38.654844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435867797484111:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:38.654907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:38.654932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:38.654383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435867562319009:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:38.658640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:38.658736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435868625748398:2049] [node 2] ICS01 socket: 535 reason# EndOfStream
2025-11-19T13:59:38.658744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435868625748307:2049] [node 2] ICP30 unregister session Session# [4:7574435868625748398:2049] VirtualId# [4:7574435868625748382:0]
2025-11-19T13:59:38.658750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435868625748398:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:38.658800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:38.658825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:38.662360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:38.662453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435869224769554:2049] [node 2] ICS01 socket: 583 reason# EndOfStream
2025-11-19T13:59:38.662461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435869224769458:2049] [node 2] ICP30 unregister session Session# [6:7574435869224769554:2049] VirtualId# [6:7574435869224769552:0]
2025-11-19T13:59:38.662467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435869224769554:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:38.662516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:38.662539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:38.658927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:38.658998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435867562319099:2049] [node 2] ICS01 socket: 556 reason# EndOfStream
2025-11-19T13:59:38.659003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435867562319010:2049] [node 2] ICP30 unregister session Session# [5:7574435867562319099:2049] VirtualId# [5:7574435867562319096:0]
2025-11-19T13:59:38.659008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435867562319099:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:38.659037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:38.659061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:38.667511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:38.667607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435869224769545:2050] [node 3] ICS07 socket disconnect 564 reason# EndOfStream
2025-11-19T13:59:38.667612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435869224769545:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:38.667662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:38.667683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435869224769545:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:38.668155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435869224769459:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4685) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:38.668168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435869224769545:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:38.668174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435869224769459:2050] [node 3] ICP30 unregister session Session# [6:7574435869224769545:2050] VirtualId# [6:7574435869224769543:0]
2025-11-19T13:59:38.668187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:38.668193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435869224769459:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4685) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:38.668195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4685) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:38.670041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:38.670130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435867562319101:2050] [node 3] ICS01 socket: 549 reason# EndOfStream
2025-11-19T13:59:38.670137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435867562319011:2050] [node 3] ICP30 unregister session Session# [5:7574435867562319101:2050] VirtualId# [5:7574435867562319093:0]
2025-11-19T13:59:38.670144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435867562319101:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:38.670195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:38.670222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:38.682744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435869224769459:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:59:38.682780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:38.682866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435869224769558:2086] [node 4] ICS01 socket: 565 reason# EndOfStream
2025-11-19T13:59:38.682875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435869224769508:2086] [node 4] ICP30 unregister session Session# [6:7574435869224769558:2086] VirtualId# [6:7574435869224769556:0]
2025-11-19T13:59:38.682883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435869224769558:2086] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:38.682958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:38.682982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:38.756262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435866711448588:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:38.756268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435866711448587:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:38.808196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435868941103055:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:38.808210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435868941103056:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:38.847956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435867895555803:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:38.847973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435867895555804:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:38.859892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435866272206780:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:38.859902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435866272206781:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:38.859906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435866272206782:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:38.920173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435866772877728:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:38.920185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435866772877729:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:38.920190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435866772877730:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:38.988386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435866389558594:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:38.988403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435866389558595:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:38.988408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435866389558596:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.008361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435871067845075:2090] [node 6] ICP01 ready to work
2025-11-19T13:59:39.008457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435871067845076:2092] [node 4] ICP01 ready to work
2025-11-19T13:59:39.009385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435870684525920:2071] [node 5] ICP01 ready to work
2025-11-19T13:59:39.009521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435871006415950:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:39.009612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435871006415951:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:39.009690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435871006415952:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:39.008816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435873236070412:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:39.011939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435873236070417:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:39.012205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435871006415946:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31844
2025-11-19T13:59:39.017530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435870567174137:2104] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.018601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872190523159:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8552
2025-11-19T13:59:39.018422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435870684525941:2078] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.023344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435870684525946:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-19T13:59:39.023379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435870684525947:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-19T13:59:39.020086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435873236070421:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:39.025384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435872190523165:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.026826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435870684525920:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17355220ns) {Configure:104(17354332ns) {ProcessConfigured:135(17058461ns) {StartInitialHandshake:157(17058073ns) {DropHandshakes:469(492ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(815ns) {SwitchToState:178(641ns)}}}}}
2025-11-19T13:59:39.026477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435871006415959:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31844
2025-11-19T13:59:39.026515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435871006415960:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31844
2025-11-19T13:59:39.031984Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435871006415946:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.033488Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435866772877728:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21678400ns) {Configure:104(21675680ns) {ProcessConfigured:135(21335611ns) {StartInitialHandshake:157(21334754ns) {DropHandshakes:469(519ns) {DropIncomingHandshake:441(90ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(1721ns) {SwitchToState:178(1022ns)}}}}}
2025-11-19T13:59:39.034895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435873236070417:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(22871865ns) {Configure:104(22871176ns) {ProcessConfigured:135(22617819ns) {StartInitialHandshake:157(22617473ns) {DropHandshakes:469(474ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(313ns) PrepareNewSessionHandshake:476(919ns) {SwitchToState:178(512ns)}}}}}
2025-11-19T13:59:39.035251Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435871006415950:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(22998050ns) {Configure:104(22996898ns) {ProcessConfigured:135(22683877ns) {StartInitialHandshake:157(22683588ns) {DropHandshakes:469(534ns) {DropIncomingHandshake:441(88ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(437ns) PrepareNewSessionHandshake:476(872ns) {SwitchToState:178(29ns)}}}}}
2025-11-19T13:59:39.037283Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435866389558595:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(27486767ns) {Configure:104(24702149ns) {ProcessConfigured:135(24435686ns) {StartInitialHandshake:157(24435364ns) {DropHandshakes:469(464ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(583ns) PrepareNewSessionHandshake:476(920ns) {SwitchToState:178(426ns)}}}}}
2025-11-19T13:59:39.037341Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435870684525946:2093] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.037368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435870684525920:2071] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.037858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435873236070430:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6522
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.033514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435872190523172:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:39.041115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435873236070431:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6522
2025-11-19T13:59:39.041646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435872190523176:2114] [node 6] ICP01 ready to work
2025-11-19T13:59:39.044476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435866272206780:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(36441004ns) {Configure:104(36438142ns) {ProcessConfigured:135(36034678ns) {StartInitialHandshake:157(36033916ns) {DropHandshakes:469(945ns) {DropIncomingHandshake:441(113ns) DropOutgoingHandshake:456(169ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(1168ns) {SwitchToState:178(853ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.047932Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435871067845089:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.047951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435871067845089:2107] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.048740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435871067845101:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-19T13:59:39.048754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435871067845102:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-19T13:59:39.049085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435870684525971:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-19T13:59:39.049130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435870684525972:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-19T13:59:39.052271Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435873236070431:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.050478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435870567174152:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25626
2025-11-19T13:59:39.050501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435870567174153:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25626
2025-11-19T13:59:39.052420Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435871006415960:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.056080Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435873236070430:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.056190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435873236070432:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6522
2025-11-19T13:59:39.055997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435870567174144:2100] [node 5] ICP01 ready to work
2025-11-19T13:59:39.056546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435870567174151:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25626
2025-11-19T13:59:39.056827Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435870684525947:2094] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.056875Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435870684525971:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.056951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435870684525941:2078] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.056963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435866389558594:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.057852Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435870684525946:2093] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:39.057729Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435871067845089:2107] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:39.059888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435870684525920:2071] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.059918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:39.059926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435870684525920:2071] [node 5] ICP22 created new session: [6:7574435870684525975:2071]
2025-11-19T13:59:39.059935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435870684525975:2071] [node 5] ICS09 handshake done sender: [6:7574435870684525946:2093] self: [6:7574435870684525966:0] peer: [5:7574435871067845077:0] socket: 539 qp: -1
2025-11-19T13:59:39.059945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435870684525975:2071] [node 5] ICS10 traffic start
2025-11-19T13:59:39.059971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:39.064023Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435870567174151:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.064239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435871067845075:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.064264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:39.064273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435871067845075:2090] [node 6] ICP22 created new session: [5:7574435871067845104:2090]
2025-11-19T13:59:39.064280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435871067845104:2090] [node 6] ICS09 handshake done sender: [5:7574435871067845089:2107] self: [5:7574435871067845077:0] peer: [6:7574435870684525966:0] socket: 532 qp: -1
2025-11-19T13:59:39.064288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435871067845104:2090] [node 6] ICS10 traffic start
2025-11-19T13:59:39.064299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:39.064985Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435872190523159:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.065274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435866711448588:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(55535763ns) {Configure:104(55535069ns) {ProcessConfigured:135(55212368ns) {StartInitialHandshake:157(55211615ns) {DropHandshakes:469(499ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(89ns)} GenerateSessionVirtualId:988(101ns) PrepareNewSessionHandshake:476(1322ns) {SwitchToState:178(682ns)}}}}}
2025-11-19T13:59:39.065313Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435871006415958:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.065317Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435871006415958:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.065883Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435871006415959:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.066227Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435871067845076:2092] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(56865550ns) {Configure:104(50288468ns) {ProcessConfigured:135(49978940ns) {StartInitialHandshake:157(49977611ns) {DropHandshakes:469(565ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:476(722ns) {SwitchToState:178(188ns)}}}}}
2025-11-19T13:59:39.067862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435871006415969:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31844
2025-11-19T13:59:39.067914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435871006415970:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31844
2025-11-19T13:59:39.068376Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435871006415944:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.068390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435871006415944:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.068768Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435871006415958:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:39.068934Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435870567174139:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.068955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435870567174139:2107] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.068871Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435871067845102:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.068979Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435871067845101:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.069004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435866772877728:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.069014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435866772877729:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.069143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435871006415952:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.069161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:39.069167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435871006415952:2111] [node 6] ICP22 created new session: [1:7574435871006415971:2111]
2025-11-19T13:59:39.069173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435871006415971:2111] [node 6] ICS09 handshake done sender: [1:7574435871006415958:2113] self: [1:7574435871006415957:0] peer: [6:7574435870684525974:0] socket: 551 qp: -1
2025-11-19T13:59:39.069178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435871006415971:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:39.069186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:39.069432Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435870567174153:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.069457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435866272206782:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.072658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435870684525977:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-19T13:59:39.072323Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435870567174144:2100] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16238041ns) {Configure:104(16237331ns) {ProcessConfigured:135(15865605ns) {StartInitialHandshake:157(15864760ns) {DropHandshakes:469(655ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(141ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:476(1440ns) {SwitchToState:178(768ns)}}}}}
2025-11-19T13:59:39.076098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435872190523167:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.076120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435872190523167:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.076203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435868941103055:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.076247Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435868941103055:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20109199ns) {IncomingHandshake:261(20108673ns) {DropIncomingHandshake:441(51ns) PrepareNewSessionHandshake:476(350ns) {SwitchToState:178(109ns)} GenerateSessionVirtualId:988(340ns) IssueIncomingHandshakeReply:193(20103233ns) {SwitchToState:178(815ns)}}}
2025-11-19T13:59:39.076311Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435873236070432:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.080091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435870684525920:2071] [node 5] ICP35 event processing took too much time PendingConnection:278(20174298ns) {HandleHandshakeStatus:329(20173286ns) {DropIncomingHandshake:441(301ns) DropOutgoingHandshake:456(2165ns) SwitchToState:178(341ns) ProcessPendingSessionEvents:495(11819012ns) {ForwardSessionEventToSession:612(11801050ns) ForwardSessionEventToSession:612(2237ns) ForwardSessionEventToSession:612(1733ns) ForwardSessionEventToSession:612(657ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(1001ns) ForwardSessionEventToSession:612(672ns) ForwardSessionEventToSession:612(837ns) ForwardSessionEventToSession:612(1052ns) ForwardSessionEventToSession:612(1972ns)}}}
2025-11-19T13:59:39.080422Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435870684525971:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.084656Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435871006415969:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.084692Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435871006415956:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.084694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435871006415956:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.083919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872190523179:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8552
2025-11-19T13:59:39.087025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435866389558594:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.087051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:39.087061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435866389558594:2048] [node 1] ICP22 created new session: [6:7574435870684525979:2048]
2025-11-19T13:59:39.087067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435870684525979:2048] [node 1] ICS09 handshake done sender: [6:7574435870684525971:2117] self: [6:7574435870684525974:0] peer: [1:7574435871006415957:0] socket: 552 qp: -1
2025-11-19T13:59:39.087075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435870684525979:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:39.087084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:39.087863Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435872190523179:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.088487Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435871006415970:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.088921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435871067845110:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.088032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435873236070435:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6522
2025-11-19T13:59:39.088094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435873236070436:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6522
2025-11-19T13:59:39.089079Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435873236070436:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.089131Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435873236070435:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.089181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435873236070438:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6522
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.092444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435870567174159:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25626
2025-11-19T13:59:39.093959Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435873236070416:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.093974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435873236070416:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.093793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435872190523176:2114] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(52090224ns) {Configure:104(52089325ns) {ProcessConfigured:135(51752386ns) {StartInitialHandshake:157(51751675ns) {DropHandshakes:469(1052ns) {DropIncomingHandshake:441(417ns) DropOutgoingHandshake:456(115ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:476(417ns) {SwitchToState:178(357ns)}}}}}
2025-11-19T13:59:39.094503Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435872190523167:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:39.094212Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435870567174139:2107] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:39.094637Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435870567174152:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.094663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435866272206781:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.094811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435872190523165:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.094823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:39.094830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435872190523165:2108] [node 4] ICP22 created new session: [3:7574435872190523183:2108]
2025-11-19T13:59:39.094835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435872190523183:2108] [node 4] ICS09 handshake done sender: [3:7574435872190523167:2111] self: [3:7574435872190523166:0] peer: [4:7574435870567174157:0] socket: 548 qp: -1
2025-11-19T13:59:39.094841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435872190523183:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:39.094851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:39.094894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435870567174137:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.094921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:39.094924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435870567174137:2104] [node 6] ICP22 created new session: [4:7574435870567174162:2104]
2025-11-19T13:59:39.094928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435870567174162:2104] [node 6] ICS09 handshake done sender: [4:7574435870567174139:2107] self: [4:7574435870567174138:0] peer: [6:7574435870684525973:0] socket: 534 qp: -1
2025-11-19T13:59:39.094931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435870567174162:2104] [node 6] ICS10 traffic start
2025-11-19T13:59:39.094937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:39.092929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435870684525981:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-19T13:59:39.094443Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435870684525947:2094] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:39.095973Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435871006415944:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:39.096063Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435871006415956:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:39.101118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435870684525941:2078] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.101145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:39.101156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435870684525941:2078] [node 4] ICP22 created new session: [6:7574435870684525982:2078]
2025-11-19T13:59:39.101165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435870684525982:2078] [node 4] ICS09 handshake done sender: [6:7574435870684525947:2094] self: [6:7574435870684525973:0] peer: [4:7574435870567174138:0] socket: 540 qp: -1
2025-11-19T13:59:39.101173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435870684525982:2078] [node 4] ICS10 traffic start
2025-11-19T13:59:39.101186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:39.099959Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435871067845102:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:39.102466Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435873236070424:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.102486Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435873236070424:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.104453Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435873236070430:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:39.105486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435868941103055:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.105501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:39.105509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435868941103055:2048] [node 1] ICP22 created new session: [2:7574435873236070442:2048]
2025-11-19T13:59:39.105516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435873236070442:2048] [node 1] ICS09 handshake done sender: [2:7574435873236070430:2115] self: [2:7574435873236070434:0] peer: [1:7574435871006415943:0] socket: 531 qp: -1
2025-11-19T13:59:39.105523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435873236070442:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:39.105555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.109334Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435873236070416:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:39.109489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435873236070412:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.109508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:39.109514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435873236070412:2107] [node 5] ICP22 created new session: [2:7574435873236070444:2107]
2025-11-19T13:59:39.109519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435873236070444:2107] [node 5] ICS09 handshake done sender: [2:7574435873236070416:2109] self: [2:7574435873236070415:0] peer: [5:7574435871067845109:0] socket: 541 qp: -1
2025-11-19T13:59:39.109524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435873236070444:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:39.109532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:39.106775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435870567174164:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25626
2025-11-19T13:59:39.107000Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435870567174153:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:39.107068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435866272206782:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.107079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:39.107083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435866272206782:2050] [node 3] ICP22 created new session: [4:7574435870567174166:2050]
2025-11-19T13:59:39.107088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435870567174166:2050] [node 3] ICS09 handshake done sender: [4:7574435870567174153:2116] self: [4:7574435870567174157:0] peer: [3:7574435872190523166:0] socket: 563 qp: -1
2025-11-19T13:59:39.107093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435870567174166:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:39.107100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:39.112174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435866772877728:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.112197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:39.112209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435866772877728:2048] [node 1] ICP22 created new session: [5:7574435871067845114:2048]
2025-11-19T13:59:39.112215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435871067845114:2048] [node 1] ICS09 handshake done sender: [5:7574435871067845102:2115] self: [5:7574435871067845108:0] peer: [1:7574435871006415955:0] socket: 559 qp: -1
2025-11-19T13:59:39.112221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435871067845114:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:39.112228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:39.115890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435866711448587:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.115922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:39.115931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435866711448587:2048] [node 2] ICP22 created new session: [1:7574435871006415978:2048]
2025-11-19T13:59:39.115941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435871006415978:2048] [node 2] ICS09 handshake done sender: [1:7574435871006415944:2107] self: [1:7574435871006415943:0] peer: [2:7574435873236070434:0] socket: 528 qp: -1
2025-11-19T13:59:39.115947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435871006415978:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:39.115971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:39.116142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435871006415951:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.116153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:39.116157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435871006415951:2110] [node 5] ICP22 created new session: [1:7574435871006415980:2110]
2025-11-19T13:59:39.116161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435871006415980:2110] [node 5] ICS09 handshake done sender: [1:7574435871006415956:2112] self: [1:7574435871006415955:0] peer: [5:7574435871067845108:0] socket: 550 qp: -1
2025-11-19T13:59:39.116165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435871006415980:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:39.116171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:39.116824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435871006415952:2111] [node 6] ICP35 event processing took too much time PendingConnection:278(47644474ns) {HandleHandshakeStatus:329(47643766ns) {DropIncomingHandshake:441(3252ns) DropOutgoingHandshake:456(148ns) SwitchToState:178(29ns) ProcessPendingSessionEvents:495(37459170ns) {ForwardSessionEventToSession:612(37448453ns) ForwardSessionEventToSession:612(946ns) ForwardSessionEventToSession:612(914ns) ForwardSessionEventToSession:612(807ns) ForwardSessionEventToSession:612(526ns) ForwardSessionEventToSession:612(956ns) ForwardSessionEventToSession:612(787ns) ForwardSessionEventToSession:612(578ns) ForwardSessionEventToSession:612(670ns) ForwardSessionEventToSession:612(968ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.122637Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435872190523165:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(27789860ns) {HandleHandshakeStatus:329(27788732ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(302ns) SwitchToState:178(47ns) ProcessPendingSessionEvents:495(20027961ns) {ForwardSessionEventToSession:612(20014624ns) ForwardSessionEventToSession:612(1326ns) ForwardSessionEventToSession:612(1037ns) ForwardSessionEventToSession:612(611ns) ForwardSessionEventToSession:612(592ns) ForwardSessionEventToSession:612(1104ns) ForwardSessionEventToSession:612(695ns) ForwardSessionEventToSession:612(566ns) ForwardSessionEventToSession:612(524ns) ForwardSessionEventToSession:612(998ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.123010Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435873236070424:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:39.123052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435870567174137:2104] [node 6] ICP35 event processing took too much time PendingConnection:278(28133726ns) {HandleHandshakeStatus:329(28133467ns) {DropIncomingHandshake:441(2634ns) DropOutgoingHandshake:456(68ns) SwitchToState:178(13782ns) ProcessPendingSessionEvents:495(20425857ns) {ForwardSessionEventToSession:612(20419451ns) ForwardSessionEventToSession:612(1059ns) ForwardSessionEventToSession:612(1030ns) ForwardSessionEventToSession:612(614ns) ForwardSessionEventToSession:612(547ns) ForwardSessionEventToSession:612(743ns)}}}
2025-11-19T13:59:39.123149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435873236070421:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.123170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:39.123176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435873236070421:2110] [node 4] ICP22 created new session: [2:7574435873236070449:2110]
2025-11-19T13:59:39.123182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435873236070449:2110] [node 4] ICS09 handshake done sender: [2:7574435873236070424:2112] self: [2:7574435873236070423:0] peer: [4:7574435870567174161:0] socket: 543 qp: -1
2025-11-19T13:59:39.123187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435873236070449:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:39.123195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:39.123727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435866272206781:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31257) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435870567174152:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.122374Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435866772877729:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31257) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435871067845101:2114] held: no
2025-11-19T13:59:39.122537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435871067845111:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-19T13:59:39.122582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435871067845112:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-19T13:59:39.123433Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435871067845111:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.123455Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435871067845101:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:39.124216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872190523186:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8552
2025-11-19T13:59:39.124359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872190523187:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8552
2025-11-19T13:59:39.126749Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435872190523187:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.126785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435867895555804:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.129300Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435870567174158:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.129320Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435870567174158:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.126950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435871067845113:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-19T13:59:39.127095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435866772877729:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.127107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:39.127115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435866772877729:2049] [node 2] ICP22 created new session: [5:7574435871067845117:2049]
2025-11-19T13:59:39.127121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435871067845117:2049] [node 2] ICS09 handshake done sender: [5:7574435871067845101:2114] self: [5:7574435871067845109:0] peer: [2:7574435873236070415:0] socket: 544 qp: -1
2025-11-19T13:59:39.127127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435871067845117:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:39.127134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:39.127237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435866772877730:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.127502Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435871067845113:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.128926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435871067845076:2092] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.131169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872190523189:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8552
2025-11-19T13:59:39.130122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435870567174169:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25626
2025-11-19T13:59:39.132827Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435873236070426:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.132847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435873236070426:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.132989Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435872190523174:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.133001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435872190523174:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.136406Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435872190523189:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.139845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435870567174170:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25626
2025-11-19T13:59:39.139918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435870567174159:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.139950Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435870567174152:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.141797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435870684525984:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-19T13:59:39.142026Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435873236070426:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:39.142014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435866272206780:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.142055Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435870567174170:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.142219Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435871006415965:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.142236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435871006415965:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.142143Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435870567174158:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:39.142634Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435870684525984:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.142668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435866389558595:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.142503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435868941103056:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.142524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:39.142530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435868941103056:2049] [node 3] ICP22 created new session: [2:7574435873236070452:2049]
2025-11-19T13:59:39.142536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435873236070452:2049] [node 3] ICS09 handshake done sender: [2:7574435873236070426:2113] self: [2:7574435873236070425:0] peer: [3:7574435872190523188:0] socket: 542 qp: -1
2025-11-19T13:59:39.142542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435873236070452:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:39.142549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:39.142918Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435872190523174:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:39.144781Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435871006415963:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.144794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435871006415963:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.146502Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435871006415963:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:39.146605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435871006415950:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.146623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:39.146630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435871006415950:2109] [node 4] ICP22 created new session: [1:7574435871006415984:2109]
2025-11-19T13:59:39.146636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435871006415984:2109] [node 4] ICS09 handshake done sender: [1:7574435871006415963:2116] self: [1:7574435871006415954:0] peer: [4:7574435870567174172:0] socket: 572 qp: -1
2025-11-19T13:59:39.146641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435871006415984:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:39.146648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:39.143955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435867895555803:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.143987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435872190523172:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.144004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:39.144012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435872190523172:2112] [node 5] ICP22 created new session: [3:7574435872190523192:2112]
2025-11-19T13:59:39.144017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435872190523192:2112] [node 5] ICS09 handshake done sender: [3:7574435872190523174:2113] self: [3:7574435872190523173:0] peer: [5:7574435871067845119:0] socket: 580 qp: -1
2025-11-19T13:59:39.144024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435872190523192:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:39.144046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.145779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435870567174144:2100] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.145795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:39.145803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435870567174144:2100] [node 5] ICP22 created new session: [4:7574435870567174173:2100]
2025-11-19T13:59:39.145810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435870567174173:2100] [node 5] ICS09 handshake done sender: [4:7574435870567174158:2117] self: [4:7574435870567174155:0] peer: [5:7574435871067845120:0] socket: 553 qp: -1
2025-11-19T13:59:39.145816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435870567174173:2100] [node 5] ICS10 traffic start
2025-11-19T13:59:39.145825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:39.145903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435870567174144:2100] [node 5] ICP18 handshake (actor [4:7574435870567174170:2121]) is too old
2025-11-19T13:59:39.145946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435866272206781:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.145951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:39.145955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435866272206781:2049] [node 2] ICP22 created new session: [4:7574435870567174175:2049]
2025-11-19T13:59:39.145960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435870567174175:2049] [node 2] ICS09 handshake done sender: [4:7574435870567174152:2115] self: [4:7574435870567174161:0] peer: [2:7574435873236070423:0] socket: 562 qp: -1
2025-11-19T13:59:39.145964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435870567174175:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:39.145969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.150541Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435871006415965:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:39.150394Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435873236070429:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.150406Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435873236070429:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.150669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435866711448588:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.150689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:39.150695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435866711448588:2049] [node 3] ICP22 created new session: [1:7574435871006415987:2049]
2025-11-19T13:59:39.150700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435871006415987:2049] [node 3] ICS09 handshake done sender: [1:7574435871006415965:2117] self: [1:7574435871006415953:0] peer: [3:7574435872190523190:0] socket: 565 qp: -1
2025-11-19T13:59:39.150705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435871006415987:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:39.150714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:39.148100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872190523194:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8552
2025-11-19T13:59:39.149373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435871067845122:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
2025-11-19T13:59:39.149521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435871067845121:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25154
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.154115Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435866272206780:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25724) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435870567174159:2118] held: no
2025-11-19T13:59:39.154234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435870567174177:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25626
2025-11-19T13:59:39.155700Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435870567174159:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:39.155840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435866272206780:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.155855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:39.155864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435866272206780:2048] [node 1] ICP22 created new session: [4:7574435870567174178:2048]
2025-11-19T13:59:39.155872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435870567174178:2048] [node 1] ICS09 handshake done sender: [4:7574435870567174159:2118] self: [4:7574435870567174172:0] peer: [1:7574435871006415954:0] socket: 575 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.161592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435870684525986:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-19T13:59:39.155879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435870567174178:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:39.155899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.162381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435870684525987:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-19T13:59:39.162537Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435873236070429:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:39.162756Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435870684525984:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:39.162865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435866389558595:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.162886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:39.162892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435866389558595:2049] [node 2] ICP22 created new session: [6:7574435870684525988:2049]
2025-11-19T13:59:39.162898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435870684525988:2049] [node 2] ICS09 handshake done sender: [6:7574435870684525984:2121] self: [6:7574435870684525985:0] peer: [2:7574435873236070418:0] socket: 585 qp: -1
2025-11-19T13:59:39.162918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435870684525988:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:39.162927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:39.164510Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435867895555804:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31257) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435872190523187:2118] held: no
2025-11-19T13:59:39.165779Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435872190523187:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.166269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435873236070417:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.166294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:39.166300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435873236070417:2108] [node 6] ICP22 created new session: [2:7574435873236070455:2108]
2025-11-19T13:59:39.166306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435873236070455:2108] [node 6] ICS09 handshake done sender: [2:7574435873236070429:2114] self: [2:7574435873236070418:0] peer: [6:7574435870684525985:0] socket: 568 qp: -1
2025-11-19T13:59:39.166312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435873236070455:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:39.166323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:39.166188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872190523196:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8552
2025-11-19T13:59:39.167942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435866711448588:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(17227404ns) {HandleHandshakeStatus:329(17226463ns) {DropIncomingHandshake:441(3415ns) DropOutgoingHandshake:456(222ns) SwitchToState:178(101ns) ProcessPendingSessionEvents:495(17148650ns) {ForwardSessionEventToSession:612(17130309ns) ForwardSessionEventToSession:612(2223ns) ForwardSessionEventToSession:612(827ns) ForwardSessionEventToSession:612(1174ns) ForwardSessionEventToSession:612(731ns) ForwardSessionEventToSession:612(1628ns) ForwardSessionEventToSession:612(766ns) ForwardSessionEventToSession:612(494ns) ForwardSessionEventToSession:612(784ns) ForwardSessionEventToSession:612(649ns)}}}
2025-11-19T13:59:39.168678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435866772877730:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26657) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435871067845111:2118] held: no
2025-11-19T13:59:39.168715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435871067845076:2092] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6756) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435871067845113:2120] held: no
2025-11-19T13:59:39.168736Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435871067845111:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:39.169953Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435867895555803:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25724) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435872190523189:2119] held: no
2025-11-19T13:59:39.169980Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435872190523189:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:39.170100Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435870684525986:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.170136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435866389558596:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.187844Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435871067845113:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:39.189571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435866772877730:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.189582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:39.189589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435866772877730:2050] [node 3] ICP22 created new session: [5:7574435871067845123:2050]
2025-11-19T13:59:39.189594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435871067845123:2050] [node 3] ICS09 handshake done sender: [5:7574435871067845111:2118] self: [5:7574435871067845119:0] peer: [3:7574435872190523173:0] socket: 581 qp: -1
2025-11-19T13:59:39.189600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435871067845123:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:39.189607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:39.189705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435871067845076:2092] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.189711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:39.189714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435871067845076:2092] [node 4] ICP22 created new session: [5:7574435871067845125:2092]
2025-11-19T13:59:39.189719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435871067845125:2092] [node 4] ICS09 handshake done sender: [5:7574435871067845113:2120] self: [5:7574435871067845120:0] peer: [4:7574435870567174155:0] socket: 584 qp: -1
2025-11-19T13:59:39.189723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435871067845125:2092] [node 4] ICS10 traffic start
2025-11-19T13:59:39.189728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:39.187994Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435872190523181:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.188009Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435872190523181:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.188600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435867895555803:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.188615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:39.188623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435867895555803:2048] [node 1] ICP22 created new session: [3:7574435872190523197:2048]
2025-11-19T13:59:39.188629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435872190523197:2048] [node 1] ICS09 handshake done sender: [3:7574435872190523189:2119] self: [3:7574435872190523190:0] peer: [1:7574435871006415953:0] socket: 586 qp: -1
2025-11-19T13:59:39.188639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435872190523197:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:39.188649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:39.188801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435867895555804:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.188808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:39.188845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435867895555804:2049] [node 2] ICP22 created new session: [3:7574435872190523199:2049]
2025-11-19T13:59:39.188849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435872190523199:2049] [node 2] ICS09 handshake done sender: [3:7574435872190523187:2118] self: [3:7574435872190523188:0] peer: [2:7574435873236070425:0] socket: 556 qp: -1
2025-11-19T13:59:39.188855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435872190523199:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:39.188861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.192453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435870684525991:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15699
2025-11-19T13:59:39.192134Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435872190523181:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:39.192903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435872190523176:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.192923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:39.192929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435872190523176:2114] [node 6] ICP22 created new session: [3:7574435872190523202:2114]
2025-11-19T13:59:39.192933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435872190523202:2114] [node 6] ICS09 handshake done sender: [3:7574435872190523181:2116] self: [3:7574435872190523177:0] peer: [6:7574435870684525990:0] socket: 567 qp: -1
2025-11-19T13:59:39.192939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435872190523202:2114] [node 6] ICS10 traffic start
2025-11-19T13:59:39.192946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:39.193347Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435870684525986:2122] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:39.193461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435866389558596:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.193478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:39.193483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435866389558596:2050] [node 3] ICP22 created new session: [6:7574435870684525992:2050]
2025-11-19T13:59:39.193487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435870684525992:2050] [node 3] ICS09 handshake done sender: [6:7574435870684525986:2122] self: [6:7574435870684525990:0] peer: [3:7574435872190523177:0] socket: 582 qp: -1
2025-11-19T13:59:39.193492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435870684525992:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:39.193498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:39.207940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:39.208035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435873236070442:2048] [node 1] ICS01 socket: 531 reason# EndOfStream
2025-11-19T13:59:39.208044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435868941103055:2048] [node 1] ICP30 unregister session Session# [2:7574435873236070442:2048] VirtualId# [2:7574435873236070434:0]
2025-11-19T13:59:39.208053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435873236070442:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.208093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:39.208118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:39.208151Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:39.208213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435872190523197:2048] [node 1] ICS01 socket: 586 reason# EndOfStream
2025-11-19T13:59:39.208218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435867895555803:2048] [node 1] ICP30 unregister session Session# [3:7574435872190523197:2048] VirtualId# [3:7574435872190523190:0]
2025-11-19T13:59:39.208222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435872190523197:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.208250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:39.208272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:39.211932Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:39.212022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435870567174178:2048] [node 1] ICS01 socket: 575 reason# EndOfStream
2025-11-19T13:59:39.212032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435866272206780:2048] [node 1] ICP30 unregister session Session# [4:7574435870567174178:2048] VirtualId# [4:7574435870567174172:0]
2025-11-19T13:59:39.212039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435870567174178:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.212094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:39.212117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:39.212152Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:39.212211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435871067845114:2048] [node 1] ICS01 socket: 559 reason# EndOfStream
2025-11-19T13:59:39.212215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435866772877728:2048] [node 1] ICP30 unregister session Session# [5:7574435871067845114:2048] VirtualId# [5:7574435871067845108:0]
2025-11-19T13:59:39.212219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435871067845114:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.212244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:39.212262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:39.212288Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:39.212347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435870684525979:2048] [node 1] ICS01 socket: 552 reason# EndOfStream
2025-11-19T13:59:39.212352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435866389558594:2048] [node 1] ICP30 unregister session Session# [6:7574435870684525979:2048] VirtualId# [6:7574435870684525974:0]
2025-11-19T13:59:39.212356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435870684525979:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.212380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:39.212400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:39.220191Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:39.220278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435872190523199:2049] [node 2] ICS01 socket: 556 reason# EndOfStream
2025-11-19T13:59:39.220287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435867895555804:2049] [node 2] ICP30 unregister session Session# [3:7574435872190523199:2049] VirtualId# [3:7574435872190523188:0]
2025-11-19T13:59:39.220294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435872190523199:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.220329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:39.220346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:39.243456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:39.243556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435870684525988:2049] [node 2] ICS01 socket: 585 reason# EndOfStream
2025-11-19T13:59:39.243567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435866389558595:2049] [node 2] ICP30 unregister session Session# [6:7574435870684525988:2049] VirtualId# [6:7574435870684525985:0]
2025-11-19T13:59:39.243576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435870684525988:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.243640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:39.243666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:39.243783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:39.243849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435871067845117:2049] [node 2] ICS01 socket: 544 reason# EndOfStream
2025-11-19T13:59:39.243854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435866772877729:2049] [node 2] ICP30 unregister session Session# [5:7574435871067845117:2049] VirtualId# [5:7574435871067845109:0]
2025-11-19T13:59:39.243858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435871067845117:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.243891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:39.243914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:39.243950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:39.244025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435870567174175:2049] [node 2] ICS01 socket: 562 reason# EndOfStream
2025-11-19T13:59:39.244030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435866272206781:2049] [node 2] ICP30 unregister session Session# [4:7574435870567174175:2049] VirtualId# [4:7574435870567174161:0]
2025-11-19T13:59:39.244034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435870567174175:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.244072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:39.244095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:39.255893Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:39.255983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435871067845123:2050] [node 3] ICS01 socket: 581 reason# EndOfStream
2025-11-19T13:59:39.255994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435866772877730:2050] [node 3] ICP30 unregister session Session# [5:7574435871067845123:2050] VirtualId# [5:7574435871067845119:0]
2025-11-19T13:59:39.256000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435871067845123:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.256057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:39.256077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:39.256110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:39.256170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435870684525992:2050] [node 3] ICS01 socket: 582 reason# EndOfStream
2025-11-19T13:59:39.256175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435866389558596:2050] [node 3] ICP30 unregister session Session# [6:7574435870684525992:2050] VirtualId# [6:7574435870684525990:0]
2025-11-19T13:59:39.256180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435870684525992:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.256204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:39.256227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:39.274706Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:39.274798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435870684525982:2078] [node 4] ICS01 socket: 540 reason# EndOfStream
2025-11-19T13:59:39.274807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435870684525941:2078] [node 4] ICP30 unregister session Session# [6:7574435870684525982:2078] VirtualId# [6:7574435870684525973:0]
2025-11-19T13:59:39.274814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435870684525982:2078] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.274867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:39.274892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:39.528724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435873293016882:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:39.528743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435873293016883:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:39.564082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435871319159176:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:39.564096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435871319159177:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:39.595950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435872899226168:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:39.595966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435872899226169:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:39.637630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435872410545902:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:39.637646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435872410545903:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:39.637656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435872410545904:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:39.663378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435872644050480:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:39.663392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435872644050481:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:39.663397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435872644050482:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:39.724010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435874401675722:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:39.724041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435874401675723:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:39.724060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435874401675724:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:39.762433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435873293016944:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:39.762522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435873293016945:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.763717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435871319159241:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:283
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.763601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435874401675757:2081] [node 5] ICP01 ready to work
2025-11-19T13:59:39.764161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435874401675758:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31028
2025-11-19T13:59:39.764370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435872644050536:2102] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.765659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435873293016955:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:39.766802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435871319159244:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:39.766892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435871319159245:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:39.767037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435871319159246:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:283
2025-11-19T13:59:39.768229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435871319159248:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.782552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435872899226232:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:39.778465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435874401675795:2102] [node 4] ICP01 ready to work
2025-11-19T13:59:39.778989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435874401675786:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31028
2025-11-19T13:59:39.783619Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435874401675758:2082] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.783678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435874401675722:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.784731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435872899226235:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:39.786165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872899226236:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34807
2025-11-19T13:59:39.786181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872899226237:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34807
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.788180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435871319159244:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19590016ns) {Configure:104(19588932ns) {ProcessConfigured:135(19240940ns) {StartInitialHandshake:157(19240118ns) {DropHandshakes:469(776ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(275ns) PrepareNewSessionHandshake:476(1278ns) {SwitchToState:178(189ns)}}}}}
2025-11-19T13:59:39.788365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435871319159254:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:283
2025-11-19T13:59:39.790248Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435874401675786:2103] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.790291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435874401675757:2081] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.790482Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435873293016950:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.790499Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435873293016950:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.792759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435872644050547:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22292
2025-11-19T13:59:39.792820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435872644050548:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22292
2025-11-19T13:59:39.793958Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435872644050547:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.793983Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435872644050548:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.794295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435872644050480:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.794409Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435873293016954:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.794419Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435873293016954:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.795213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435872644050551:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22292
2025-11-19T13:59:39.795307Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435873293016954:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:39.795502Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435872644050548:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:39.795572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435873293016944:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.795587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:39.795592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435873293016944:2107] [node 5] ICP22 created new session: [1:7574435873293016962:2107]
2025-11-19T13:59:39.797192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435873293016962:2107] [node 5] ICS09 handshake done sender: [1:7574435873293016954:2112] self: [1:7574435873293016953:0] peer: [5:7574435872644050550:0] socket: 548 qp: -1
2025-11-19T13:59:39.797207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435873293016962:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:39.797224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:39.797477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435872644050480:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.797492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:39.797497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435872644050480:2048] [node 1] ICP22 created new session: [5:7574435872644050552:2048]
2025-11-19T13:59:39.797501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435872644050552:2048] [node 1] ICS09 handshake done sender: [5:7574435872644050548:2111] self: [5:7574435872644050550:0] peer: [1:7574435873293016953:0] socket: 557 qp: -1
2025-11-19T13:59:39.797506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435872644050552:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:39.797524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:39.799453Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435872644050538:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.799469Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435872644050538:2103] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.803142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435872410545968:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1340
2025-11-19T13:59:39.803171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435872410545969:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1340
2025-11-19T13:59:39.803672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435871319159245:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(34743725ns) {Configure:104(34742474ns) {ProcessConfigured:135(34526492ns) {StartInitialHandshake:157(34525932ns) {DropHandshakes:469(139ns) {DropIncomingHandshake:441(25ns) DropOutgoingHandshake:456(23ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(954ns) {SwitchToState:178(757ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.804875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435873293016965:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7206
2025-11-19T13:59:39.806072Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435873293016950:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:39.806211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435873293016945:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.806225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:39.806232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435873293016945:2108] [node 6] ICP22 created new session: [1:7574435873293016967:2108]
2025-11-19T13:59:39.806238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435873293016967:2108] [node 6] ICS09 handshake done sender: [1:7574435873293016950:2110] self: [1:7574435873293016949:0] peer: [6:7574435874401675798:0] socket: 536 qp: -1
2025-11-19T13:59:39.806245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435873293016967:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:39.806252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:39.804145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435872644050558:2114] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.807954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435874401675805:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31028
2025-11-19T13:59:39.808043Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435872899226232:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25415552ns) {Configure:104(25413973ns) {ProcessConfigured:135(25115157ns) {StartInitialHandshake:157(25114482ns) {DropHandshakes:469(682ns) {DropIncomingHandshake:441(177ns) DropOutgoingHandshake:456(208ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:476(1432ns) {SwitchToState:178(836ns)}}}}}
2025-11-19T13:59:39.808111Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435872899226236:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.808897Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435871319159254:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.808936Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435871319159241:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.810350Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435874401675758:2082] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:39.810574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435874401675722:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.810597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:39.810607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435874401675722:2048] [node 1] ICP22 created new session: [6:7574435874401675806:2048]
2025-11-19T13:59:39.810617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435874401675806:2048] [node 1] ICS09 handshake done sender: [6:7574435874401675758:2082] self: [6:7574435874401675798:0] peer: [1:7574435873293016949:0] socket: 538 qp: -1
2025-11-19T13:59:39.810624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435874401675806:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:39.810644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:39.809201Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435872410545969:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.811956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435872644050481:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19670661ns) {Configure:104(19669739ns) {ProcessConfigured:135(19331070ns) {StartInitialHandshake:157(19330541ns) {DropHandshakes:469(831ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(150ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(1231ns) {SwitchToState:178(349ns)}}}}}
2025-11-19T13:59:39.812568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435872410545965:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:39.812755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435872899226243:2113] [node 5] ICP01 ready to work
2025-11-19T13:59:39.816028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435874401675804:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31028
2025-11-19T13:59:39.816591Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435871319159246:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.816711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435871319159176:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.816829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435871319159248:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(47694885ns) {Configure:104(47693822ns) {ProcessConfigured:135(47483717ns) {StartInitialHandshake:157(47483226ns) {DropHandshakes:469(275ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(23ns)} GenerateSessionVirtualId:988(55ns) PrepareNewSessionHandshake:476(604ns) {SwitchToState:178(357ns)}}}}}
2025-11-19T13:59:39.817013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435872410545902:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17936032ns) {Configure:104(17935351ns) {ProcessConfigured:135(17588389ns) {StartInitialHandshake:157(17587996ns) {DropHandshakes:469(711ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(128ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:476(1103ns) {SwitchToState:178(40ns)}}}}}
2025-11-19T13:59:39.817072Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435872410545968:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.817093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435872410545902:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.818301Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435873293016947:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.818314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435873293016947:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.819533Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435873293016947:2109] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.819863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435873293016882:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.819884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:39.819894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435873293016882:2048] [node 2] ICP22 created new session: [1:7574435873293016971:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.819900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435873293016971:2048] [node 2] ICS09 handshake done sender: [1:7574435873293016947:2109] self: [1:7574435873293016946:0] peer: [2:7574435871319159258:0] socket: 534 qp: -1
2025-11-19T13:59:39.819910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435873293016971:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:39.819926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:39.820113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435871319159261:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:283
2025-11-19T13:59:39.820134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435871319159262:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:283
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.820293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435874401675809:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31028
2025-11-19T13:59:39.820958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435872410545966:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:39.821149Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435871319159241:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:39.821145Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435873293016965:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.821240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435871319159176:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.821254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:39.821261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435871319159176:2048] [node 1] ICP22 created new session: [2:7574435871319159263:2048]
2025-11-19T13:59:39.821267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435871319159263:2048] [node 1] ICS09 handshake done sender: [2:7574435871319159241:2109] self: [2:7574435871319159258:0] peer: [1:7574435873293016946:0] socket: 537 qp: -1
2025-11-19T13:59:39.821273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435871319159263:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:39.821281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:39.821570Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435874401675804:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.821607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435874401675724:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.824187Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435874401675786:2103] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:39.826783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435872410545903:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.824051Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435872644050538:2103] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:39.827061Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435872899226237:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.827112Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435872899226239:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.827117Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435872899226239:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.826128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435873293016882:2048] [node 2] ICP18 handshake (actor [1:7574435873293016965:2115]) is too old
2025-11-19T13:59:39.829111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435873293016973:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7206
2025-11-19T13:59:39.826570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435874401675757:2081] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.826593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:39.826600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435874401675757:2081] [node 5] ICP22 created new session: [6:7574435874401675811:2081]
2025-11-19T13:59:39.826606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435874401675811:2081] [node 5] ICS09 handshake done sender: [6:7574435874401675786:2103] self: [6:7574435874401675800:0] peer: [5:7574435872644050537:0] socket: 545 qp: -1
2025-11-19T13:59:39.826613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435874401675811:2081] [node 5] ICS10 traffic start
2025-11-19T13:59:39.826623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.830803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435874401675813:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31028
2025-11-19T13:59:39.828824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435872899226169:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.829347Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435872899226239:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:39.829444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435872899226235:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.829459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:39.829465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435872899226235:2109] [node 6] ICP22 created new session: [3:7574435872899226251:2109]
2025-11-19T13:59:39.829470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435872899226251:2109] [node 6] ICS09 handshake done sender: [3:7574435872899226239:2112] self: [3:7574435872899226238:0] peer: [6:7574435874401675810:0] socket: 552 qp: -1
2025-11-19T13:59:39.829477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435872899226251:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:39.829485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:39.830343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435872644050565:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22292
2025-11-19T13:59:39.831487Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435872644050565:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.831521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435872644050482:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.831606Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435872899226248:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.831616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435872899226248:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.832213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435872644050567:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22292
2025-11-19T13:59:39.832290Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435872899226248:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:39.832442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435872899226243:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.832453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:39.832459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435872899226243:2113] [node 5] ICP22 created new session: [3:7574435872899226255:2113]
2025-11-19T13:59:39.832465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435872899226255:2113] [node 5] ICS09 handshake done sender: [3:7574435872899226248:2115] self: [3:7574435872899226247:0] peer: [5:7574435872644050566:0] socket: 582 qp: -1
2025-11-19T13:59:39.832471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435872899226255:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:39.832480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.838198Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435872899226244:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.838208Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435872899226244:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.839065Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435872899226244:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:39.839498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435872899226232:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.839510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:39.839515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435872899226232:2108] [node 4] ICP22 created new session: [3:7574435872899226260:2108]
2025-11-19T13:59:39.839519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435872899226260:2108] [node 4] ICS09 handshake done sender: [3:7574435872899226244:2114] self: [3:7574435872899226233:0] peer: [4:7574435872410545986:0] socket: 570 qp: -1
2025-11-19T13:59:39.839524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435872899226260:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:39.839529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:39.836447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435872410545983:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1340
2025-11-19T13:59:39.836511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435872410545984:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1340
2025-11-19T13:59:39.836547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435872410545985:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1340
2025-11-19T13:59:39.838042Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435872410545984:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.838082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435872410545904:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.838889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435872410545987:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1340
2025-11-19T13:59:39.839184Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435872410545984:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:39.839239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435872410545904:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.839254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:39.839259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435872410545904:2050] [node 3] ICP22 created new session: [4:7574435872410545988:2050]
2025-11-19T13:59:39.839265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435872410545988:2050] [node 3] ICS09 handshake done sender: [4:7574435872410545984:2117] self: [4:7574435872410545986:0] peer: [3:7574435872899226233:0] socket: 589 qp: -1
2025-11-19T13:59:39.839271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435872410545988:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:39.839301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:39.840032Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435873293016957:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.840050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435873293016957:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.843981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435872644050536:2102] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.844033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:39.844042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435872644050536:2102] [node 6] ICP22 created new session: [5:7574435872644050568:2102]
2025-11-19T13:59:39.844049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435872644050568:2102] [node 6] ICS09 handshake done sender: [5:7574435872644050538:2103] self: [5:7574435872644050537:0] peer: [6:7574435874401675800:0] socket: 539 qp: -1
2025-11-19T13:59:39.844058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435872644050568:2102] [node 6] ICS10 traffic start
2025-11-19T13:59:39.844067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:39.844187Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435872644050536:2102] [node 6] ICP35 event processing took too much time PendingConnection:278(19851263ns) {HandleHandshakeStatus:329(19850680ns) {DropIncomingHandshake:441(18750ns) DropOutgoingHandshake:456(233ns) SwitchToState:178(582ns) ProcessPendingSessionEvents:495(32431ns) {ForwardSessionEventToSession:612(21826ns) ForwardSessionEventToSession:612(1327ns) ForwardSessionEventToSession:612(1067ns) ForwardSessionEventToSession:612(632ns) ForwardSessionEventToSession:612(743ns) ForwardSessionEventToSession:612(888ns) ForwardSessionEventToSession:612(732ns) ForwardSessionEventToSession:612(774ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(1206ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.847983Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435871319159253:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.847997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435871319159253:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.848836Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435871319159240:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.848840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435871319159240:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.849601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435871319159265:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:283
2025-11-19T13:59:39.853667Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435872410545966:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(32572950ns) {Configure:104(32571653ns) {ProcessConfigured:135(32225670ns) {StartInitialHandshake:157(32225078ns) {DropHandshakes:469(523ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:476(1801ns) {SwitchToState:178(1222ns)}}}}}
2025-11-19T13:59:39.853760Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435872410545983:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.854408Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435874401675804:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:39.854646Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435872644050565:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:39.855684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435872644050482:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.855702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:39.855708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435872644050482:2050] [node 3] ICP22 created new session: [5:7574435872644050570:2050]
2025-11-19T13:59:39.855715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435872644050570:2050] [node 3] ICS09 handshake done sender: [5:7574435872644050565:2117] self: [5:7574435872644050566:0] peer: [3:7574435872899226247:0] socket: 583 qp: -1
2025-11-19T13:59:39.855722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435872644050570:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:39.855728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:39.856023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435874401675724:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.856037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:39.856043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435874401675724:2050] [node 3] ICP22 created new session: [6:7574435874401675814:2050]
2025-11-19T13:59:39.856048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435874401675814:2050] [node 3] ICS09 handshake done sender: [6:7574435874401675804:2117] self: [6:7574435874401675810:0] peer: [3:7574435872899226238:0] socket: 556 qp: -1
2025-11-19T13:59:39.856053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435874401675814:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:39.856059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:39.854037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435872410545991:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1340
2025-11-19T13:59:39.855458Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435872410545985:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.859331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872899226263:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34807
2025-11-19T13:59:39.859353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872899226264:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34807
2025-11-19T13:59:39.859404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872899226265:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34807
2025-11-19T13:59:39.860908Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435873293016957:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:39.860968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435872899226168:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(46998239ns) {Configure:104(46997438ns) {ProcessConfigured:135(46642218ns) {StartInitialHandshake:157(46641389ns) {DropHandshakes:469(670ns) {DropIncomingHandshake:441(115ns) DropOutgoingHandshake:456(195ns)} GenerateSessionVirtualId:988(274ns) PrepareNewSessionHandshake:476(1205ns) {SwitchToState:178(853ns)}}}}}
2025-11-19T13:59:39.861240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435873293016955:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.861253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:39.861259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435873293016955:2111] [node 4] ICP22 created new session: [1:7574435873293016975:2111]
2025-11-19T13:59:39.861264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435873293016975:2111] [node 4] ICS09 handshake done sender: [1:7574435873293016957:2113] self: [1:7574435873293016956:0] peer: [4:7574435872410545978:0] socket: 560 qp: -1
2025-11-19T13:59:39.861270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435873293016975:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:39.861276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:39.861390Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435871319159253:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.862575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435873293016977:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7206
2025-11-19T13:59:39.862191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872899226262:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34807
2025-11-19T13:59:39.862305Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435872899226264:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.862334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435872899226168:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.862710Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435871319159240:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:39.862798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435871319159176:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(41529735ns) {HandleHandshakeStatus:329(41529139ns) {DropIncomingHandshake:441(201ns) DropOutgoingHandshake:456(1446ns) SwitchToState:178(73ns) ProcessPendingSessionEvents:495(22405649ns) {ForwardSessionEventToSession:612(22394998ns) ForwardSessionEventToSession:612(1075ns) ForwardSessionEventToSession:612(932ns) ForwardSessionEventToSession:612(590ns) ForwardSessionEventToSession:612(558ns) ForwardSessionEventToSession:612(985ns) ForwardSessionEventToSession:612(623ns) ForwardSessionEventToSession:612(528ns) ForwardSessionEventToSession:612(627ns) ForwardSessionEventToSession:612(677ns)}}}
2025-11-19T13:59:39.862858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435871319159244:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.862866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:39.862870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435871319159244:2110] [node 4] ICP22 created new session: [2:7574435871319159268:2110]
2025-11-19T13:59:39.862874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435871319159268:2110] [node 4] ICS09 handshake done sender: [2:7574435871319159253:2114] self: [2:7574435871319159250:0] peer: [4:7574435872410545981:0] socket: 554 qp: -1
2025-11-19T13:59:39.862878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435871319159268:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:39.862883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.863688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435872410545994:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1340
2025-11-19T13:59:39.864650Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435872899226237:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:39.864773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435872899226169:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.864785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:39.864791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435872899226169:2049] [node 2] ICP22 created new session: [3:7574435872899226267:2049]
2025-11-19T13:59:39.864796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435872899226267:2049] [node 2] ICS09 handshake done sender: [3:7574435872899226237:2111] self: [3:7574435872899226250:0] peer: [2:7574435871319159239:0] socket: 551 qp: -1
2025-11-19T13:59:39.864802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435872899226267:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:39.864809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:39.864925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435871319159177:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.864940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:39.864945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435871319159177:2049] [node 3] ICP22 created new session: [2:7574435871319159271:2049]
2025-11-19T13:59:39.864949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435871319159271:2049] [node 3] ICS09 handshake done sender: [2:7574435871319159240:2108] self: [2:7574435871319159239:0] peer: [3:7574435872899226250:0] socket: 549 qp: -1
2025-11-19T13:59:39.864952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435871319159271:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:39.864958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:39.867199Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435873293016959:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.867215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435873293016959:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.868261Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435872410545969:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:39.868644Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435873293016977:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.869056Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435872410545968:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:39.869166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435872410545902:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.869182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:39.869187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435872410545902:2048] [node 1] ICP22 created new session: [4:7574435872410545995:2048]
2025-11-19T13:59:39.869192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435872410545995:2048] [node 1] ICS09 handshake done sender: [4:7574435872410545969:2111] self: [4:7574435872410545978:0] peer: [1:7574435873293016956:0] socket: 563 qp: -1
2025-11-19T13:59:39.869198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435872410545995:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:39.869204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:39.869314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435872410545903:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.869321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:39.869325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435872410545903:2049] [node 2] ICP22 created new session: [4:7574435872410545997:2049]
2025-11-19T13:59:39.869328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435872410545997:2049] [node 2] ICS09 handshake done sender: [4:7574435872410545968:2110] self: [4:7574435872410545981:0] peer: [2:7574435871319159250:0] socket: 555 qp: -1
2025-11-19T13:59:39.869331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435872410545997:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:39.869335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:39.871447Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435871319159265:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.871481Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435871319159261:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.871678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435871319159244:2110] [node 4] ICP18 handshake (actor [2:7574435871319159261:2118]) is too old
2025-11-19T13:59:39.870163Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435873293016973:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.876484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435874401675816:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31028
2025-11-19T13:59:39.876538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435874401675817:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31028
2025-11-19T13:59:39.876978Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435874401675816:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.877013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435874401675723:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.877770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435873293016955:2111] [node 4] ICP35 event processing took too much time PendingConnection:278(16493871ns) {HandleHandshakeStatus:329(16492855ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(168ns) SwitchToState:178(172ns) ProcessPendingSessionEvents:495(16425870ns) {ForwardSessionEventToSession:612(16410023ns) ForwardSessionEventToSession:612(1678ns) ForwardSessionEventToSession:612(1094ns) ForwardSessionEventToSession:612(2180ns) ForwardSessionEventToSession:612(930ns) ForwardSessionEventToSession:612(613ns) ForwardSessionEventToSession:612(607ns) ForwardSessionEventToSession:612(524ns) ForwardSessionEventToSession:612(630ns) ForwardSessionEventToSession:612(1216ns)}}}
2025-11-19T13:59:39.877860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435873293016955:2111] [node 4] ICP18 handshake (actor [1:7574435873293016973:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.878911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435872644050572:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22292
2025-11-19T13:59:39.878974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435872644050573:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22292
2025-11-19T13:59:39.878199Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435871319159259:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.878217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435871319159259:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.879451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435874401675819:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31028
2025-11-19T13:59:39.879576Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435871319159259:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:39.879739Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435874401675816:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:39.879845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435874401675723:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.879860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:39.879866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435874401675723:2049] [node 2] ICP22 created new session: [6:7574435874401675820:2049]
2025-11-19T13:59:39.879872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435874401675820:2049] [node 2] ICS09 handshake done sender: [6:7574435874401675816:2121] self: [6:7574435874401675818:0] peer: [2:7574435871319159252:0] socket: 541 qp: -1
2025-11-19T13:59:39.879878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435874401675820:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:39.879884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:39.881060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435871319159248:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.881082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:39.881087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435871319159248:2113] [node 6] ICP22 created new session: [2:7574435871319159276:2113]
2025-11-19T13:59:39.881093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435871319159276:2113] [node 6] ICS09 handshake done sender: [2:7574435871319159259:2117] self: [2:7574435871319159252:0] peer: [6:7574435874401675818:0] socket: 546 qp: -1
2025-11-19T13:59:39.881097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435871319159276:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:39.881103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:39.885692Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435872410545980:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.885706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435872410545980:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.887007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435872899226243:2113] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:39.885002Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435872644050572:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.885050Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435872644050573:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.885087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435872644050481:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.885097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435872644050558:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:39.886692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435872644050576:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22292
2025-11-19T13:59:39.887131Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435872410545980:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:39.887308Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435873293016959:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:39.887303Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435872644050573:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:39.887400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435872410545965:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.887418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:39.887425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435872410545965:2108] [node 5] ICP22 created new session: [4:7574435872410546002:2108]
2025-11-19T13:59:39.887431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435872410546002:2108] [node 5] ICS09 handshake done sender: [4:7574435872410545980:2115] self: [4:7574435872410545976:0] peer: [5:7574435872644050575:0] socket: 553 qp: -1
2025-11-19T13:59:39.887437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435872410546002:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:39.887445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:39.887599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435873293016883:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.887622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:39.887629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435873293016883:2049] [node 3] ICP22 created new session: [1:7574435873293016980:2049]
2025-11-19T13:59:39.887636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435873293016980:2049] [node 3] ICS09 handshake done sender: [1:7574435873293016959:2114] self: [1:7574435873293016958:0] peer: [3:7574435872899226266:0] socket: 568 qp: -1
2025-11-19T13:59:39.887641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435873293016980:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:39.887649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:39.887801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435872644050558:2114] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.887824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:39.887829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435872644050558:2114] [node 4] ICP22 created new session: [5:7574435872644050577:2114]
2025-11-19T13:59:39.887834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435872644050577:2114] [node 4] ICS09 handshake done sender: [5:7574435872644050573:2120] self: [5:7574435872644050575:0] peer: [4:7574435872410545976:0] socket: 543 qp: -1
2025-11-19T13:59:39.887841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435872644050577:2114] [node 4] ICS10 traffic start
2025-11-19T13:59:39.887852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:39.888171Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435871319159256:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.888189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435871319159256:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.889942Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435874401675817:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.889983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435874401675795:2102] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.890117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435872644050579:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22292
2025-11-19T13:59:39.890290Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435871319159256:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:39.890087Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435872410545990:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:39.890096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435872410545990:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:39.890387Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435872644050572:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:39.890451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435872644050481:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.890462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:39.890467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435872644050481:2049] [node 2] ICP22 created new session: [5:7574435872644050580:2049]
2025-11-19T13:59:39.890473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435872644050580:2049] [node 2] ICS09 handshake done sender: [5:7574435872644050572:2119] self: [5:7574435872644050574:0] peer: [2:7574435871319159251:0] socket: 535 qp: -1
2025-11-19T13:59:39.890478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435872644050580:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:39.890485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:39.890726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435871319159245:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.890740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:39.890745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435871319159245:2111] [node 5] ICP22 created new session: [2:7574435871319159279:2111]
2025-11-19T13:59:39.890750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435871319159279:2111] [node 5] ICS09 handshake done sender: [2:7574435871319159256:2116] self: [2:7574435871319159251:0] peer: [5:7574435872644050574:0] socket: 601 qp: -1
2025-11-19T13:59:39.890753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435871319159279:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:39.890759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.891450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435872899226269:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34807
2025-11-19T13:59:39.891617Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435872899226264:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:39.891679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435872899226168:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.891692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:39.891697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435872899226168:2048] [node 1] ICP22 created new session: [3:7574435872899226270:2048]
2025-11-19T13:59:39.891702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435872899226270:2048] [node 1] ICS09 handshake done sender: [3:7574435872899226264:2119] self: [3:7574435872899226266:0] peer: [1:7574435873293016958:0] socket: 598 qp: -1
2025-11-19T13:59:39.891707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435872899226270:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:39.891715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:39.893010Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435872410545990:2120] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:39.893105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435872410545966:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:39.893123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:39.893128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435872410545966:2109] [node 6] ICP22 created new session: [4:7574435872410546005:2109]
2025-11-19T13:59:39.893132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435872410546005:2109] [node 6] ICS09 handshake done sender: [4:7574435872410545990:2120] self: [4:7574435872410545979:0] peer: [6:7574435874401675822:0] socket: 564 qp: -1
2025-11-19T13:59:39.893137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435872410546005:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:39.893144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:39.894753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435874401675795:2102] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15365) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435874401675817:2122] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:39.896949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435874401675823:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31028
2025-11-19T13:59:39.904089Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435874401675817:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:39.907967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435874401675795:2102] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:39.907987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:39.907995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435874401675795:2102] [node 4] ICP22 created new session: [6:7574435874401675824:2102]
2025-11-19T13:59:39.908001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435874401675824:2102] [node 4] ICS09 handshake done sender: [6:7574435874401675817:2122] self: [6:7574435874401675822:0] peer: [4:7574435872410545979:0] socket: 544 qp: -1
2025-11-19T13:59:39.908010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435874401675824:2102] [node 4] ICS10 traffic start
2025-11-19T13:59:39.908026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:39.921283Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:39.921384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574435871319159263:2048] [node 1] ICS07 socket disconnect 537 reason# EndOfStream
2025-11-19T13:59:39.921390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435871319159263:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.921433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:39.921451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435871319159263:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:39.923565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:39.923644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435872899226270:2048] [node 1] ICS01 socket: 598 reason# EndOfStream
2025-11-19T13:59:39.923653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435872899226168:2048] [node 1] ICP30 unregister session Session# [3:7574435872899226270:2048] VirtualId# [3:7574435872899226266:0]
2025-11-19T13:59:39.923659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435872899226270:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.923700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:39.923722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:39.927354Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:39.927433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435872410545995:2048] [node 1] ICS01 socket: 563 reason# EndOfStream
2025-11-19T13:59:39.927440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435872410545902:2048] [node 1] ICP30 unregister session Session# [4:7574435872410545995:2048] VirtualId# [4:7574435872410545978:0]
2025-11-19T13:59:39.927446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435872410545995:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.927488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:39.927513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:39.927589Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:39.927648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435874401675806:2048] [node 1] ICS01 socket: 538 reason# EndOfStream
2025-11-19T13:59:39.927652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435874401675722:2048] [node 1] ICP30 unregister session Session# [6:7574435874401675806:2048] VirtualId# [6:7574435874401675798:0]
2025-11-19T13:59:39.927655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435874401675806:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.927681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:39.927711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:39.930130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:39.930218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435872644050552:2048] [node 1] ICS07 socket disconnect 557 reason# EndOfStream
2025-11-19T13:59:39.930224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435872644050552:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.930277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:39.930297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435872644050552:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:39.931072Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435872644050480:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9756) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:39.931086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435872644050552:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:39.931092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435872644050480:2048] [node 1] ICP30 unregister session Session# [5:7574435872644050552:2048] VirtualId# [5:7574435872644050550:0]
2025-11-19T13:59:39.931105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:39.931111Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435872644050480:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9756) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:39.931114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9756) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:39.939016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:39.939106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435872899226267:2049] [node 2] ICS01 socket: 551 reason# EndOfStream
2025-11-19T13:59:39.939115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435872899226169:2049] [node 2] ICP30 unregister session Session# [3:7574435872899226267:2049] VirtualId# [3:7574435872899226250:0]
2025-11-19T13:59:39.939121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435872899226267:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.939155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:39.939177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:39.943803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435872644050480:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:39.943845Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:39.943926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435872644050580:2049] [node 2] ICS01 socket: 535 reason# EndOfStream
2025-11-19T13:59:39.943936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435872644050481:2049] [node 2] ICP30 unregister session Session# [5:7574435872644050580:2049] VirtualId# [5:7574435872644050574:0]
2025-11-19T13:59:39.943943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435872644050580:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.944014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:39.944042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:39.943827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:39.943914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435872410545997:2049] [node 2] ICS07 socket disconnect 555 reason# EndOfStream
2025-11-19T13:59:39.943921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435872410545997:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.943982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:39.944005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435872410545997:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:39.944247Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:39.944313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435874401675820:2049] [node 2] ICS01 socket: 541 reason# EndOfStream
2025-11-19T13:59:39.944318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435874401675723:2049] [node 2] ICP30 unregister session Session# [6:7574435874401675820:2049] VirtualId# [6:7574435874401675818:0]
2025-11-19T13:59:39.944323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435874401675820:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.944349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:39.944370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:39.944656Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435872410545903:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6913) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:39.944669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435872410545997:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:39.944673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435872410545903:2049] [node 2] ICP30 unregister session Session# [4:7574435872410545997:2049] VirtualId# [4:7574435872410545981:0]
2025-11-19T13:59:39.944686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:39.944691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435872410545903:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:39.944694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6913) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:39.957689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435872410545903:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:39.966900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:39.967004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435872644050570:2050] [node 3] ICS01 socket: 583 reason# EndOfStream
2025-11-19T13:59:39.967013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435872644050482:2050] [node 3] ICP30 unregister session Session# [5:7574435872644050570:2050] VirtualId# [5:7574435872644050566:0]
2025-11-19T13:59:39.967020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435872644050570:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.967069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:39.967090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:39.967177Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:39.967236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435874401675814:2050] [node 3] ICS01 socket: 556 reason# EndOfStream
2025-11-19T13:59:39.967241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435874401675724:2050] [node 3] ICP30 unregister session Session# [6:7574435874401675814:2050] VirtualId# [6:7574435874401675810:0]
2025-11-19T13:59:39.967246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435874401675814:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.967270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:39.967289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:39.981059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:39.981150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435874401675824:2102] [node 4] ICS01 socket: 544 reason# EndOfStream
2025-11-19T13:59:39.981158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435874401675795:2102] [node 4] ICP30 unregister session Session# [6:7574435874401675824:2102] VirtualId# [6:7574435874401675822:0]
2025-11-19T13:59:39.981166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435874401675824:2102] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:39.981223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:39.981250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:40.043953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435874714458848:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:40.043968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435874714458849:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:40.084195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435876097962031:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:40.084211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435876097962032:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:40.116322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435874736114580:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:40.116337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435874736114581:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:40.140001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435876585613920:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:40.140022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435876585613921:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:40.140031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435876585613922:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:40.168082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435877800628973:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:40.168104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435877800628974:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:40.168108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435877800628975:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:40.188294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435874736114601:2066] [node 5] ICP01 ready to work
2025-11-19T13:59:40.188296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435876585613938:2064] [node 6] ICP01 ready to work
2025-11-19T13:59:40.188763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435876097962077:2089] [node 4] ICP01 ready to work
2025-11-19T13:59:40.188865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435876097962078:2091] [node 5] ICP01 ready to work
2025-11-19T13:59:40.193280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435874714458916:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:40.195292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435877800629016:2070] [node 6] ICP01 ready to work
2025-11-19T13:59:40.198072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435874714458915:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.207905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435874736114580:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(19712431ns) {Configure:104(19710732ns) {ProcessConfigured:135(18470838ns) {StartInitialHandshake:157(18470038ns) {DropHandshakes:469(528ns) {DropIncomingHandshake:441(122ns) DropOutgoingHandshake:456(97ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(1782ns) {SwitchToState:178(1067ns)}}}}}
2025-11-19T13:59:40.208450Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435874736114601:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(20097732ns) {Configure:104(20096815ns) {ProcessConfigured:135(18935997ns) {StartInitialHandshake:157(18935648ns) {DropHandshakes:469(133ns) {DropIncomingHandshake:441(21ns) DropOutgoingHandshake:456(41ns)} GenerateSessionVirtualId:988(38ns) PrepareNewSessionHandshake:476(1410ns) {SwitchToState:178(455ns)}}}}}
2025-11-19T13:59:40.208548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435874736114639:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26907
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.208922Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435876097962078:2091] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19988268ns) {Configure:104(19551156ns) {ProcessConfigured:135(19311452ns) {StartInitialHandshake:157(19310664ns) {DropHandshakes:469(569ns) {DropIncomingHandshake:441(26ns) DropOutgoingHandshake:456(15ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:476(615ns) {SwitchToState:178(333ns)}}}}}
No RDMA context found for GID: 2025-11-19T13:59:40.210686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435877800629036:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56083
::ffff:127.1.0.0
2025-11-19T13:59:40.210709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435877800629037:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56083
2025-11-19T13:59:40.210784Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435876585613920:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(22626044ns) {Configure:104(22623915ns) {ProcessConfigured:135(20968204ns) {StartInitialHandshake:157(20967323ns) {DropHandshakes:469(571ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(121ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(677ns) {SwitchToState:178(302ns)}}}}}
2025-11-19T13:59:40.210875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435876585613978:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33362
2025-11-19T13:59:40.212441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435876585613938:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(24084019ns) {Configure:104(24083345ns) {ProcessConfigured:135(22491469ns) {StartInitialHandshake:157(22490714ns) {DropHandshakes:469(175ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(964ns) {SwitchToState:178(470ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.219812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435877800628993:2063] [node 4] ICP01 ready to work
2025-11-19T13:59:40.219934Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435877800629037:2103] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.179203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435877957252732:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:40.179218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435877957252733:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:40.179224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435877957252734:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:40.194237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435877957252760:2067] [node 4] ICP01 ready to work
2025-11-19T13:59:40.194316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435877957252761:2071] [node 5] ICP01 ready to work
2025-11-19T13:59:40.210593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435877957252732:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21808817ns) {Configure:104(21802668ns) {ProcessConfigured:135(20925750ns) {StartInitialHandshake:157(20925461ns) {DropHandshakes:469(409ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:476(719ns) {SwitchToState:178(220ns)}}}}}
2025-11-19T13:59:40.211020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435877957252733:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(22258560ns) {Configure:104(22248164ns) {ProcessConfigured:135(21132790ns) {StartInitialHandshake:157(21132587ns) {DropHandshakes:469(191ns) {DropIncomingHandshake:441(46ns) DropOutgoingHandshake:456(13ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:476(262ns) {SwitchToState:178(41ns)}}}}}
2025-11-19T13:59:40.213433Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435877957252734:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(24653644ns) {Configure:104(24652497ns) {ProcessConfigured:135(23322276ns) {StartInitialHandshake:157(23321673ns) {DropHandshakes:469(338ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(273ns) PrepareNewSessionHandshake:476(1994ns) {SwitchToState:178(880ns)}}}}}
2025-11-19T13:59:40.214932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435877957252798:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48215
2025-11-19T13:59:40.219844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435874714458912:2106] [node 5] ICP01 ready to work
2025-11-19T13:59:40.220988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435877957252760:2067] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(26625654ns) {Configure:104(26624652ns) {ProcessConfigured:135(26335186ns) {StartInitialHandshake:157(26334482ns) {DropHandshakes:469(630ns) {DropIncomingHandshake:441(124ns) DropOutgoingHandshake:456(183ns)} GenerateSessionVirtualId:988(152ns) PrepareNewSessionHandshake:476(939ns) {SwitchToState:178(440ns)}}}}}
2025-11-19T13:59:40.223035Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435876097962099:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.223054Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435876097962099:2106] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.222351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435876585613921:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(25921140ns) {Configure:104(25920059ns) {ProcessConfigured:135(25573382ns) {StartInitialHandshake:157(25572764ns) {DropHandshakes:469(597ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(89ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(992ns) {SwitchToState:178(443ns)}}}}}
2025-11-19T13:59:40.223670Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435874714458915:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25545638ns) {Configure:104(25544962ns) {ProcessConfigured:135(25229514ns) {StartInitialHandshake:157(25229184ns) {DropHandshakes:469(409ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(633ns) {SwitchToState:178(275ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.224015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435876585613996:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33362
2025-11-19T13:59:40.223579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435876097962108:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37451
2025-11-19T13:59:40.223597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435876097962109:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37451
2025-11-19T13:59:40.221239Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435877800628975:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(25898052ns) {Configure:104(25897215ns) {ProcessConfigured:135(25583709ns) {StartInitialHandshake:157(25583311ns) {DropHandshakes:469(491ns) {DropIncomingHandshake:441(112ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(107ns) PrepareNewSessionHandshake:476(8704ns) {SwitchToState:178(8044ns)}}}}}
2025-11-19T13:59:40.226586Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435877800629016:2070] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(28314456ns) {Configure:104(28312602ns) {ProcessConfigured:135(27963526ns) {StartInitialHandshake:157(27962551ns) {DropHandshakes:469(520ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(271ns) PrepareNewSessionHandshake:476(841ns) {SwitchToState:178(661ns)}}}}}
2025-11-19T13:59:40.228574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435876097962110:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37451
2025-11-19T13:59:40.228578Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435876097962099:2106] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:40.232323Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435876585613978:2101] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.232375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435876585613921:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.233938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435874736114639:2103] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.235281Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435874736114581:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16492536ns) {Configure:104(16491720ns) {ProcessConfigured:135(16119253ns) {StartInitialHandshake:157(16118964ns) {DropHandshakes:469(800ns) {DropIncomingHandshake:441(208ns) DropOutgoingHandshake:456(76ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:476(413ns) {SwitchToState:178(254ns)}}}}}
2025-11-19T13:59:40.235308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435874736114581:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.233483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435876585613922:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17151961ns) {Configure:104(17150684ns) {ProcessConfigured:135(16715331ns) {StartInitialHandshake:157(16714581ns) {DropHandshakes:469(893ns) {DropIncomingHandshake:441(147ns) DropOutgoingHandshake:456(146ns)} GenerateSessionVirtualId:988(346ns) PrepareNewSessionHandshake:476(676ns) {SwitchToState:178(389ns)}}}}}
2025-11-19T13:59:40.235702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435876585613999:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33362
2025-11-19T13:59:40.235680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435877957252761:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(34439718ns) {Configure:104(15730016ns) {ProcessConfigured:135(15350853ns) {StartInitialHandshake:157(15350319ns) {DropHandshakes:469(1187ns) {DropIncomingHandshake:441(100ns) DropOutgoingHandshake:456(200ns)} GenerateSessionVirtualId:988(258ns) PrepareNewSessionHandshake:476(812ns) {SwitchToState:178(343ns)}}}}}
2025-11-19T13:59:40.236018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435874736114648:2079] [node 4] ICP01 ready to work
2025-11-19T13:59:40.231920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435876097962107:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37451
2025-11-19T13:59:40.232245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435876097962086:2094] [node 6] ICP01 ready to work
2025-11-19T13:59:40.234427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435876097962031:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(45931982ns) {Configure:104(45930868ns) {ProcessConfigured:135(45514402ns) {StartInitialHandshake:157(45514052ns) {DropHandshakes:469(575ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(96ns)} GenerateSessionVirtualId:988(170ns) PrepareNewSessionHandshake:476(472ns) {SwitchToState:178(174ns)}}}}}
2025-11-19T13:59:40.234473Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435876097962094:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.234477Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435876097962094:2104] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.235952Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435876097962094:2104] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:40.236204Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435877800629036:2102] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.236234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435877800628975:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.236139Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435876585613978:2101] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:40.236376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435876585613921:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.236396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:40.236401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435876585613921:2049] [node 2] ICP22 created new session: [4:7574435876585614000:2049]
2025-11-19T13:59:40.236407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435876585614000:2049] [node 2] ICS09 handshake done sender: [4:7574435876585613978:2101] self: [4:7574435876585613997:0] peer: [2:7574435876097962082:0] socket: 556 qp: -1
2025-11-19T13:59:40.236414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435876585614000:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:40.236433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.239917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435874736114649:2075] [node 6] ICP01 ready to work
2025-11-19T13:59:40.239975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435874736114658:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26907
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.241636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435876097962077:2089] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.241659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:40.241670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435876097962077:2089] [node 4] ICP22 created new session: [2:7574435876097962117:2089]
2025-11-19T13:59:40.241678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435876097962117:2089] [node 4] ICS09 handshake done sender: [2:7574435876097962094:2104] self: [2:7574435876097962082:0] peer: [4:7574435876585613997:0] socket: 543 qp: -1
2025-11-19T13:59:40.241685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435876097962117:2089] [node 4] ICS10 traffic start
2025-11-19T13:59:40.241708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.243649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435874714458916:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(50316908ns) {Configure:104(50316010ns) {ProcessConfigured:135(50021578ns) {StartInitialHandshake:157(50020844ns) {DropHandshakes:469(701ns) {DropIncomingHandshake:441(235ns) DropOutgoingHandshake:456(100ns)} GenerateSessionVirtualId:988(37ns) PrepareNewSessionHandshake:476(935ns) {SwitchToState:178(752ns)}}}}}
2025-11-19T13:59:40.238140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435877800628974:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.238175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435877800628974:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(17989997ns) {IncomingHandshake:261(17989530ns) {DropIncomingHandshake:441(268ns) PrepareNewSessionHandshake:476(611ns) {SwitchToState:178(172ns)} GenerateSessionVirtualId:988(235ns) IssueIncomingHandshakeReply:193(17983510ns) {SwitchToState:178(612ns)}}}
2025-11-19T13:59:40.241285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435877800629054:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56083
2025-11-19T13:59:40.238054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435876585613987:2084] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.244145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435874736114662:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26907
2025-11-19T13:59:40.244166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435874736114663:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26907
2025-11-19T13:59:40.244286Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435877800629037:2103] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:40.243318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435876097962116:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37451
2025-11-19T13:59:40.246196Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435874736114640:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.246210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435874736114640:2104] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.247825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435877800628974:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.247847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:40.247855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435877800628974:2049] [node 2] ICP22 created new session: [5:7574435877800629055:2049]
2025-11-19T13:59:40.247862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435877800629055:2049] [node 2] ICS09 handshake done sender: [5:7574435877800629037:2103] self: [5:7574435877800629045:0] peer: [2:7574435876097962083:0] socket: 554 qp: -1
2025-11-19T13:59:40.247870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435877800629055:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:40.247889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:40.245930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435876097962078:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.245954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:40.245966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435876097962078:2091] [node 5] ICP22 created new session: [2:7574435876097962119:2091]
2025-11-19T13:59:40.245973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435876097962119:2091] [node 5] ICS09 handshake done sender: [2:7574435876097962099:2106] self: [2:7574435876097962083:0] peer: [5:7574435877800629045:0] socket: 552 qp: -1
2025-11-19T13:59:40.245983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435876097962119:2091] [node 5] ICS10 traffic start
2025-11-19T13:59:40.246024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:40.247929Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435876097962096:2105] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.247942Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435876097962096:2105] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.250308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435874736114664:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26907
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.251914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435877957252811:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48215
2025-11-19T13:59:40.252251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435874714458927:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23588
2025-11-19T13:59:40.252272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435874714458928:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23588
2025-11-19T13:59:40.252327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435874714458929:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23588
2025-11-19T13:59:40.252338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435874714458930:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23588
2025-11-19T13:59:40.252347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435874714458931:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23588
2025-11-19T13:59:40.253981Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435874714458928:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.254810Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435876097962116:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.255921Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435877957252811:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.256009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435877957252733:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.256515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435877800628993:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(36529503ns) {Configure:104(36528794ns) {ProcessConfigured:135(36154744ns) {StartInitialHandshake:157(36154102ns) {DropHandshakes:469(494ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(42ns)} GenerateSessionVirtualId:988(260ns) PrepareNewSessionHandshake:476(539ns) {SwitchToState:178(317ns)}}}}}
2025-11-19T13:59:40.256731Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435876585613921:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(20329305ns) {HandleHandshakeStatus:329(20328903ns) {DropIncomingHandshake:441(145ns) DropOutgoingHandshake:456(1824ns) SwitchToState:178(51ns) ProcessPendingSessionEvents:495(10521420ns) {ForwardSessionEventToSession:612(10509590ns) ForwardSessionEventToSession:612(1333ns) ForwardSessionEventToSession:612(946ns) ForwardSessionEventToSession:612(849ns) ForwardSessionEventToSession:612(564ns) ForwardSessionEventToSession:612(1176ns) ForwardSessionEventToSession:612(866ns) ForwardSessionEventToSession:612(529ns) ForwardSessionEventToSession:612(589ns) ForwardSessionEventToSession:612(1042ns)}}}
2025-11-19T13:59:40.257718Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435874714458931:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.256203Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435876097962121:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.256215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435876097962121:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.259973Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435874736114640:2104] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:40.260176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435874736114601:2066] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.260191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:40.260198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435874736114601:2066] [node 5] ICP22 created new session: [3:7574435874736114667:2066]
2025-11-19T13:59:40.260205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435874736114667:2066] [node 5] ICS09 handshake done sender: [3:7574435874736114640:2104] self: [3:7574435874736114606:0] peer: [5:7574435877800629053:0] socket: 551 qp: -1
2025-11-19T13:59:40.260211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435874736114667:2066] [node 5] ICS10 traffic start
2025-11-19T13:59:40.260223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.260588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435877800629059:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56083
2025-11-19T13:59:40.261722Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435877800629036:2102] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:40.261861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435877800628975:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.261874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:40.261880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435877800628975:2050] [node 3] ICP22 created new session: [5:7574435877800629060:2050]
2025-11-19T13:59:40.261885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435877800629060:2050] [node 3] ICS09 handshake done sender: [5:7574435877800629036:2102] self: [5:7574435877800629053:0] peer: [3:7574435874736114606:0] socket: 553 qp: -1
2025-11-19T13:59:40.261891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435877800629060:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:40.261908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.258729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435877957252813:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48215
2025-11-19T13:59:40.260259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435876097962078:2091] [node 5] ICP35 event processing took too much time PendingConnection:278(31537146ns) {HandleHandshakeStatus:329(31536188ns) {DropIncomingHandshake:441(359ns) DropOutgoingHandshake:456(363ns) SwitchToState:178(200ns) ProcessPendingSessionEvents:495(4077611ns) {ForwardSessionEventToSession:612(4065968ns) ForwardSessionEventToSession:612(1643ns) ForwardSessionEventToSession:612(1198ns) ForwardSessionEventToSession:612(741ns) ForwardSessionEventToSession:612(637ns) ForwardSessionEventToSession:612(1077ns) ForwardSessionEventToSession:612(592ns) ForwardSessionEventToSession:612(620ns) ForwardSessionEventToSession:612(496ns) ForwardSessionEventToSession:612(835ns)}}}
2025-11-19T13:59:40.263993Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435876097962096:2105] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:40.262024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435874736114670:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26907
2025-11-19T13:59:40.265042Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435876097962121:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:40.265107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435876097962032:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.265126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:40.265133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435876097962032:2049] [node 3] ICP22 created new session: [2:7574435876097962126:2049]
2025-11-19T13:59:40.265139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435876097962126:2049] [node 3] ICS09 handshake done sender: [2:7574435876097962096:2105] self: [2:7574435876097962095:0] peer: [3:7574435874736114656:0] socket: 544 qp: -1
2025-11-19T13:59:40.265144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435876097962126:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:40.265153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:40.265545Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435874736114639:2103] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.265813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435877957252814:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48215
2025-11-19T13:59:40.265830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435877957252815:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48215
2025-11-19T13:59:40.266861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435874736114581:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.266881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:40.266885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435874736114581:2049] [node 2] ICP22 created new session: [3:7574435874736114671:2049]
2025-11-19T13:59:40.266890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435874736114671:2049] [node 2] ICS09 handshake done sender: [3:7574435874736114639:2103] self: [3:7574435874736114656:0] peer: [2:7574435876097962095:0] socket: 548 qp: -1
2025-11-19T13:59:40.266894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435874736114671:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:40.266900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:40.268616Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435877957252814:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.268650Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435877957252811:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:40.268654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435877957252734:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.269310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435877957252733:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.269332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:40.269339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435877957252733:2049] [node 2] ICP22 created new session: [6:7574435877957252817:2049]
2025-11-19T13:59:40.269346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435877957252817:2049] [node 2] ICS09 handshake done sender: [6:7574435877957252811:2116] self: [6:7574435877957252812:0] peer: [2:7574435876097962112:0] socket: 580 qp: -1
2025-11-19T13:59:40.269352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435877957252817:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:40.269374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:40.268836Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435874736114661:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.268844Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435874736114661:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.271479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435876097962086:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.271497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:40.271504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435876097962086:2094] [node 6] ICP22 created new session: [2:7574435876097962128:2094]
2025-11-19T13:59:40.271510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435876097962128:2094] [node 6] ICS09 handshake done sender: [2:7574435876097962121:2119] self: [2:7574435876097962112:0] peer: [6:7574435877957252812:0] socket: 563 qp: -1
2025-11-19T13:59:40.271516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435876097962128:2094] [node 6] ICS10 traffic start
2025-11-19T13:59:40.271526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-11-19T13:59:40.287074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435876585614005:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33362
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.287698Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435876585614005:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.287960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435876585614006:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33362
2025-11-19T13:59:40.288057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435876585613980:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.288061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435876585613980:2102] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.288573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435876585614008:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33362
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.288989Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435876585613980:2102] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:40.287654Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435877957252813:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.287705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435877957252819:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48215
2025-11-19T13:59:40.287823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435877957252761:2071] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.287846Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435877957252798:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.287892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435877957252760:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.289441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435877800629063:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56083
2025-11-19T13:59:40.289125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435876585613938:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.289144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:40.289151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435876585613938:2064] [node 6] ICP22 created new session: [4:7574435876585614010:2064]
2025-11-19T13:59:40.289156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435876585614010:2064] [node 6] ICS09 handshake done sender: [4:7574435876585613980:2102] self: [4:7574435876585613944:0] peer: [6:7574435877957252822:0] socket: 559 qp: -1
2025-11-19T13:59:40.289163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435876585614010:2064] [node 6] ICS10 traffic start
2025-11-19T13:59:40.289171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:40.289657Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435876097962107:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.289699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435876097962031:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.289764Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435877957252733:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(20431607ns) {HandleHandshakeStatus:329(20431246ns) {DropIncomingHandshake:441(189ns) DropOutgoingHandshake:456(5688ns) SwitchToState:178(236ns) ProcessPendingSessionEvents:495(2282656ns) {ForwardSessionEventToSession:612(2271193ns) ForwardSessionEventToSession:612(1469ns) ForwardSessionEventToSession:612(731ns) ForwardSessionEventToSession:612(612ns) ForwardSessionEventToSession:612(512ns) ForwardSessionEventToSession:612(1304ns) ForwardSessionEventToSession:612(715ns) ForwardSessionEventToSession:612(838ns) ForwardSessionEventToSession:612(658ns) ForwardSessionEventToSession:612(1756ns)}}}
2025-11-19T13:59:40.290155Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435874736114661:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:40.290355Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435877800629052:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.290363Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435877800629052:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.290631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435877957252823:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48215
2025-11-19T13:59:40.290863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435874736114649:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.290880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:40.290886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435874736114649:2075] [node 6] ICP22 created new session: [3:7574435874736114675:2075]
2025-11-19T13:59:40.290893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435874736114675:2075] [node 6] ICS09 handshake done sender: [3:7574435874736114661:2116] self: [3:7574435874736114660:0] peer: [6:7574435877957252816:0] socket: 587 qp: -1
2025-11-19T13:59:40.290899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435874736114675:2075] [node 6] ICS10 traffic start
2025-11-19T13:59:40.290907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:40.290069Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435876585613996:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.291387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435876097962131:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37451
2025-11-19T13:59:40.289448Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435874714458930:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.289533Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435874714458929:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.289944Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435874714458927:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.289965Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435874714458906:2104] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.289968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435874714458906:2104] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.291265Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435874714458906:2104] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:40.291363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435874714458848:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.291377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:40.291382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435874714458848:2048] [node 2] ICP22 created new session: [1:7574435874714458938:2048]
2025-11-19T13:59:40.291388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435874714458938:2048] [node 2] ICS09 handshake done sender: [1:7574435874714458906:2104] self: [1:7574435874714458905:0] peer: [2:7574435876097962130:0] socket: 540 qp: -1
2025-11-19T13:59:40.291393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435874714458938:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:40.291408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:40.292438Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435874736114658:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.292476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435874736114580:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.292820Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435876097962107:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:40.293829Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435874736114662:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.293027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435876097962031:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.293043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:40.293048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435876097962031:2048] [node 1] ICP22 created new session: [2:7574435876097962132:2048]
2025-11-19T13:59:40.293053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435876097962132:2048] [node 1] ICS09 handshake done sender: [2:7574435876097962107:2113] self: [2:7574435876097962130:0] peer: [1:7574435874714458905:0] socket: 541 qp: -1
2025-11-19T13:59:40.293058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435876097962132:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:40.293068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:40.294281Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435877957252798:2104] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:40.293976Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435874714458920:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.293988Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435874714458920:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.294520Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435876585614006:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.294549Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435876585614008:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.294563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435876585613920:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.294607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435876585613922:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.294616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435876585613938:2064] [node 6] ICP18 handshake (actor [4:7574435876585614008:2120]) is too old
2025-11-19T13:59:40.295151Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435874736114664:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.295610Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435874714458920:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:40.296328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435874714458849:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.296350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:40.296357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435874714458849:2049] [node 3] ICP22 created new session: [1:7574435874714458942:2049]
2025-11-19T13:59:40.296362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435874714458942:2049] [node 3] ICS09 handshake done sender: [1:7574435874714458920:2111] self: [1:7574435874714458911:0] peer: [3:7574435874736114677:0] socket: 545 qp: -1
2025-11-19T13:59:40.296368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435874714458942:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:40.296375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:40.295680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435877957252760:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.295696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:40.295702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435877957252760:2067] [node 4] ICP22 created new session: [6:7574435877957252824:2067]
2025-11-19T13:59:40.295708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435877957252824:2067] [node 4] ICS09 handshake done sender: [6:7574435877957252798:2104] self: [6:7574435877957252822:0] peer: [4:7574435876585613944:0] socket: 561 qp: -1
2025-11-19T13:59:40.295719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435877957252824:2067] [node 4] ICS10 traffic start
2025-11-19T13:59:40.295736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:40.296493Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435877800629052:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:40.296593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435877800629016:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.296600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:40.296605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435877800629016:2070] [node 6] ICP22 created new session: [5:7574435877800629065:2070]
2025-11-19T13:59:40.296610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435877800629065:2070] [node 6] ICS09 handshake done sender: [5:7574435877800629052:2115] self: [5:7574435877800629028:0] peer: [6:7574435877957252821:0] socket: 542 qp: -1
2025-11-19T13:59:40.296614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435877800629065:2070] [node 6] ICS10 traffic start
2025-11-19T13:59:40.296621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.296824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435877800629067:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56083
2025-11-19T13:59:40.297001Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435877957252814:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.297139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435877957252826:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48215
2025-11-19T13:59:40.297548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435874736114649:2075] [node 6] ICP18 handshake (actor [3:7574435874736114662:2117]) is too old
2025-11-19T13:59:40.297729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435874736114665:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.297738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435874736114665:2120] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.298024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435877800629063:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.298054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435877800628993:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.298385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435877957252734:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.298399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:40.298405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435877957252734:2050] [node 3] ICP22 created new session: [6:7574435877957252828:2050]
2025-11-19T13:59:40.298410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435877957252828:2050] [node 3] ICS09 handshake done sender: [6:7574435877957252814:2118] self: [6:7574435877957252816:0] peer: [3:7574435874736114660:0] socket: 590 qp: -1
2025-11-19T13:59:40.298415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435877957252828:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:40.298420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:40.300831Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435874714458923:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.300846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435874714458923:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.301075Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435877957252813:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:40.302465Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435877800629067:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.302503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435877800628973:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.302980Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435874714458925:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.302988Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435874714458925:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.306132Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435874714458925:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:40.307057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435874714458912:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.307079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:40.307088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435874714458912:2106] [node 5] ICP22 created new session: [1:7574435874714458946:2106]
2025-11-19T13:59:40.307097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435874714458946:2106] [node 5] ICS09 handshake done sender: [1:7574435874714458925:2113] self: [1:7574435874714458922:0] peer: [5:7574435877800629069:0] socket: 601 qp: -1
2025-11-19T13:59:40.307105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435874714458946:2106] [node 5] ICS10 traffic start
2025-11-19T13:59:40.307114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:40.307976Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435876585614004:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.307997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435876585614004:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.308345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435877957252761:2071] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.308363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:40.308367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435877957252761:2071] [node 5] ICP22 created new session: [6:7574435877957252830:2071]
2025-11-19T13:59:40.308372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435877957252830:2071] [node 5] ICS09 handshake done sender: [6:7574435877957252813:2117] self: [6:7574435877957252821:0] peer: [5:7574435877800629028:0] socket: 564 qp: -1
2025-11-19T13:59:40.308378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435877957252830:2071] [node 5] ICS10 traffic start
2025-11-19T13:59:40.308386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.310121Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435874736114665:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.310674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435876585614015:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33362
2025-11-19T13:59:40.309940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435877957252832:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48215
2025-11-19T13:59:40.311915Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435876585614006:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:40.310541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435877800629070:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56083
2025-11-19T13:59:40.310559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435877800629071:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56083
2025-11-19T13:59:40.312547Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435874736114580:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9308) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435874736114658:2115] held: no
2025-11-19T13:59:40.312582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435874736114648:2079] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.312600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:40.312606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435874736114648:2079] [node 4] ICP22 created new session: [3:7574435874736114680:2079]
2025-11-19T13:59:40.312612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435874736114680:2079] [node 4] ICS09 handshake done sender: [3:7574435874736114665:2120] self: [3:7574435874736114659:0] peer: [4:7574435876585614014:0] socket: 594 qp: -1
2025-11-19T13:59:40.312619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435874736114680:2079] [node 4] ICS10 traffic start
2025-11-19T13:59:40.312628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:40.313122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435874714458849:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(16759829ns) {HandleHandshakeStatus:329(16759096ns) {DropIncomingHandshake:441(3722ns) DropOutgoingHandshake:456(153ns) SwitchToState:178(47ns) ProcessPendingSessionEvents:495(16679058ns) {ForwardSessionEventToSession:612(16666774ns) ForwardSessionEventToSession:612(1575ns) ForwardSessionEventToSession:612(817ns) ForwardSessionEventToSession:612(635ns) ForwardSessionEventToSession:612(530ns) ForwardSessionEventToSession:612(921ns) ForwardSessionEventToSession:612(613ns) ForwardSessionEventToSession:612(642ns) ForwardSessionEventToSession:612(648ns) ForwardSessionEventToSession:612(731ns)}}}
2025-11-19T13:59:40.313909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435876585613922:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.313935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:40.313941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435876585613922:2050] [node 3] ICP22 created new session: [4:7574435876585614016:2050]
2025-11-19T13:59:40.313948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435876585614016:2050] [node 3] ICS09 handshake done sender: [4:7574435876585614006:2119] self: [4:7574435876585614014:0] peer: [3:7574435874736114659:0] socket: 576 qp: -1
2025-11-19T13:59:40.313953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435876585614016:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:40.313972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:40.313867Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435877800629067:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.314404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435874736114682:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26907
2025-11-19T13:59:40.317647Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435877957252832:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.317696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435877957252732:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.318793Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435874736114658:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:40.318949Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435874714458926:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.318960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435874714458926:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.320224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435877957252834:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48215
2025-11-19T13:59:40.320749Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435874714458926:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:40.320421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435874736114580:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.320441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:40.320450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435874736114580:2048] [node 1] ICP22 created new session: [3:7574435874736114683:2048]
2025-11-19T13:59:40.320456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435874736114683:2048] [node 1] ICS09 handshake done sender: [3:7574435874736114658:2115] self: [3:7574435874736114677:0] peer: [1:7574435874714458911:0] socket: 549 qp: -1
2025-11-19T13:59:40.320463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435874736114683:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:40.320475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:40.320887Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435877957252832:2123] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:40.320973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435877957252732:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.320991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:40.320995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435877957252732:2048] [node 1] ICP22 created new session: [6:7574435877957252835:2048]
2025-11-19T13:59:40.321000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435877957252835:2048] [node 1] ICS09 handshake done sender: [6:7574435877957252832:2123] self: [6:7574435877957252833:0] peer: [1:7574435874714458917:0] socket: 605 qp: -1
2025-11-19T13:59:40.321005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435877957252835:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:40.321012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:40.321291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435874714458916:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.321305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:40.321309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435874714458916:2109] [node 6] ICP22 created new session: [1:7574435874714458949:2109]
2025-11-19T13:59:40.321314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435874714458949:2109] [node 6] ICS09 handshake done sender: [1:7574435874714458926:2114] self: [1:7574435874714458917:0] peer: [6:7574435877957252833:0] socket: 574 qp: -1
2025-11-19T13:59:40.321317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435874714458949:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:40.321322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:40.321801Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435876585614004:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:40.321888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435876585613987:2084] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.321900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:40.321905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435876585613987:2084] [node 5] ICP22 created new session: [4:7574435876585614019:2084]
2025-11-19T13:59:40.321909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435876585614019:2084] [node 5] ICS09 handshake done sender: [4:7574435876585614004:2117] self: [4:7574435876585614003:0] peer: [5:7574435877800629068:0] socket: 597 qp: -1
2025-11-19T13:59:40.321914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435876585614019:2084] [node 5] ICS10 traffic start
2025-11-19T13:59:40.321919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.324591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435877800628973:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9308) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435877800629067:2120] held: no
2025-11-19T13:59:40.324685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435877800628973:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.324694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:40.324700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435877800628973:2048] [node 1] ICP22 created new session: [5:7574435877800629073:2048]
2025-11-19T13:59:40.324705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435877800629073:2048] [node 1] ICS09 handshake done sender: [5:7574435877800629067:2120] self: [5:7574435877800629069:0] peer: [1:7574435874714458922:0] socket: 602 qp: -1
2025-11-19T13:59:40.324709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435877800629073:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:40.324728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:40.324856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435877800629072:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56083
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.328163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435876585614021:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33362
2025-11-19T13:59:40.328429Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435874714458923:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:40.328685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435874714458915:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.328710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:40.328716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435874714458915:2108] [node 4] ICP22 created new session: [1:7574435874714458952:2108]
2025-11-19T13:59:40.328721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435874714458952:2108] [node 4] ICS09 handshake done sender: [1:7574435874714458923:2112] self: [1:7574435874714458918:0] peer: [4:7574435876585614013:0] socket: 567 qp: -1
2025-11-19T13:59:40.328727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435874714458952:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:40.328736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:40.329546Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435876585613996:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:40.329649Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435877800628993:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21122) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435877800629063:2119] held: no
2025-11-19T13:59:40.329814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435876585613920:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.329829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:40.329837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435876585613920:2048] [node 1] ICP22 created new session: [4:7574435876585614022:2048]
2025-11-19T13:59:40.329843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435876585614022:2048] [node 1] ICS09 handshake done sender: [4:7574435876585613996:2114] self: [4:7574435876585614013:0] peer: [1:7574435874714458918:0] socket: 568 qp: -1
2025-11-19T13:59:40.329848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435876585614022:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:40.329854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:40.332179Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435877800629063:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:40.332326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435877800628993:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.332339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:40.332344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435877800628993:2063] [node 4] ICP22 created new session: [5:7574435877800629075:2063]
2025-11-19T13:59:40.332349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435877800629075:2063] [node 4] ICS09 handshake done sender: [5:7574435877800629063:2119] self: [5:7574435877800629068:0] peer: [4:7574435876585614003:0] socket: 598 qp: -1
2025-11-19T13:59:40.332355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435877800629075:2063] [node 4] ICS10 traffic start
2025-11-19T13:59:40.332364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:40.348004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:40.348108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435876097962132:2048] [node 1] ICS01 socket: 541 reason# EndOfStream
2025-11-19T13:59:40.348118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435876097962031:2048] [node 1] ICP30 unregister session Session# [2:7574435876097962132:2048] VirtualId# [2:7574435876097962130:0]
2025-11-19T13:59:40.348125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435876097962132:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:40.348193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:40.348218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:40.348286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:40.348348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435874736114683:2048] [node 1] ICS01 socket: 549 reason# EndOfStream
2025-11-19T13:59:40.348354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435874736114580:2048] [node 1] ICP30 unregister session Session# [3:7574435874736114683:2048] VirtualId# [3:7574435874736114677:0]
2025-11-19T13:59:40.348358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435874736114683:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:40.348385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:40.348405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:40.350587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:40.350667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435877800629073:2048] [node 1] ICS01 socket: 602 reason# EndOfStream
2025-11-19T13:59:40.350676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435877800628973:2048] [node 1] ICP30 unregister session Session# [5:7574435877800629073:2048] VirtualId# [5:7574435877800629069:0]
2025-11-19T13:59:40.350681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435877800629073:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:40.350726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:40.350747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:40.350774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:40.350835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435876585614022:2048] [node 1] ICS01 socket: 568 reason# EndOfStream
2025-11-19T13:59:40.350839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435876585613920:2048] [node 1] ICP30 unregister session Session# [4:7574435876585614022:2048] VirtualId# [4:7574435876585614013:0]
2025-11-19T13:59:40.350843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435876585614022:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:40.350879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:40.350896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:40.350935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:40.351001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435877957252835:2048] [node 1] ICS01 socket: 605 reason# EndOfStream
2025-11-19T13:59:40.351008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435877957252732:2048] [node 1] ICP30 unregister session Session# [6:7574435877957252835:2048] VirtualId# [6:7574435877957252833:0]
2025-11-19T13:59:40.351013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435877957252835:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:40.351036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:40.351063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:40.360565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:40.360646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435874736114671:2049] [node 2] ICS01 socket: 548 reason# EndOfStream
2025-11-19T13:59:40.360654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435874736114581:2049] [node 2] ICP30 unregister session Session# [3:7574435874736114671:2049] VirtualId# [3:7574435874736114656:0]
2025-11-19T13:59:40.360660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435874736114671:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:40.360692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:40.360711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:40.364337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:40.364409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435877800629055:2049] [node 2] ICS07 socket disconnect 554 reason# EndOfStream
2025-11-19T13:59:40.364414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435877800629055:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:40.364448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:40.364469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435877800629055:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:40.363946Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:40.364030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435876585614000:2049] [node 2] ICS07 socket disconnect 556 reason# EndOfStream
2025-11-19T13:59:40.364036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435876585614000:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:40.364086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:40.364109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435876585614000:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:40.364629Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:40.364690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435877957252817:2049] [node 2] ICS07 socket disconnect 580 reason# EndOfStream
2025-11-19T13:59:40.364693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435877957252817:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:40.364720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:40.364738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435877957252817:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:40.365417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435877957252733:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19346) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:40.365429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435877957252817:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:40.365433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435877957252733:2049] [node 2] ICP30 unregister session Session# [6:7574435877957252817:2049] VirtualId# [6:7574435877957252812:0]
2025-11-19T13:59:40.365447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:40.365452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435877957252733:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:40.365454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:40.365675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435877800628974:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19346) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:40.365687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435877800629055:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:40.365690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435877800628974:2049] [node 2] ICP30 unregister session Session# [5:7574435877800629055:2049] VirtualId# [5:7574435877800629045:0]
2025-11-19T13:59:40.365699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:40.365703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435877800628974:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:40.365704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:40.365979Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435876585613921:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19346) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:40.365990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435876585614000:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:40.365993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435876585613921:2049] [node 2] ICP30 unregister session Session# [4:7574435876585614000:2049] VirtualId# [4:7574435876585613997:0]
2025-11-19T13:59:40.366002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:40.366005Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435876585613921:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:40.366007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:40.371999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435877800628974:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:40.378202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435876585613921:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:40.378314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435877957252733:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:40.387872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:40.387976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435877800629060:2050] [node 3] ICS07 socket disconnect 553 reason# EndOfStream
2025-11-19T13:59:40.387983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435877800629060:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:40.388040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:40.388062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435877800629060:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:40.390368Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:40.390452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435877957252828:2050] [node 3] ICS01 socket: 590 reason# EndOfStream
2025-11-19T13:59:40.390460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435877957252734:2050] [node 3] ICP30 unregister session Session# [6:7574435877957252828:2050] VirtualId# [6:7574435877957252816:0]
2025-11-19T13:59:40.390466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435877957252828:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:40.390516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:40.390540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:40.398196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435877800628975:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7017) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:40.398218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435877800629060:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:40.398225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435877800628975:2050] [node 3] ICP30 unregister session Session# [5:7574435877800629060:2050] VirtualId# [5:7574435877800629053:0]
2025-11-19T13:59:40.398244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:40.398250Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435877800628975:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:40.398254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7017) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:40.410141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:40.410235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435877957252824:2067] [node 4] ICS01 socket: 561 reason# EndOfStream
2025-11-19T13:59:40.410244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435877957252760:2067] [node 4] ICP30 unregister session Session# [6:7574435877957252824:2067] VirtualId# [6:7574435877957252822:0]
2025-11-19T13:59:40.410251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435877957252824:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:40.410306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:40.410330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:40.545482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435875329108854:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:40.545498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435875329108855:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:40.634521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435876823461700:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:40.634537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435876823461701:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:40.688415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435875407049917:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:40.688430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435875407049918:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:40.728841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435874856825424:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:40.728857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435874856825425:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:40.728862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435874856825426:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:40.804357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435878837725033:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:40.804372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435878837725034:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:40.804376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435878837725035:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:40.836749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435878179819162:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:40.836763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435878179819163:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:40.836767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435878179819164:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:40.860115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435878179819178:2062] [node 4] ICP01 ready to work
2025-11-19T13:59:40.860585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435874856825455:2072] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.861666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435878179819186:2068] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57690
2025-11-19T13:59:40.862028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435878837725066:2076] [node 4] ICP01 ready to work
2025-11-19T13:59:40.860030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435875329108918:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:40.860131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435875329108919:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:40.864309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435875407049978:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:40.864028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435876823461761:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:40.867135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435878179819195:2072] [node 5] ICP01 ready to work
2025-11-19T13:59:40.875831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435878837725086:2090] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.879603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435875407049979:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56432
2025-11-19T13:59:40.879624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435875407049980:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56432
2025-11-19T13:59:40.880144Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435876823461700:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16062537ns) {Configure:104(16061795ns) {ProcessConfigured:135(15720953ns) {StartInitialHandshake:157(15720453ns) {DropHandshakes:469(628ns) {DropIncomingHandshake:441(149ns) DropOutgoingHandshake:456(126ns)} GenerateSessionVirtualId:988(154ns) PrepareNewSessionHandshake:476(853ns) {SwitchToState:178(246ns)}}}}}
2025-11-19T13:59:40.880794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435876823461768:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56921
2025-11-19T13:59:40.882413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435876823461773:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.892118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435874856825497:2079] [node 5] ICP01 ready to work
2025-11-19T13:59:40.893779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435874856825494:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14188
2025-11-19T13:59:40.893801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435874856825495:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14188
2025-11-19T13:59:40.893818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435874856825496:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14188
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.894406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435876823461777:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:40.897524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435875407049992:2113] [node 4] ICP01 ready to work
2025-11-19T13:59:40.898908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435875407049991:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56432
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.915903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435878179819163:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(48696838ns) {Configure:104(48695509ns) {ProcessConfigured:135(48379299ns) {StartInitialHandshake:157(48378666ns) {DropHandshakes:469(608ns) {DropIncomingHandshake:441(161ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(107ns) PrepareNewSessionHandshake:476(1781ns) {SwitchToState:178(981ns)}}}}}
2025-11-19T13:59:40.917795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435878179819241:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57690
2025-11-19T13:59:40.928432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435875407049996:2116] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.935231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435875329108930:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.936464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435875407049999:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56432
2025-11-19T13:59:40.936617Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435875407049979:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.940869Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435876823461768:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.940946Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435874856825496:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.940999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435874856825425:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.940152Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435875407049991:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.940242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435875407049918:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.941876Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435876823461765:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.941887Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435876823461765:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.943405Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435878179819241:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.943537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435878179819164:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.945927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435875329108925:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60748
2025-11-19T13:59:40.943783Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435874856825494:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.948524Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435878179819186:2068] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.948590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435878179819178:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.946910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435878837725111:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15704
2025-11-19T13:59:40.951597Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435875407049986:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.951618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435875407049986:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.951857Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435874856825495:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.956961Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435878837725111:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.957005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435878837725066:2076] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.953502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435875329108926:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60748
2025-11-19T13:59:40.953631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435875329108927:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60748
2025-11-19T13:59:40.953682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435875329108928:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60748
2025-11-19T13:59:40.953727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435875329108929:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60748
2025-11-19T13:59:40.953980Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435875329108925:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.957469Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435876823461767:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.957482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435876823461767:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.962437Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435874856825457:2077] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.962461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435874856825457:2077] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.961860Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435875329108927:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.962522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435878179819244:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57690
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.964827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435878179819245:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57690
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.965662Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435874856825499:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.965674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435874856825499:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.966443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435874856825508:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14188
2025-11-19T13:59:40.966456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435874856825509:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14188
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.966647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435878179819246:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57690
2025-11-19T13:59:40.966817Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435876823461765:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:40.967099Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435876823461767:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.967471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435878837725113:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15704
2025-11-19T13:59:40.967272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435876823461701:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.967302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:40.967312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435876823461701:2049] [node 3] ICP22 created new session: [2:7574435876823461783:2049]
2025-11-19T13:59:40.967320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435876823461783:2049] [node 3] ICS09 handshake done sender: [2:7574435876823461765:2109] self: [2:7574435876823461764:0] peer: [3:7574435875407050001:0] socket: 564 qp: -1
2025-11-19T13:59:40.967328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435876823461783:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:40.967342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:40.967515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435876823461761:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.967523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:40.967529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435876823461761:2107] [node 4] ICP22 created new session: [2:7574435876823461785:2107]
2025-11-19T13:59:40.967633Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435875407049986:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:40.967955Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435875407049980:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.967984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435875407049978:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.968003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:40.968011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435875407049978:2107] [node 6] ICP22 created new session: [3:7574435875407050004:2107]
2025-11-19T13:59:40.968017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435875407050004:2107] [node 6] ICS09 handshake done sender: [3:7574435875407049986:2112] self: [3:7574435875407049985:0] peer: [6:7574435878179819242:0] socket: 561 qp: -1
2025-11-19T13:59:40.968023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435875407050004:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:40.968033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.968749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435875407050003:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56432
2025-11-19T13:59:40.968900Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435874856825457:2077] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:40.971447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435876823461785:2107] [node 4] ICS09 handshake done sender: [2:7574435876823461767:2110] self: [2:7574435876823461766:0] peer: [4:7574435874856825505:0] socket: 565 qp: -1
2025-11-19T13:59:40.971467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435876823461785:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:40.971482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:40.973125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435878179819164:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28892) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435878179819241:2116] held: no
2025-11-19T13:59:40.973199Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435878179819241:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:40.973535Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435878179819186:2068] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.973626Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435878179819244:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.973636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435878179819164:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.973643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:40.973649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435878179819164:2050] [node 3] ICP22 created new session: [6:7574435878179819248:2050]
2025-11-19T13:59:40.973653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435878179819248:2050] [node 3] ICS09 handshake done sender: [6:7574435878179819241:2116] self: [6:7574435878179819242:0] peer: [3:7574435875407049985:0] socket: 569 qp: -1
2025-11-19T13:59:40.973658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435878179819248:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:40.973663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:40.973779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435878179819178:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.973786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:40.973789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435878179819178:2062] [node 4] ICP22 created new session: [6:7574435878179819251:2062]
2025-11-19T13:59:40.973793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435878179819251:2062] [node 4] ICS09 handshake done sender: [6:7574435878179819186:2068] self: [6:7574435878179819243:0] peer: [4:7574435874856825456:0] socket: 549 qp: -1
2025-11-19T13:59:40.973797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435878179819251:2062] [node 4] ICS10 traffic start
2025-11-19T13:59:40.973802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:40.973941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435878179819247:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57690
2025-11-19T13:59:40.970693Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435874856825499:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:40.972266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435874856825455:2072] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.972286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:40.972293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435874856825455:2072] [node 6] ICP22 created new session: [4:7574435874856825513:2072]
2025-11-19T13:59:40.972300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435874856825513:2072] [node 6] ICS09 handshake done sender: [4:7574435874856825457:2077] self: [4:7574435874856825456:0] peer: [6:7574435878179819243:0] socket: 548 qp: -1
2025-11-19T13:59:40.972306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435874856825513:2072] [node 6] ICS10 traffic start
2025-11-19T13:59:40.972317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:40.972459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435874856825497:2079] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.972469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:40.972474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435874856825497:2079] [node 5] ICP22 created new session: [4:7574435874856825515:2079]
2025-11-19T13:59:40.972479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435874856825515:2079] [node 5] ICS09 handshake done sender: [4:7574435874856825499:2115] self: [4:7574435874856825498:0] peer: [5:7574435878837725112:0] socket: 576 qp: -1
2025-11-19T13:59:40.972483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435874856825515:2079] [node 5] ICS10 traffic start
2025-11-19T13:59:40.972487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:40.970984Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435875407049991:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:40.971068Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435875329108929:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.971101Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435875329108928:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.971200Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435875329108926:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.976734Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435878837725111:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:40.979934Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435874856825496:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.980225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435874856825508:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.980277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435874856825512:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14188
2025-11-19T13:59:40.980314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435874856825425:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.980326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:40.980332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435874856825425:2049] [node 2] ICP22 created new session: [4:7574435874856825519:2049]
2025-11-19T13:59:40.980337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435874856825519:2049] [node 2] ICS09 handshake done sender: [4:7574435874856825496:2114] self: [4:7574435874856825505:0] peer: [2:7574435876823461766:0] socket: 568 qp: -1
2025-11-19T13:59:40.980343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435874856825519:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:40.980350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:40.980904Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435875407049999:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.981006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435874856825424:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.982568Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435875329108921:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.982584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435875329108921:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.982762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435875407049918:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.982791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:40.982801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435875407049918:2049] [node 2] ICP22 created new session: [3:7574435875407050008:2049]
2025-11-19T13:59:40.982809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435875407050008:2049] [node 2] ICS09 handshake done sender: [3:7574435875407049991:2114] self: [3:7574435875407050001:0] peer: [2:7574435876823461764:0] socket: 566 qp: -1
2025-11-19T13:59:40.982818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435875407050008:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:40.982827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:40.982901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435875407049918:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:40.983434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435878179819162:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.986235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435876823461789:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56921
2025-11-19T13:59:40.986260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435876823461790:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56921
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.987069Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435875329108924:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.987084Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435875329108924:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.988706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435875407049917:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.988831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435875407050010:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56432
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.990478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435878837725066:2076] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.990501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:40.990509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435878837725066:2076] [node 4] ICP22 created new session: [5:7574435878837725115:2076]
2025-11-19T13:59:40.990517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435878837725115:2076] [node 4] ICS09 handshake done sender: [5:7574435878837725111:2115] self: [5:7574435878837725112:0] peer: [4:7574435874856825498:0] socket: 577 qp: -1
2025-11-19T13:59:40.990525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435878837725115:2076] [node 4] ICS10 traffic start
2025-11-19T13:59:40.990534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:40.990639Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435878837725066:2076] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:40.990742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435878837725114:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15704
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.992128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435874856825522:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14188
2025-11-19T13:59:40.993861Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435875329108921:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.994157Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435874856825508:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:40.993982Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435878179819247:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.994102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435878179819163:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:40.994125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435878179819255:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57690
2025-11-19T13:59:40.994248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435874856825424:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:40.994274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:40.994282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435874856825424:2048] [node 1] ICP22 created new session: [4:7574435874856825523:2048]
2025-11-19T13:59:40.994290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435874856825523:2048] [node 1] ICS09 handshake done sender: [4:7574435874856825508:2118] self: [4:7574435874856825521:0] peer: [1:7574435875329108920:0] socket: 585 qp: -1
2025-11-19T13:59:40.994298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435874856825523:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:40.994307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:40.994420Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435876823461779:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.994431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435876823461779:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:40.995624Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435876823461779:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:40.995708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435876823461777:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:40.995723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:40.995729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435876823461777:2113] [node 6] ICP22 created new session: [2:7574435876823461792:2113]
2025-11-19T13:59:40.995735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435876823461792:2113] [node 6] ICS09 handshake done sender: [2:7574435876823461779:2115] self: [2:7574435876823461778:0] peer: [6:7574435878179819256:0] socket: 593 qp: -1
2025-11-19T13:59:40.995741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435876823461792:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:40.995763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.996093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435878179819257:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57690
2025-11-19T13:59:40.996172Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435876823461789:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:40.996193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435876823461777:2113] [node 6] ICP18 handshake (actor [2:7574435876823461789:2116]) is too old
2025-11-19T13:59:40.997668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435878179819163:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23006) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435878179819247:2120] held: no
2025-11-19T13:59:40.997699Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435878179819247:2120] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:40.999928Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435878837725114:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.000171Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435875407050010:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.000013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435878837725117:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15704
2025-11-19T13:59:41.000082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435878837725118:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15704
2025-11-19T13:59:41.000228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435878837725034:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.000535Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435875329108924:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.001905Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435876823461776:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.001918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435876823461776:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.002782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435876823461794:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56921
2025-11-19T13:59:41.003168Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435876823461776:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:41.003538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435876823461773:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.003556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:41.003563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435876823461773:2112] [node 5] ICP22 created new session: [2:7574435876823461796:2112]
2025-11-19T13:59:41.003569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435876823461796:2112] [node 5] ICS09 handshake done sender: [2:7574435876823461776:2114] self: [2:7574435876823461774:0] peer: [5:7574435878837725120:0] socket: 562 qp: -1
2025-11-19T13:59:41.003576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435876823461796:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:41.003586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:41.003922Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435875329108916:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.003938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435875329108916:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.004982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435875329108918:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.005003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:41.005011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435875329108918:2107] [node 4] ICP22 created new session: [1:7574435879624076241:2107]
2025-11-19T13:59:41.005019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435879624076241:2107] [node 4] ICS09 handshake done sender: [1:7574435875329108921:2110] self: [1:7574435875329108920:0] peer: [4:7574435874856825521:0] socket: 578 qp: -1
2025-11-19T13:59:41.005028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435879624076241:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:41.005039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:41.005243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435875329108919:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.005257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:41.005262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435875329108919:2109] [node 6] ICP22 created new session: [1:7574435879624076243:2109]
2025-11-19T13:59:41.005267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435879624076243:2109] [node 6] ICS09 handshake done sender: [1:7574435875329108924:2112] self: [1:7574435875329108922:0] peer: [6:7574435878179819253:0] socket: 579 qp: -1
2025-11-19T13:59:41.005271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435879624076243:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:41.005277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:41.006223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435878179819163:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.006889Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435874856825512:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.007068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435874856825426:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.007065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:41.007079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435878179819163:2049] [node 2] ICP22 created new session: [6:7574435882474786554:2049]
2025-11-19T13:59:41.007089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435882474786554:2049] [node 2] ICS09 handshake done sender: [6:7574435878179819247:2120] self: [6:7574435878179819256:0] peer: [2:7574435876823461778:0] socket: 550 qp: -1
2025-11-19T13:59:41.007097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435882474786554:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:41.007108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:41.007280Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435878179819244:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:41.007429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435875407049994:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.007443Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435875407049994:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.009133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435881118429094:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56921
2025-11-19T13:59:41.009274Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435875407049994:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:41.009433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435875407049992:2113] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.009453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:41.009460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435875407049992:2113] [node 4] ICP22 created new session: [3:7574435879702017312:2113]
2025-11-19T13:59:41.009466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435879702017312:2113] [node 4] ICS09 handshake done sender: [3:7574435875407049994:2115] self: [3:7574435875407049993:0] peer: [4:7574435879151792821:0] socket: 554 qp: -1
2025-11-19T13:59:41.009472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435879702017312:2113] [node 4] ICS10 traffic start
2025-11-19T13:59:41.009481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.009892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435879702017311:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56432
2025-11-19T13:59:41.010218Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435878837725118:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.010409Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435878837725117:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.011891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435878837725035:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.011941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435883132692417:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15704
2025-11-19T13:59:41.014227Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435875329108916:2108] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.014494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435883132692418:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15704
2025-11-19T13:59:41.014635Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435878837725114:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:41.014395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435875329108855:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.014417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:41.014431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435875329108855:2049] [node 3] ICP22 created new session: [1:7574435879624076248:2049]
2025-11-19T13:59:41.014440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435879624076248:2049] [node 3] ICS09 handshake done sender: [1:7574435875329108916:2108] self: [1:7574435875329108914:0] peer: [3:7574435875407050011:0] socket: 574 qp: -1
2025-11-19T13:59:41.014448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435879624076248:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:41.014458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:41.014785Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435876823461794:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.014820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435876823461773:2112] [node 5] ICP18 handshake (actor [2:7574435876823461794:2118]) is too old
2025-11-19T13:59:41.015936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435878837725034:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.015959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:41.015967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435878837725034:2049] [node 2] ICP22 created new session: [5:7574435883132692421:2049]
2025-11-19T13:59:41.015974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435883132692421:2049] [node 2] ICS09 handshake done sender: [5:7574435878837725114:2117] self: [5:7574435878837725120:0] peer: [2:7574435876823461774:0] socket: 553 qp: -1
2025-11-19T13:59:41.015985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435883132692421:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:41.015993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:41.016856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435878837725034:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.018136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435874856825426:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28892) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435874856825512:2120] held: no
2025-11-19T13:59:41.018217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435879151792822:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14188
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.018993Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435874856825512:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:41.019087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435874856825426:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.019102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:41.019108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435874856825426:2050] [node 3] ICP22 created new session: [4:7574435879151792823:2050]
2025-11-19T13:59:41.019114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435879151792823:2050] [node 3] ICS09 handshake done sender: [4:7574435874856825512:2120] self: [4:7574435879151792821:0] peer: [3:7574435875407049993:0] socket: 592 qp: -1
2025-11-19T13:59:41.019120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435879151792823:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:41.019129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:41.019511Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435875407049998:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.019531Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435875407049998:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.018401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435878179819162:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.018430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:41.018436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435878179819162:2048] [node 1] ICP22 created new session: [6:7574435882474786556:2048]
2025-11-19T13:59:41.018444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435882474786556:2048] [node 1] ICS09 handshake done sender: [6:7574435878179819244:2117] self: [6:7574435878179819253:0] peer: [1:7574435875329108922:0] socket: 580 qp: -1
2025-11-19T13:59:41.018451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435882474786556:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:41.018460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:41.018544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435878179819162:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:41.018752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435882474786558:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57690
2025-11-19T13:59:41.021474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435875407049917:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19693) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435875407049999:2118] held: no
2025-11-19T13:59:41.021525Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435875407049999:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:41.021777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435875407049917:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.021788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:41.021794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435875407049917:2048] [node 1] ICP22 created new session: [3:7574435879702017315:2048]
2025-11-19T13:59:41.021800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435879702017315:2048] [node 1] ICS09 handshake done sender: [3:7574435875407049999:2118] self: [3:7574435875407050011:0] peer: [1:7574435875329108914:0] socket: 575 qp: -1
2025-11-19T13:59:41.021807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435879702017315:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:41.021812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:41.022036Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435875407049998:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:41.022135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435875407049996:2116] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.022150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:41.022158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435875407049996:2116] [node 5] ICP22 created new session: [3:7574435879702017317:2116]
2025-11-19T13:59:41.022163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435879702017317:2116] [node 5] ICS09 handshake done sender: [3:7574435875407049998:2117] self: [3:7574435875407049997:0] peer: [5:7574435883132692419:0] socket: 598 qp: -1
2025-11-19T13:59:41.022169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435879702017317:2116] [node 5] ICS10 traffic start
2025-11-19T13:59:41.022176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:41.029188Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435881118429094:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.029238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435876823461700:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.027980Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435878837725035:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28892) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435878837725118:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.031202Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435883132692418:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.031258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435883132692424:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15704
2025-11-19T13:59:41.035800Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435875329108935:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.035826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435875329108935:2119] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.037845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435881118429096:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56921
2025-11-19T13:59:41.038488Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435875329108935:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:41.038735Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435881118429094:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:41.037126Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435878837725118:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:41.038817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435876823461700:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.038839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:41.038847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435876823461700:2048] [node 1] ICP22 created new session: [2:7574435881118429097:2048]
2025-11-19T13:59:41.038855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435881118429097:2048] [node 1] ICS09 handshake done sender: [2:7574435881118429094:2119] self: [2:7574435881118429095:0] peer: [1:7574435875329108934:0] socket: 606 qp: -1
2025-11-19T13:59:41.038863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435881118429097:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:41.038874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:41.040843Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435882474786558:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.040880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435878179819195:2072] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.041537Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435875329108933:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.041548Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435875329108933:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.042287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435875329108854:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.042303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:41.042312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435875329108854:2048] [node 2] ICP22 created new session: [1:7574435879624076251:2048]
2025-11-19T13:59:41.042323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435879624076251:2048] [node 2] ICS09 handshake done sender: [1:7574435875329108935:2119] self: [1:7574435875329108934:0] peer: [2:7574435881118429095:0] socket: 605 qp: -1
2025-11-19T13:59:41.042330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435879624076251:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:41.042337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:41.043185Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435875329108933:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:41.043274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435875329108930:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.043289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:41.043297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435875329108930:2111] [node 5] ICP22 created new session: [1:7574435879624076254:2111]
2025-11-19T13:59:41.043304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435879624076254:2111] [node 5] ICS09 handshake done sender: [1:7574435875329108933:2118] self: [1:7574435875329108932:0] peer: [5:7574435883132692425:0] socket: 571 qp: -1
2025-11-19T13:59:41.043311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435879624076254:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:41.043319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:41.040469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435878837725033:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.040552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435878837725035:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.040562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:41.040568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435878837725035:2050] [node 3] ICP22 created new session: [5:7574435883132692426:2050]
2025-11-19T13:59:41.040574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435883132692426:2050] [node 3] ICS09 handshake done sender: [5:7574435878837725118:2119] self: [5:7574435883132692419:0] peer: [3:7574435875407049997:0] socket: 599 qp: -1
2025-11-19T13:59:41.040580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435883132692426:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:41.040588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:41.044161Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435878837725092:2098] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.044169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435878837725092:2098] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.047300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435878837725033:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19693) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435883132692418:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.047577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435883132692428:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15704
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.047650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435882474786560:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57690
2025-11-19T13:59:41.047797Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435878837725092:2098] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:41.048277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435878837725086:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.048299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:41.048305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435878837725086:2090] [node 6] ICP22 created new session: [5:7574435883132692430:2090]
2025-11-19T13:59:41.048311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435883132692430:2090] [node 6] ICS09 handshake done sender: [5:7574435878837725092:2098] self: [5:7574435878837725091:0] peer: [6:7574435882474786559:0] socket: 551 qp: -1
2025-11-19T13:59:41.048316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435883132692430:2090] [node 6] ICS10 traffic start
2025-11-19T13:59:41.048323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:41.048452Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435883132692418:2121] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:41.052069Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435882474786558:2123] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:41.052167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435878179819195:2072] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.052182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:41.052191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435878179819195:2072] [node 5] ICP22 created new session: [6:7574435882474786561:2072]
2025-11-19T13:59:41.052197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435882474786561:2072] [node 5] ICS09 handshake done sender: [6:7574435882474786558:2123] self: [6:7574435882474786559:0] peer: [5:7574435878837725091:0] socket: 552 qp: -1
2025-11-19T13:59:41.052203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435882474786561:2072] [node 5] ICS10 traffic start
2025-11-19T13:59:41.052214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:41.053944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435878837725033:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.053960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:41.053969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435878837725033:2048] [node 1] ICP22 created new session: [5:7574435883132692432:2048]
2025-11-19T13:59:41.053978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435883132692432:2048] [node 1] ICS09 handshake done sender: [5:7574435883132692418:2121] self: [5:7574435883132692425:0] peer: [1:7574435875329108932:0] socket: 608 qp: -1
2025-11-19T13:59:41.053986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435883132692432:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:41.053995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:41.066406Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:41.066532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435881118429097:2048] [node 1] ICS01 socket: 606 reason# EndOfStream
2025-11-19T13:59:41.066544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435876823461700:2048] [node 1] ICP30 unregister session Session# [2:7574435881118429097:2048] VirtualId# [2:7574435881118429095:0]
2025-11-19T13:59:41.066553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435881118429097:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.066608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:41.066634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:41.068251Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:41.068353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435879702017315:2048] [node 1] ICS01 socket: 575 reason# EndOfStream
2025-11-19T13:59:41.068360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435875407049917:2048] [node 1] ICP30 unregister session Session# [3:7574435879702017315:2048] VirtualId# [3:7574435875407050011:0]
2025-11-19T13:59:41.068365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435879702017315:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.068393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:41.068412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:41.072704Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:41.072792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435874856825523:2048] [node 1] ICS01 socket: 585 reason# EndOfStream
2025-11-19T13:59:41.072807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435874856825424:2048] [node 1] ICP30 unregister session Session# [4:7574435874856825523:2048] VirtualId# [4:7574435874856825521:0]
2025-11-19T13:59:41.072815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435874856825523:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.072877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:41.072906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:41.073120Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:41.073185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435882474786556:2048] [node 1] ICS01 socket: 580 reason# EndOfStream
2025-11-19T13:59:41.073191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435878179819162:2048] [node 1] ICP30 unregister session Session# [6:7574435882474786556:2048] VirtualId# [6:7574435878179819253:0]
2025-11-19T13:59:41.073197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435882474786556:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.073242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:41.073271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:41.073304Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:41.073368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435883132692432:2048] [node 1] ICS01 socket: 608 reason# EndOfStream
2025-11-19T13:59:41.073373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435878837725033:2048] [node 1] ICP30 unregister session Session# [5:7574435883132692432:2048] VirtualId# [5:7574435883132692425:0]
2025-11-19T13:59:41.073378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435883132692432:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.073415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:41.073439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:41.080114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:41.080216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435875407050008:2049] [node 2] ICS01 socket: 566 reason# EndOfStream
2025-11-19T13:59:41.080226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435875407049918:2049] [node 2] ICP30 unregister session Session# [3:7574435875407050008:2049] VirtualId# [3:7574435875407050001:0]
2025-11-19T13:59:41.080234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435875407050008:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.080275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:41.080298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:41.083836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:41.083931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435874856825519:2049] [node 2] ICS01 socket: 568 reason# EndOfStream
2025-11-19T13:59:41.083940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435874856825425:2049] [node 2] ICP30 unregister session Session# [4:7574435874856825519:2049] VirtualId# [4:7574435874856825505:0]
2025-11-19T13:59:41.083947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435874856825519:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.083999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:41.084024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:41.086910Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:41.087029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435883132692421:2049] [node 2] ICS07 socket disconnect 553 reason# EndOfStream
2025-11-19T13:59:41.087038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435883132692421:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.087121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:41.087152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435883132692421:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:41.087222Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:41.087310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435882474786554:2049] [node 2] ICS01 socket: 550 reason# EndOfStream
2025-11-19T13:59:41.087319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435878179819163:2049] [node 2] ICP30 unregister session Session# [6:7574435882474786554:2049] VirtualId# [6:7574435878179819256:0]
2025-11-19T13:59:41.087325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435882474786554:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.087353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:41.087380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:41.098455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435878837725034:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23006) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:41.098480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435883132692421:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:41.098487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435878837725034:2049] [node 2] ICP30 unregister session Session# [5:7574435883132692421:2049] VirtualId# [5:7574435878837725120:0]
2025-11-19T13:59:41.098514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:41.098523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435878837725034:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23006) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:41.098526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23006) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:41.107064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:41.107183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435878179819248:2050] [node 3] ICS01 socket: 569 reason# EndOfStream
2025-11-19T13:59:41.107194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435878179819164:2050] [node 3] ICP30 unregister session Session# [6:7574435878179819248:2050] VirtualId# [6:7574435878179819242:0]
2025-11-19T13:59:41.107204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435878179819248:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.107265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:41.107295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:41.107347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:41.107424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435883132692426:2050] [node 3] ICS01 socket: 599 reason# EndOfStream
2025-11-19T13:59:41.107429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435878837725035:2050] [node 3] ICP30 unregister session Session# [5:7574435883132692426:2050] VirtualId# [5:7574435883132692419:0]
2025-11-19T13:59:41.107434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435883132692426:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.107467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:41.107491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:41.108341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435878837725034:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:41.125731Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:41.125823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435878179819251:2062] [node 4] ICS01 socket: 549 reason# EndOfStream
2025-11-19T13:59:41.125832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435878179819178:2062] [node 4] ICP30 unregister session Session# [6:7574435878179819251:2062] VirtualId# [6:7574435878179819243:0]
2025-11-19T13:59:41.125840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435878179819251:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.125892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:41.125916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:41.221011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435881903765413:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:41.221025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435881903765414:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:41.292525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435879214425104:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:41.292546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435879214425105:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:41.300047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435880841518677:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:41.300059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435880841518678:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:41.376039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435880089741845:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:41.376054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435880089741846:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:41.376059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435880089741847:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:41.411900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435882369413294:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:41.411914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435882369413295:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:41.411920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435882369413296:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:41.492018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435879207293615:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:41.492033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435879207293617:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:41.492054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435879207293616:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:41.524693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435882369413334:2084] [node 4] ICP01 ready to work
2025-11-19T13:59:41.524781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435882369413335:2086] [node 6] ICP01 ready to work
2025-11-19T13:59:41.525730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435879207293646:2066] [node 5] ICP01 ready to work
2025-11-19T13:59:41.525825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435879207293647:2072] [node 4] ICP01 ready to work
2025-11-19T13:59:41.531211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435880089741907:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.540492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435879214425167:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:41.541241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435879214425166:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14187
2025-11-19T13:59:41.545589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435879214425175:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:41.545678Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435879214425166:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.545781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435879214425177:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.547933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435882369413296:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17654554ns) {Configure:104(17653400ns) {ProcessConfigured:135(17348201ns) {StartInitialHandshake:157(17347851ns) {DropHandshakes:469(543ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(142ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(947ns) {SwitchToState:178(679ns)}}}}}
2025-11-19T13:59:41.548208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435882369413364:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15133
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.551845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435880089741907:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20552684ns) {Configure:104(20551816ns) {ProcessConfigured:135(20189386ns) {StartInitialHandshake:157(20188865ns) {DropHandshakes:469(579ns) {DropIncomingHandshake:441(86ns) DropOutgoingHandshake:456(41ns)} GenerateSessionVirtualId:988(89ns) PrepareNewSessionHandshake:476(2154ns) {SwitchToState:178(1577ns)}}}}}
2025-11-19T13:59:41.552022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435880089741916:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:41.552885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435880089741917:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28703
2025-11-19T13:59:41.553037Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435881903765414:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21361533ns) {Configure:104(21358567ns) {ProcessConfigured:135(21086404ns) {StartInitialHandshake:157(21086266ns) {DropHandshakes:469(292ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(124ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(1010ns) {SwitchToState:178(230ns)}}}}}
2025-11-19T13:59:41.553178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435881903765479:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:41.553253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435881903765480:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:41.553332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435881903765481:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:41.554202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435881903765482:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3399
2025-11-19T13:59:41.554257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435881903765483:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3399
2025-11-19T13:59:41.554267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435881903765484:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3399
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.554800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435879207293692:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49436
2025-11-19T13:59:41.554819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435879207293693:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49436
2025-11-19T13:59:41.555937Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435881903765482:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.560312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435880841518743:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:41.560412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435880841518744:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:41.560484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435880841518745:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.566414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435880841518754:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49995
2025-11-19T13:59:41.566472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435880841518755:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49995
2025-11-19T13:59:41.566484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435880841518756:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49995
2025-11-19T13:59:41.566494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435880841518757:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49995
2025-11-19T13:59:41.566807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435879207293646:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(40929667ns) {Configure:104(40928838ns) {ProcessConfigured:135(40628135ns) {StartInitialHandshake:157(40627493ns) {DropHandshakes:469(437ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(69ns) PrepareNewSessionHandshake:476(1278ns) {SwitchToState:178(792ns)}}}}}
2025-11-19T13:59:41.566959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435881903765413:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(40487086ns) {Configure:104(40486595ns) {ProcessConfigured:135(40084933ns) {StartInitialHandshake:157(40084573ns) {DropHandshakes:469(552ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(93ns)} GenerateSessionVirtualId:988(311ns) PrepareNewSessionHandshake:476(1095ns) {SwitchToState:178(150ns)}}}}}
2025-11-19T13:59:41.571812Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435880841518754:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.572028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435879214425175:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(26212723ns) {Configure:104(26211683ns) {ProcessConfigured:135(25897771ns) {StartInitialHandshake:157(25897242ns) {DropHandshakes:469(761ns) {DropIncomingHandshake:441(179ns) DropOutgoingHandshake:456(201ns)} GenerateSessionVirtualId:988(248ns) PrepareNewSessionHandshake:476(1508ns) {SwitchToState:178(856ns)}}}}}
2025-11-19T13:59:41.574024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435881903765483:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.574138Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435879207293692:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.574172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435879207293647:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.590999Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435882369413335:2086] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(63163730ns) {Configure:104(63160715ns) {ProcessConfigured:135(62835802ns) {StartInitialHandshake:157(62835151ns) {DropHandshakes:469(478ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(1854ns) {SwitchToState:178(558ns)}}}}}
2025-11-19T13:59:41.591815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435882369413368:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15133
2025-11-19T13:59:41.591829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435882369413369:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15133
2025-11-19T13:59:41.592872Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435880089741915:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.592898Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435880089741915:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.615550Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435879214425104:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(80073225ns) {Configure:104(80072385ns) {ProcessConfigured:135(79656981ns) {StartInitialHandshake:157(79656429ns) {DropHandshakes:469(914ns) {DropIncomingHandshake:441(132ns) DropOutgoingHandshake:456(131ns)} GenerateSessionVirtualId:988(330ns) PrepareNewSessionHandshake:476(2047ns) {SwitchToState:178(1151ns)}}}}}
2025-11-19T13:59:41.615707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435879214425183:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14187
2025-11-19T13:59:41.615721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435879214425184:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14187
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.625053Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435880089741915:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:41.625202Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435879214425183:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.625223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435880089741907:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.625240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:41.625249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435880089741907:2108] [node 6] ICP22 created new session: [4:7574435880089741925:2108]
2025-11-19T13:59:41.625254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435880089741925:2108] [node 6] ICS09 handshake done sender: [4:7574435880089741915:2112] self: [4:7574435880089741908:0] peer: [6:7574435879207293695:0] socket: 566 qp: -1
2025-11-19T13:59:41.625260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435880089741925:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:41.625268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.627975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435882369413375:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15133
2025-11-19T13:59:41.628044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435882369413376:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15133
2025-11-19T13:59:41.623275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435879207293696:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49436
2025-11-19T13:59:41.623299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435879207293697:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49436
2025-11-19T13:59:41.625749Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435879207293692:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:41.625802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435879207293647:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.625812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:41.625817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435879207293647:2072] [node 4] ICP22 created new session: [6:7574435879207293698:2072]
2025-11-19T13:59:41.625823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435879207293698:2072] [node 4] ICS09 handshake done sender: [6:7574435879207293692:2114] self: [6:7574435879207293695:0] peer: [4:7574435880089741908:0] socket: 573 qp: -1
2025-11-19T13:59:41.625827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435879207293698:2072] [node 4] ICS10 traffic start
2025-11-19T13:59:41.625846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.629577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435880089741928:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28703
2025-11-19T13:59:41.632367Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435882369413375:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.633633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435882369413334:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.636009Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435880089741917:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.636049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435880089741920:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.636053Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435880089741920:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.636767Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435880089741920:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.636864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435880089741916:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.636893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:41.636900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435880089741916:2111] [node 5] ICP22 created new session: [4:7574435880089741931:2111]
2025-11-19T13:59:41.636905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435880089741931:2111] [node 5] ICS09 handshake done sender: [4:7574435880089741920:2114] self: [4:7574435880089741919:0] peer: [5:7574435882369413377:0] socket: 590 qp: -1
2025-11-19T13:59:41.636911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435880089741931:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:41.636920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:41.637001Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435882369413376:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.637168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435882369413378:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15133
2025-11-19T13:59:41.637359Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435879207293696:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.637392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435879207293646:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.635580Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435880841518757:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.638250Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435879207293693:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.638279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435879207293615:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.638589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435882369413334:2084] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8048) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435882369413375:2118] held: no
2025-11-19T13:59:41.638632Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435882369413368:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.638645Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435882369413375:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:41.638733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435882369413334:2084] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.638746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:41.638751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435882369413334:2084] [node 4] ICP22 created new session: [5:7574435882369413380:2084]
2025-11-19T13:59:41.638755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435882369413380:2084] [node 4] ICS09 handshake done sender: [5:7574435882369413375:2118] self: [5:7574435882369413377:0] peer: [4:7574435880089741919:0] socket: 592 qp: -1
2025-11-19T13:59:41.638761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435882369413380:2084] [node 4] ICS10 traffic start
2025-11-19T13:59:41.638767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:41.638895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435882369413294:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.639173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435879214425188:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14187
2025-11-19T13:59:41.642622Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435879214425188:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.642631Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435880841518756:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.642826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435880841518677:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.642917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435880841518761:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49995
2025-11-19T13:59:41.642848Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435879214425184:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.642977Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435880089741928:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.643035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435880089741846:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.643202Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435882369413364:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.643224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435882369413295:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.643339Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435881903765486:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.643351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435881903765486:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.643444Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435881903765488:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.643452Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435881903765488:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.643389Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435882369413367:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.643403Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435882369413367:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.643506Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435879214425179:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.643516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435879214425179:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.643552Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435880841518761:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.643580Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435880841518755:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.643616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435880841518678:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.644182Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435879214425171:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.644194Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435879214425171:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.644657Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435882369413369:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.644681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435882369413296:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.644776Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435880841518749:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.644785Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435880841518749:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.644757Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435879214425169:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.644767Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435879214425169:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.645346Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435879214425179:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:41.645719Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435879214425171:2110] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:41.645960Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435881903765478:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.645971Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435881903765478:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.646816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435880841518766:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49995
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.647613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435879207293703:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49436
2025-11-19T13:59:41.648819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435879207293704:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49436
2025-11-19T13:59:41.652298Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435881903765486:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:41.652371Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435881903765478:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.652230Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435882369413367:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:41.652524Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435880841518749:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:41.652138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435879214425167:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.652164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:41.652173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435879214425167:2107] [node 5] ICP22 created new session: [2:7574435879214425194:2107]
2025-11-19T13:59:41.652179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435879214425194:2107] [node 5] ICS09 handshake done sender: [2:7574435879214425171:2110] self: [2:7574435879214425170:0] peer: [5:7574435882369413384:0] socket: 559 qp: -1
2025-11-19T13:59:41.652186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435879214425194:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:41.652196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:41.652337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435879214425175:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.652347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:41.652351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435879214425175:2111] [node 4] ICP22 created new session: [2:7574435879214425196:2111]
2025-11-19T13:59:41.652355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435879214425196:2111] [node 4] ICS09 handshake done sender: [2:7574435879214425179:2113] self: [2:7574435879214425178:0] peer: [4:7574435880089741934:0] socket: 586 qp: -1
2025-11-19T13:59:41.652360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435879214425196:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:41.652364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.652475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435880089741935:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28703
2025-11-19T13:59:41.652692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435882369413387:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15133
2025-11-19T13:59:41.653439Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435880089741928:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:41.652627Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435879207293696:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:41.652695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435879207293646:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.652705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:41.652711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435879207293646:2066] [node 5] ICP22 created new session: [6:7574435879207293705:2066]
2025-11-19T13:59:41.652717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435879207293705:2066] [node 5] ICS09 handshake done sender: [6:7574435879207293696:2117] self: [6:7574435879207293701:0] peer: [5:7574435882369413338:0] socket: 585 qp: -1
2025-11-19T13:59:41.652722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435879207293705:2066] [node 5] ICS10 traffic start
2025-11-19T13:59:41.652739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.654100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435882369413388:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15133
2025-11-19T13:59:41.654210Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435881903765484:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.654276Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435881903765488:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:41.654749Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435879214425169:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:41.654381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435881903765414:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.654395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:41.654401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435881903765414:2049] [node 3] ICP22 created new session: [1:7574435881903765500:2049]
2025-11-19T13:59:41.654406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435881903765500:2049] [node 3] ICS09 handshake done sender: [1:7574435881903765478:2110] self: [1:7574435881903765477:0] peer: [3:7574435880841518762:0] socket: 570 qp: -1
2025-11-19T13:59:41.654411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435881903765500:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:41.654419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:41.654531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435881903765479:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.654538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:41.654542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435881903765479:2107] [node 5] ICP22 created new session: [1:7574435881903765502:2107]
2025-11-19T13:59:41.654548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435881903765502:2107] [node 5] ICS09 handshake done sender: [1:7574435881903765486:2114] self: [1:7574435881903765485:0] peer: [5:7574435882369413383:0] socket: 571 qp: -1
2025-11-19T13:59:41.654551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435881903765502:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:41.654555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:41.654679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435881903765414:2049] [node 3] ICP18 handshake (actor [1:7574435881903765484:2113]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.655059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435882369413389:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15133
2025-11-19T13:59:41.654868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435879214425105:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.654885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:41.654890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435879214425105:2049] [node 3] ICP22 created new session: [2:7574435879214425200:2049]
2025-11-19T13:59:41.654894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435879214425200:2049] [node 3] ICS09 handshake done sender: [2:7574435879214425169:2109] self: [2:7574435879214425168:0] peer: [3:7574435880841518765:0] socket: 558 qp: -1
2025-11-19T13:59:41.654899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435879214425200:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:41.654906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:41.655138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435880089741846:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.655153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:41.655158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435880089741846:2049] [node 2] ICP22 created new session: [4:7574435880089741936:2049]
2025-11-19T13:59:41.655162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435880089741936:2049] [node 2] ICS09 handshake done sender: [4:7574435880089741928:2116] self: [4:7574435880089741934:0] peer: [2:7574435879214425178:0] socket: 587 qp: -1
2025-11-19T13:59:41.655167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435880089741936:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:41.655174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:41.655509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435880841518743:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.655528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:41.655533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435880841518743:2108] [node 5] ICP22 created new session: [3:7574435880841518768:2108]
2025-11-19T13:59:41.655538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435880841518768:2108] [node 5] ICS09 handshake done sender: [3:7574435880841518749:2112] self: [3:7574435880841518748:0] peer: [5:7574435882369413385:0] socket: 579 qp: -1
2025-11-19T13:59:41.655542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435880841518768:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:41.655548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:41.656841Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435879207293693:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.660440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435879214425202:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14187
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.659541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435881903765480:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.659563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:41.659570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435881903765480:2108] [node 6] ICP22 created new session: [1:7574435881903765507:2108]
2025-11-19T13:59:41.659576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435881903765507:2108] [node 6] ICS09 handshake done sender: [1:7574435881903765488:2115] self: [1:7574435881903765487:0] peer: [6:7574435879207293702:0] socket: 572 qp: -1
2025-11-19T13:59:41.659581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435881903765507:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:41.659602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:41.660731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435881903765506:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3399
2025-11-19T13:59:41.663261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435882369413335:2086] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.663287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:41.663295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435882369413335:2086] [node 6] ICP22 created new session: [5:7574435882369413390:2086]
2025-11-19T13:59:41.663301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435882369413390:2086] [node 6] ICS09 handshake done sender: [5:7574435882369413367:2113] self: [5:7574435882369413338:0] peer: [6:7574435879207293701:0] socket: 584 qp: -1
2025-11-19T13:59:41.663307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435882369413390:2086] [node 6] ICS10 traffic start
2025-11-19T13:59:41.663316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:41.664376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435879207293615:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.664394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:41.664401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435879207293615:2048] [node 1] ICP22 created new session: [6:7574435879207293708:2048]
2025-11-19T13:59:41.664408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435879207293708:2048] [node 1] ICS09 handshake done sender: [6:7574435879207293693:2115] self: [6:7574435879207293702:0] peer: [1:7574435881903765487:0] socket: 574 qp: -1
2025-11-19T13:59:41.664414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435879207293708:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:41.664421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.665442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435879207293707:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49436
2025-11-19T13:59:41.665475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435879207293615:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:41.668474Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435882369413364:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.668671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435880089741938:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28703
2025-11-19T13:59:41.668020Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435880841518757:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:41.669323Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435881903765506:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.669420Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435880089741938:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.669432Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435879207293707:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.669455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435880089741845:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.669452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435879207293616:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.670175Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435879214425202:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.669591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435880841518677:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18189) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435880841518757:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.670559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435880841518678:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27447) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435880841518755:2116] held: no
2025-11-19T13:59:41.670579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435880841518677:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.670590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:41.670596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435880841518677:2048] [node 1] ICP22 created new session: [3:7574435880841518772:2048]
2025-11-19T13:59:41.670603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435880841518772:2048] [node 1] ICS09 handshake done sender: [3:7574435880841518757:2118] self: [3:7574435880841518762:0] peer: [1:7574435881903765477:0] socket: 577 qp: -1
2025-11-19T13:59:41.670609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435880841518772:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:41.670615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:41.670838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435880841518770:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49995
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.670918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435881903765490:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.670934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435881903765490:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.670957Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435882369413295:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27447) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435882369413364:2112] held: no
2025-11-19T13:59:41.670991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435882369413295:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.671000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:41.671005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435882369413295:2049] [node 2] ICP22 created new session: [5:7574435882369413392:2049]
2025-11-19T13:59:41.671010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435882369413392:2049] [node 2] ICS09 handshake done sender: [5:7574435882369413364:2112] self: [5:7574435882369413384:0] peer: [2:7574435879214425170:0] socket: 561 qp: -1
2025-11-19T13:59:41.671015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435882369413392:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:41.671022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:41.671101Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435882369413369:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:41.670990Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435879214425181:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.670998Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435879214425181:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.671480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435879214425104:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.671785Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435882369413368:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:41.671955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435879207293711:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49436
2025-11-19T13:59:41.672764Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435881903765495:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.672772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435881903765495:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.674863Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435879214425181:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:41.675029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435879214425177:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.675046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:41.675052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435879214425177:2112] [node 6] ICP22 created new session: [2:7574435879214425206:2112]
2025-11-19T13:59:41.675058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435879214425206:2112] [node 6] ICS09 handshake done sender: [2:7574435879214425181:2114] self: [2:7574435879214425180:0] peer: [6:7574435879207293710:0] socket: 591 qp: -1
2025-11-19T13:59:41.675062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435879214425206:2112] [node 6] ICS10 traffic start
2025-11-19T13:59:41.675068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.675420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435879214425205:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14187
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.677764Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435879207293616:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27447) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435879207293707:2121] held: no
2025-11-19T13:59:41.677857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435879207293712:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49436
2025-11-19T13:59:41.678225Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435879207293707:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:41.678292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435879207293616:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.678341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:41.678347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435879207293616:2049] [node 2] ICP22 created new session: [6:7574435879207293713:2049]
2025-11-19T13:59:41.678353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435879207293713:2049] [node 2] ICS09 handshake done sender: [6:7574435879207293707:2121] self: [6:7574435879207293710:0] peer: [2:7574435879214425180:0] socket: 569 qp: -1
2025-11-19T13:59:41.678358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435879207293713:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:41.678367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:41.678717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435882369413296:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.678737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:41.676412Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435880841518755:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:41.677975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435880089741940:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28703
2025-11-19T13:59:41.677994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435880089741941:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28703
2025-11-19T13:59:41.678743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435882369413296:2050] [node 3] ICP22 created new session: [5:7574435882369413395:2050]
2025-11-19T13:59:41.678759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435882369413395:2050] [node 3] ICS09 handshake done sender: [5:7574435882369413369:2115] self: [5:7574435882369413385:0] peer: [3:7574435880841518748:0] socket: 583 qp: -1
2025-11-19T13:59:41.678766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435882369413395:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:41.678772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:41.678851Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435882369413296:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:41.679021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435882369413294:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.679029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:41.679034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435882369413294:2048] [node 1] ICP22 created new session: [5:7574435882369413397:2048]
2025-11-19T13:59:41.679039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435882369413397:2048] [node 1] ICS09 handshake done sender: [5:7574435882369413368:2114] self: [5:7574435882369413383:0] peer: [1:7574435881903765485:0] socket: 582 qp: -1
2025-11-19T13:59:41.679044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435882369413397:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:41.679051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:41.681967Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435879207293711:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.682003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435879207293617:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.680029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435880841518678:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.680046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:41.680052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435880841518678:2049] [node 2] ICP22 created new session: [3:7574435880841518775:2049]
2025-11-19T13:59:41.680059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435880841518775:2049] [node 2] ICS09 handshake done sender: [3:7574435880841518755:2116] self: [3:7574435880841518765:0] peer: [2:7574435879214425168:0] socket: 575 qp: -1
2025-11-19T13:59:41.680064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435880841518775:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:41.680072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:41.684024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435880841518751:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.684032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435880841518751:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:41.685316Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435880089741938:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:41.685505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435880089741845:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.685521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:41.685527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435880089741845:2048] [node 1] ICP22 created new session: [4:7574435880089741942:2048]
2025-11-19T13:59:41.685533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435880089741942:2048] [node 1] ICS09 handshake done sender: [4:7574435880089741938:2118] self: [4:7574435880089741939:0] peer: [1:7574435881903765489:0] socket: 567 qp: -1
2025-11-19T13:59:41.685539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435880089741942:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:41.685546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.687109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435879207293716:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49436
2025-11-19T13:59:41.687253Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435880841518751:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:41.687441Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435879207293711:2122] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:41.684087Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435881903765490:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:41.684157Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435881903765495:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:41.688294Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435879214425202:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:41.687561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435879207293617:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.687575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:41.687580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435879207293617:2050] [node 3] ICP22 created new session: [6:7574435879207293717:2050]
2025-11-19T13:59:41.687585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435879207293717:2050] [node 3] ICS09 handshake done sender: [6:7574435879207293711:2122] self: [6:7574435879207293715:0] peer: [3:7574435880841518750:0] socket: 562 qp: -1
2025-11-19T13:59:41.687590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435879207293717:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:41.687597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:41.689107Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435880089741941:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.689137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435880089741847:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:41.688794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435880841518744:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.688813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:41.688818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435880841518744:2109] [node 6] ICP22 created new session: [3:7574435880841518778:2109]
2025-11-19T13:59:41.688823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435880841518778:2109] [node 6] ICS09 handshake done sender: [3:7574435880841518751:2113] self: [3:7574435880841518750:0] peer: [6:7574435879207293715:0] socket: 554 qp: -1
2025-11-19T13:59:41.688827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435880841518778:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:41.688833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:41.689557Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435880841518753:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:41.689564Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435880841518753:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.690595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435880089741945:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28703
2025-11-19T13:59:41.690718Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435880841518753:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:41.690875Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435880089741941:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:41.690976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435880089741847:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.690992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:41.690998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435880089741847:2050] [node 3] ICP22 created new session: [4:7574435880089741946:2050]
2025-11-19T13:59:41.691004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435880089741946:2050] [node 3] ICS09 handshake done sender: [4:7574435880089741941:2120] self: [4:7574435880089741944:0] peer: [3:7574435880841518752:0] socket: 617 qp: -1
2025-11-19T13:59:41.691009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435880089741946:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:41.691016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:41.700296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435880841518745:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.700332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:41.700341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435880841518745:2110] [node 4] ICP22 created new session: [3:7574435880841518781:2110]
2025-11-19T13:59:41.700347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435880841518781:2110] [node 4] ICS09 handshake done sender: [3:7574435880841518753:2114] self: [3:7574435880841518752:0] peer: [4:7574435880089741944:0] socket: 616 qp: -1
2025-11-19T13:59:41.700354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435880841518781:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:41.700366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:41.701453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435881903765413:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.701475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:41.701482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435881903765413:2048] [node 2] ICP22 created new session: [1:7574435881903765513:2048]
2025-11-19T13:59:41.701489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435881903765513:2048] [node 2] ICS09 handshake done sender: [1:7574435881903765495:2117] self: [1:7574435881903765473:0] peer: [2:7574435879214425204:0] socket: 565 qp: -1
2025-11-19T13:59:41.701494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435881903765513:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:41.701502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:41.701586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435881903765481:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:41.701594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:41.701598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435881903765481:2109] [node 4] ICP22 created new session: [1:7574435881903765515:2109]
2025-11-19T13:59:41.701601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435881903765515:2109] [node 4] ICS09 handshake done sender: [1:7574435881903765490:2116] self: [1:7574435881903765489:0] peer: [4:7574435880089741939:0] socket: 557 qp: -1
2025-11-19T13:59:41.701605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435881903765515:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:41.701610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:41.702109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435879214425104:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:41.702126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:41.702132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435879214425104:2048] [node 1] ICP22 created new session: [2:7574435879214425208:2048]
2025-11-19T13:59:41.702137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435879214425208:2048] [node 1] ICS09 handshake done sender: [2:7574435879214425202:2119] self: [2:7574435879214425204:0] peer: [1:7574435881903765473:0] socket: 581 qp: -1
2025-11-19T13:59:41.702142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435879214425208:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:41.702162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:41.704892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435881903765512:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3399
2025-11-19T13:59:41.709651Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:41.709732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435879214425208:2048] [node 1] ICS01 socket: 581 reason# EndOfStream
2025-11-19T13:59:41.709740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435879214425104:2048] [node 1] ICP30 unregister session Session# [2:7574435879214425208:2048] VirtualId# [2:7574435879214425204:0]
2025-11-19T13:59:41.709747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435879214425208:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.709775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:41.709793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:41.710224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:41.710290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435880841518772:2048] [node 1] ICS01 socket: 577 reason# EndOfStream
2025-11-19T13:59:41.710295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435880841518677:2048] [node 1] ICP30 unregister session Session# [3:7574435880841518772:2048] VirtualId# [3:7574435880841518762:0]
2025-11-19T13:59:41.710299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435880841518772:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.710323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:41.710349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:41.713591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:41.713664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435880089741942:2048] [node 1] ICS01 socket: 567 reason# EndOfStream
2025-11-19T13:59:41.713671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435880089741845:2048] [node 1] ICP30 unregister session Session# [4:7574435880089741942:2048] VirtualId# [4:7574435880089741939:0]
2025-11-19T13:59:41.713676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435880089741942:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.713724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:41.713744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:41.713817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:41.713875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435882369413397:2048] [node 1] ICS01 socket: 582 reason# EndOfStream
2025-11-19T13:59:41.713878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435882369413294:2048] [node 1] ICP30 unregister session Session# [5:7574435882369413397:2048] VirtualId# [5:7574435882369413383:0]
2025-11-19T13:59:41.713883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435882369413397:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.713910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:41.713929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:41.716819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:41.716892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435879207293708:2048] [node 1] ICS01 socket: 574 reason# EndOfStream
2025-11-19T13:59:41.716903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435879207293615:2048] [node 1] ICP30 unregister session Session# [6:7574435879207293708:2048] VirtualId# [6:7574435879207293702:0]
2025-11-19T13:59:41.716909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435879207293708:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.716947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:41.716975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:41.718222Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:41.718289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435880841518775:2049] [node 2] ICS01 socket: 575 reason# EndOfStream
2025-11-19T13:59:41.718299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435880841518678:2049] [node 2] ICP30 unregister session Session# [3:7574435880841518775:2049] VirtualId# [3:7574435880841518765:0]
2025-11-19T13:59:41.718304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435880841518775:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.718329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:41.718348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:41.728415Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:41.728507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435879207293713:2049] [node 2] ICS01 socket: 569 reason# EndOfStream
2025-11-19T13:59:41.728517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435879207293616:2049] [node 2] ICP30 unregister session Session# [6:7574435879207293713:2049] VirtualId# [6:7574435879207293710:0]
2025-11-19T13:59:41.728526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435879207293713:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.728587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:41.728626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:41.730890Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:41.730979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435882369413392:2049] [node 2] ICS01 socket: 561 reason# EndOfStream
2025-11-19T13:59:41.730986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435882369413295:2049] [node 2] ICP30 unregister session Session# [5:7574435882369413392:2049] VirtualId# [5:7574435882369413384:0]
2025-11-19T13:59:41.731001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435882369413392:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.731047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:41.731070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:41.731104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:41.731166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435880089741936:2049] [node 2] ICS01 socket: 587 reason# EndOfStream
2025-11-19T13:59:41.731169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435880089741846:2049] [node 2] ICP30 unregister session Session# [4:7574435880089741936:2049] VirtualId# [4:7574435880089741934:0]
2025-11-19T13:59:41.731173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435880089741936:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.731198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:41.731217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:41.748143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:41.748235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435879207293717:2050] [node 3] ICS01 socket: 562 reason# EndOfStream
2025-11-19T13:59:41.748245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435879207293617:2050] [node 3] ICP30 unregister session Session# [6:7574435879207293717:2050] VirtualId# [6:7574435879207293715:0]
2025-11-19T13:59:41.748253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435879207293717:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.748315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:41.748338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:41.751860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:41.751955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435882369413395:2050] [node 3] ICS01 socket: 583 reason# EndOfStream
2025-11-19T13:59:41.751962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435882369413296:2050] [node 3] ICP30 unregister session Session# [5:7574435882369413395:2050] VirtualId# [5:7574435882369413385:0]
2025-11-19T13:59:41.751969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435882369413395:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.752024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:41.752047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:41.783450Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:41.783550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435879207293698:2072] [node 4] ICS01 socket: 573 reason# EndOfStream
2025-11-19T13:59:41.783559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435879207293647:2072] [node 4] ICP30 unregister session Session# [6:7574435879207293698:2072] VirtualId# [6:7574435879207293695:0]
2025-11-19T13:59:41.783567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435879207293698:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.783637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:41.783663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:41.797648Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:59:41.797742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435879207293705:2066] [node 5] ICS01 socket: 585 reason# EndOfStream
2025-11-19T13:59:41.797752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435879207293646:2066] [node 5] ICP30 unregister session Session# [6:7574435879207293705:2066] VirtualId# [6:7574435879207293701:0]
2025-11-19T13:59:41.797759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435879207293705:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:41.797800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:59:41.797822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:41.983868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435879123679930:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:41.983889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435879123679931:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:42.031950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435885445058966:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:42.031980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435885445058967:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:42.051917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435885339630147:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:42.052153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435885339630146:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:42.068078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435883959044884:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:42.068092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435883959044885:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:42.068096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435883959044886:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:42.147988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435885822107579:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:42.148010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435885822107580:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:42.148014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435885822107581:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:42.219930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435887215808661:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:42.219944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435887215808662:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:42.219949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435887215808663:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:42.268112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435885822107599:2065] [node 4] ICP01 ready to work
2025-11-19T13:59:42.268598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435883959044938:2095] [node 6] ICP01 ready to work
2025-11-19T13:59:42.268697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435883959044939:2098] [node 5] ICP01 ready to work
2025-11-19T13:59:42.269152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435885822107613:2069] [node 6] ICP01 ready to work
2025-11-19T13:59:42.267802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435883418647287:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.271069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435885445059026:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26675
2025-11-19T13:59:42.268631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435887215808684:2067] [node 5] ICP01 ready to work
2025-11-19T13:59:42.270698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435887215808693:2076] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21304
2025-11-19T13:59:42.272097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435887215808714:2078] [node 4] ICP01 ready to work
2025-11-19T13:59:42.274996Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435887215808693:2076] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.275038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435887215808714:2078] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.284335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435885445059030:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:42.282744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435883418647289:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:42.282981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435883418647295:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:42.282002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435885339630206:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25085
2025-11-19T13:59:42.282028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435885339630207:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25085
2025-11-19T13:59:42.284266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435885339630210:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:42.284576Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435883959044942:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.284607Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435883959044942:2100] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.288596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435885445059035:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:42.292020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435885822107579:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23816949ns) {Configure:104(23816125ns) {ProcessConfigured:135(22692529ns) {StartInitialHandshake:157(22692004ns) {DropHandshakes:469(399ns) {DropIncomingHandshake:441(170ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(1816ns) {SwitchToState:178(993ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.295478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435885445059043:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26675
2025-11-19T13:59:42.295498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435885445059044:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26675
2025-11-19T13:59:42.292691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435879123679931:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(24334856ns) {Configure:104(24333528ns) {ProcessConfigured:135(17464271ns) {StartInitialHandshake:157(17463408ns) {DropHandshakes:469(689ns) {DropIncomingHandshake:441(124ns) DropOutgoingHandshake:456(166ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:476(806ns) {SwitchToState:178(725ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.302033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435883959044958:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37727
2025-11-19T13:59:42.302097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435883959044959:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37727
2025-11-19T13:59:42.304099Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435883959044958:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.304775Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435885445059044:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.307985Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435885445059043:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.312610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435885822107656:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55300
2025-11-19T13:59:42.317426Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435883418647289:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(34395254ns) {Configure:104(34393563ns) {ProcessConfigured:135(34036884ns) {StartInitialHandshake:157(34036274ns) {DropHandshakes:469(889ns) {DropIncomingHandshake:441(129ns) DropOutgoingHandshake:456(134ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:476(1302ns) {SwitchToState:178(47ns)}}}}}
2025-11-19T13:59:42.318225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435883959044959:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.320820Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435885822107656:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.320862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435885822107599:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.320003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435883959044884:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(35679132ns) {Configure:104(35677963ns) {ProcessConfigured:135(35283115ns) {StartInitialHandshake:157(35282434ns) {DropHandshakes:469(805ns) {DropIncomingHandshake:441(157ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:476(953ns) {SwitchToState:178(886ns)}}}}}
2025-11-19T13:59:42.320126Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435883959044942:2100] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:42.320114Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435885445059034:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.320133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435885445059034:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.321170Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435885445059034:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:42.321267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435885445059030:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.321280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:42.321290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435885445059030:2108] [node 4] ICP22 created new session: [2:7574435885445059047:2108]
2025-11-19T13:59:42.321297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435885445059047:2108] [node 4] ICS09 handshake done sender: [2:7574435885445059034:2112] self: [2:7574435885445059033:0] peer: [4:7574435883959044960:0] socket: 570 qp: -1
2025-11-19T13:59:42.321305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435885445059047:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:42.321316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.323945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435885339630146:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(39622556ns) {Configure:104(39621419ns) {ProcessConfigured:135(39213906ns) {StartInitialHandshake:157(39213164ns) {DropHandshakes:469(658ns) {DropIncomingHandshake:441(111ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:476(1703ns) {SwitchToState:178(1133ns)}}}}}
2025-11-19T13:59:42.324113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435885339630216:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:42.324933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435885339630219:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25085
2025-11-19T13:59:42.324969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435885339630220:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25085
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.325240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435885445058966:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(40856404ns) {Configure:104(40855940ns) {ProcessConfigured:135(40439874ns) {StartInitialHandshake:157(40439602ns) {DropHandshakes:469(479ns) {DropIncomingHandshake:441(102ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(333ns) PrepareNewSessionHandshake:476(767ns) {SwitchToState:178(633ns)}}}}}
2025-11-19T13:59:42.325277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435885445058966:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.327033Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883418647299:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.327048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435883418647299:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.327630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435883418647305:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44128
2025-11-19T13:59:42.327643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435883418647306:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44128
2025-11-19T13:59:42.325550Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435885339630210:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(37829639ns) {Configure:104(37825657ns) {ProcessConfigured:135(37453554ns) {StartInitialHandshake:157(37453313ns) {DropHandshakes:469(1235ns) {DropIncomingHandshake:441(206ns) DropOutgoingHandshake:456(103ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:476(583ns) {SwitchToState:178(25ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.328091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435885445059052:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26675
2025-11-19T13:59:42.328122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435885445059053:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26675
2025-11-19T13:59:42.329259Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435883418647299:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:42.329618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435879123679930:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.329635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:42.329641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435879123679930:2048] [node 2] ICP22 created new session: [1:7574435883418647308:2048]
2025-11-19T13:59:42.329647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435883418647308:2048] [node 2] ICS09 handshake done sender: [1:7574435883418647299:2112] self: [1:7574435883418647298:0] peer: [2:7574435885445059050:0] socket: 578 qp: -1
2025-11-19T13:59:42.329653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435883418647308:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:42.329674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:42.329467Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435885445059044:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:42.330056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435885445058966:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.330066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:42.330072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435885445058966:2048] [node 1] ICP22 created new session: [2:7574435885445059054:2048]
2025-11-19T13:59:42.330077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435885445059054:2048] [node 1] ICS09 handshake done sender: [2:7574435885445059044:2116] self: [2:7574435885445059050:0] peer: [1:7574435883418647298:0] socket: 580 qp: -1
2025-11-19T13:59:42.330082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435885445059054:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:42.330088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:42.330722Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883418647306:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.331345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435879123679930:2048] [node 2] ICP18 handshake (actor [1:7574435883418647306:2116]) is too old
2025-11-19T13:59:42.332365Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885339630220:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.332405Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885339630219:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.332430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435885339630146:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.332440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435885339630147:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.332587Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883418647297:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.332594Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435883418647297:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.333238Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435883418647297:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:42.333325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435879123679931:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.333341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:42.333349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435879123679931:2049] [node 3] ICP22 created new session: [1:7574435883418647312:2049]
2025-11-19T13:59:42.333355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435883418647312:2049] [node 3] ICS09 handshake done sender: [1:7574435883418647297:2111] self: [1:7574435883418647293:0] peer: [3:7574435885339630227:0] socket: 577 qp: -1
2025-11-19T13:59:42.333361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435883418647312:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:42.333380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.335635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435885339630229:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25085
2025-11-19T13:59:42.334838Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435885445059039:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.334854Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435885445059039:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.335458Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435885445059039:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:42.335772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435885445058967:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.335786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:42.335792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435885445058967:2049] [node 3] ICP22 created new session: [2:7574435885445059057:2049]
2025-11-19T13:59:42.335799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435885445059057:2049] [node 3] ICS09 handshake done sender: [2:7574435885445059039:2114] self: [2:7574435885445059038:0] peer: [3:7574435885339630228:0] socket: 572 qp: -1
2025-11-19T13:59:42.335805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435885445059057:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:42.335813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.336167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435885339630230:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25085
2025-11-19T13:59:42.336350Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435885339630220:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:42.336405Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435885339630219:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:42.336457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435885339630146:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.336472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:42.336477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435885339630146:2048] [node 1] ICP22 created new session: [3:7574435885339630231:2048]
2025-11-19T13:59:42.336482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435885339630231:2048] [node 1] ICS09 handshake done sender: [3:7574435885339630220:2115] self: [3:7574435885339630227:0] peer: [1:7574435883418647293:0] socket: 587 qp: -1
2025-11-19T13:59:42.336488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435885339630231:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:42.336506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:42.336646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435885339630147:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.336652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:42.336657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435885339630147:2049] [node 2] ICP22 created new session: [3:7574435885339630233:2049]
2025-11-19T13:59:42.336662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435885339630233:2049] [node 2] ICS09 handshake done sender: [3:7574435885339630219:2114] self: [3:7574435885339630228:0] peer: [2:7574435885445059038:0] socket: 586 qp: -1
2025-11-19T13:59:42.336665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435885339630233:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:42.336670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:42.338562Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435885445059026:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.339212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435887215808661:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(31719827ns) {Configure:104(31717909ns) {ProcessConfigured:135(28320472ns) {StartInitialHandshake:157(28319627ns) {DropHandshakes:469(557ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(1438ns) {SwitchToState:178(791ns)}}}}}
2025-11-19T13:59:42.339520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435883959044885:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.339576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435883959044885:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(35400475ns) {Configure:104(35399566ns) {ProcessConfigured:135(35002851ns) {IncomingHandshake:261(34991186ns) {DropIncomingHandshake:441(163ns) PrepareNewSessionHandshake:476(665ns) {SwitchToState:178(154ns)} GenerateSessionVirtualId:988(508ns) IssueIncomingHandshakeReply:193(34983715ns) {SwitchToState:178(1004ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.344975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435887215808741:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21304
2025-11-19T13:59:42.345065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435887215808742:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21304
2025-11-19T13:59:42.345106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435887215808743:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21304
2025-11-19T13:59:42.345220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435887215808744:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21304
2025-11-19T13:59:42.345233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435887215808745:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21304
2025-11-19T13:59:42.344286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435885822107659:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55300
2025-11-19T13:59:42.344364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435885822107660:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55300
2025-11-19T13:59:42.347867Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435887215808741:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.347920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435887215808662:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.348180Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435887215808693:2076] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:42.349092Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435887215808745:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.349111Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435887215808742:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.349122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435887215808714:2078] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.349136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:42.349144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435887215808714:2078] [node 4] ICP22 created new session: [6:7574435887215808747:2078]
2025-11-19T13:59:42.349149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435887215808747:2078] [node 4] ICS09 handshake done sender: [6:7574435887215808693:2076] self: [6:7574435887215808721:0] peer: [4:7574435883959044940:0] socket: 560 qp: -1
2025-11-19T13:59:42.349157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435887215808747:2078] [node 4] ICS10 traffic start
2025-11-19T13:59:42.349167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:42.348611Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435885822107659:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.348690Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435885822107660:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.348822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435885822107579:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.351830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435885339630214:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:42.348254Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435885445059037:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.348271Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435885445059037:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.351907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435885445059045:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:42.352324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435883959044938:2095] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.352348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:42.352356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435883959044938:2095] [node 6] ICP22 created new session: [4:7574435883959044965:2095]
2025-11-19T13:59:42.352362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435883959044965:2095] [node 6] ICS09 handshake done sender: [4:7574435883959044942:2100] self: [4:7574435883959044940:0] peer: [6:7574435887215808721:0] socket: 558 qp: -1
2025-11-19T13:59:42.352370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435883959044965:2095] [node 6] ICS10 traffic start
2025-11-19T13:59:42.352379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:42.352055Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435885822107581:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(43950287ns) {Configure:104(43944443ns) {ProcessConfigured:135(42431005ns) {StartInitialHandshake:157(42430178ns) {DropHandshakes:469(108ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(28ns)} GenerateSessionVirtualId:988(79ns) PrepareNewSessionHandshake:476(1370ns) {SwitchToState:178(655ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.356431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435883959044967:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37727
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.364047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435885822107663:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55300
2025-11-19T13:59:42.364651Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883418647291:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.364676Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435883418647291:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.364828Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885339630206:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.364633Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435883959044944:2101] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.364650Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435883959044944:2101] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.364991Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435885445059037:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.368054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435887215808661:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.368087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435887215808684:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.367460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435883959044938:2095] [node 6] ICP35 event processing took too much time PendingConnection:278(47190833ns) {HandleHandshakeStatus:329(47189736ns) {DropIncomingHandshake:441(922ns) DropOutgoingHandshake:456(240ns) SwitchToState:178(58ns) ProcessPendingSessionEvents:495(51294ns) {ForwardSessionEventToSession:612(41367ns) ForwardSessionEventToSession:612(864ns) ForwardSessionEventToSession:612(942ns) ForwardSessionEventToSession:612(809ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(1398ns) ForwardSessionEventToSession:612(846ns) ForwardSessionEventToSession:612(1383ns)}}}
2025-11-19T13:59:42.368390Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435883959044958:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:42.368219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435887215808749:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21304
2025-11-19T13:59:42.371873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435885445059035:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.371909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:42.371918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435885445059035:2111] [node 6] ICP22 created new session: [2:7574435885445059062:2111]
2025-11-19T13:59:42.371926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435885445059062:2111] [node 6] ICS09 handshake done sender: [2:7574435885445059037:2113] self: [2:7574435885445059036:0] peer: [6:7574435887215808746:0] socket: 571 qp: -1
2025-11-19T13:59:42.371933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435885445059062:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:42.371947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:42.372101Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885339630207:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.371884Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435885822107627:2089] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.371904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435885822107627:2089] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.372363Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435885822107663:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.372392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435885822107581:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.373503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435883418647315:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44128
2025-11-19T13:59:42.373521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435883418647316:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44128
2025-11-19T13:59:42.373982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435883959044885:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.375475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435885822107666:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55300
2025-11-19T13:59:42.374001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:42.375494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435885822107667:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55300
2025-11-19T13:59:42.374006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435883959044885:2049] [node 2] ICP22 created new session: [4:7574435883959044970:2049]
2025-11-19T13:59:42.374012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435883959044970:2049] [node 2] ICS09 handshake done sender: [4:7574435883959044958:2112] self: [4:7574435883959044960:0] peer: [2:7574435885445059033:0] socket: 582 qp: -1
2025-11-19T13:59:42.374017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435883959044970:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:42.374038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:42.375677Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435883418647291:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:42.372575Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435885445059045:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(20580462ns) {Configure:104(20579784ns) {ProcessConfigured:135(20223850ns) {StartInitialHandshake:157(20209558ns) {DropHandshakes:469(557ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(346ns) PrepareNewSessionHandshake:476(1002ns) {SwitchToState:178(709ns)}} IncomingHandshake:261(5258ns) {DropIncomingHandshake:441(32ns) PrepareNewSessionHandshake:476(176ns) {SwitchToState:178(117ns)} GenerateSessionVirtualId:988(228ns) IssueIncomingHandshakeReply:193(246ns)}}}}
2025-11-19T13:59:42.373136Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435887215808741:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:42.375919Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435885822107660:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:42.376060Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435883959044944:2101] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:42.376210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435883959044939:2098] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.376229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:42.376236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435883959044939:2098] [node 5] ICP22 created new session: [4:7574435883959044973:2098]
2025-11-19T13:59:42.376240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435883959044973:2098] [node 5] ICS09 handshake done sender: [4:7574435883959044944:2101] self: [4:7574435883959044941:0] peer: [5:7574435885822107658:0] socket: 559 qp: -1
2025-11-19T13:59:42.376245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435883959044973:2098] [node 5] ICS10 traffic start
2025-11-19T13:59:42.376252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:42.376044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435883418647287:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.376062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:42.376070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435883418647287:2107] [node 5] ICP22 created new session: [1:7574435883418647318:2107]
2025-11-19T13:59:42.376075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435883418647318:2107] [node 5] ICS09 handshake done sender: [1:7574435883418647291:2109] self: [1:7574435883418647290:0] peer: [5:7574435885822107662:0] socket: 567 qp: -1
2025-11-19T13:59:42.376081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435883418647318:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:42.376090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:42.376249Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883418647315:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.376024Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435887215808714:2078] [node 4] ICP35 event processing took too much time PendingConnection:278(26856921ns) {HandleHandshakeStatus:329(26856088ns) {DropIncomingHandshake:441(211ns) DropOutgoingHandshake:456(2104ns) SwitchToState:178(97ns) ProcessPendingSessionEvents:495(11230703ns) {ForwardSessionEventToSession:612(11215573ns) ForwardSessionEventToSession:612(1876ns) ForwardSessionEventToSession:612(968ns) ForwardSessionEventToSession:612(643ns) ForwardSessionEventToSession:612(793ns) ForwardSessionEventToSession:612(1446ns) ForwardSessionEventToSession:612(671ns) ForwardSessionEventToSession:612(707ns) ForwardSessionEventToSession:612(1120ns) ForwardSessionEventToSession:612(1191ns)}}}
2025-11-19T13:59:42.379975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435883959044968:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37727
2025-11-19T13:59:42.380080Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883418647301:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.380098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435883418647301:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.380208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435885822107665:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55300
2025-11-19T13:59:42.380909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435885822107579:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.380929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:42.380936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435885822107579:2048] [node 1] ICP22 created new session: [5:7574435885822107668:2048]
2025-11-19T13:59:42.380943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435885822107668:2048] [node 1] ICS09 handshake done sender: [5:7574435885822107660:2117] self: [5:7574435885822107662:0] peer: [1:7574435883418647290:0] socket: 589 qp: -1
2025-11-19T13:59:42.380953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435885822107668:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:42.380972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:42.381110Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435885822107665:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.381218Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435885822107656:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:42.381333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435885822107599:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.381346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:42.381352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435885822107599:2065] [node 4] ICP22 created new session: [5:7574435885822107670:2065]
2025-11-19T13:59:42.381356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435885822107670:2065] [node 4] ICS09 handshake done sender: [5:7574435885822107656:2114] self: [5:7574435885822107658:0] peer: [4:7574435883959044941:0] socket: 568 qp: -1
2025-11-19T13:59:42.381360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435885822107670:2065] [node 4] ICS10 traffic start
2025-11-19T13:59:42.381366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:42.381479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435885822107580:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.380155Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885339630226:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.380171Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435885339630226:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.382724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435887215808662:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435887215808741:2116] held: no
2025-11-19T13:59:42.382758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435887215808662:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.382770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:42.382775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435887215808662:2049] [node 2] ICP22 created new session: [6:7574435887215808753:2049]
2025-11-19T13:59:42.382787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435887215808753:2049] [node 2] ICS09 handshake done sender: [6:7574435887215808741:2116] self: [6:7574435887215808746:0] peer: [2:7574435885445059036:0] socket: 574 qp: -1
2025-11-19T13:59:42.382796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435887215808753:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:42.382806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:42.382618Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435885445059064:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.382633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435885445059064:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.383322Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883418647316:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.385600Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883418647304:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.385605Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435883418647304:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.385144Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435883959044968:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.385229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435883959044884:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.386567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435883959044976:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37727
2025-11-19T13:59:42.386589Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435885822107627:2089] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:42.386673Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435883418647304:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:42.383929Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435885445059064:2120] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:42.384045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435885445059045:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.384063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:42.384071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435885445059045:2110] [node 5] ICP22 created new session: [2:7574435885445059068:2110]
2025-11-19T13:59:42.384078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435885445059068:2110] [node 5] ICS09 handshake done sender: [2:7574435885445059064:2120] self: [2:7574435885445059060:0] peer: [5:7574435885822107672:0] socket: 563 qp: -1
2025-11-19T13:59:42.384084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435885445059068:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:42.384092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.388072Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435885822107580:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435885822107665:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.388466Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435883959044968:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:42.388550Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435883418647301:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:42.388597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435883418647289:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.388616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:42.388624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435883418647289:2108] [node 4] ICP22 created new session: [1:7574435883418647324:2108]
2025-11-19T13:59:42.388631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435883418647324:2108] [node 4] ICS09 handshake done sender: [1:7574435883418647304:2114] self: [1:7574435883418647296:0] peer: [4:7574435883959044975:0] socket: 593 qp: -1
2025-11-19T13:59:42.388637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435883418647324:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:42.388643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:42.391841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435885339630214:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(39710940ns) {Configure:104(39709264ns) {ProcessConfigured:135(39374236ns) {StartInitialHandshake:157(39372804ns) {DropHandshakes:469(575ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(132ns)} GenerateSessionVirtualId:988(297ns) PrepareNewSessionHandshake:476(1769ns) {SwitchToState:178(1407ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.393863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435885822107675:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55300
2025-11-19T13:59:42.393731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435885339630241:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25085
2025-11-19T13:59:42.394258Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435887215808744:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.394302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435887215808663:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.396123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435885822107613:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.396155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:42.396161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435885822107613:2069] [node 6] ICP22 created new session: [5:7574435885822107676:2069]
2025-11-19T13:59:42.396168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435885822107676:2069] [node 6] ICS09 handshake done sender: [5:7574435885822107627:2089] self: [5:7574435885822107616:0] peer: [6:7574435887215808751:0] socket: 575 qp: -1
2025-11-19T13:59:42.396178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435885822107676:2069] [node 6] ICS10 traffic start
2025-11-19T13:59:42.396189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:42.396681Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435885822107665:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:42.396757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435885822107580:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.396771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:42.396777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435885822107580:2049] [node 2] ICP22 created new session: [5:7574435885822107678:2049]
2025-11-19T13:59:42.396782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435885822107678:2049] [node 2] ICS09 handshake done sender: [5:7574435885822107665:2119] self: [5:7574435885822107672:0] peer: [2:7574435885445059060:0] socket: 611 qp: -1
2025-11-19T13:59:42.396787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435885822107678:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:42.396793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.395951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435887215808755:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21304
2025-11-19T13:59:42.398492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435887215808757:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21304
2025-11-19T13:59:42.398973Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885339630241:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.400000Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435885339630226:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:42.401004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435885339630210:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.401025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:42.401033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435885339630210:2109] [node 5] ICP22 created new session: [3:7574435885339630244:2109]
2025-11-19T13:59:42.401039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435885339630244:2109] [node 5] ICS09 handshake done sender: [3:7574435885339630226:2117] self: [3:7574435885339630212:0] peer: [5:7574435885822107664:0] socket: 595 qp: -1
2025-11-19T13:59:42.401045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435885339630244:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:42.401054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:42.401496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435887215808684:2067] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1240) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435887215808742:2117] held: no
2025-11-19T13:59:42.401395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435883959044884:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.401417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:42.401426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435883959044884:2048] [node 1] ICP22 created new session: [4:7574435883959044977:2048]
2025-11-19T13:59:42.401432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435883959044977:2048] [node 1] ICS09 handshake done sender: [4:7574435883959044968:2117] self: [4:7574435883959044975:0] peer: [1:7574435883418647296:0] socket: 604 qp: -1
2025-11-19T13:59:42.401438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435883959044977:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:42.401445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.402641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435883418647295:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.402664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:42.402668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435883418647295:2110] [node 6] ICP22 created new session: [1:7574435883418647326:2110]
2025-11-19T13:59:42.402674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435883418647326:2110] [node 6] ICS09 handshake done sender: [1:7574435883418647301:2113] self: [1:7574435883418647300:0] peer: [6:7574435887215808750:0] socket: 597 qp: -1
2025-11-19T13:59:42.402680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435883418647326:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:42.402688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:42.402519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435883959044979:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37727
2025-11-19T13:59:42.403506Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885339630222:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.403526Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435885339630222:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.404304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435885822107680:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55300
2025-11-19T13:59:42.404329Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435883959044979:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.404362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435883959044886:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.405093Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885339630238:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.405101Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435885339630238:2120] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.406059Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435885822107663:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:42.406068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435883959044981:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37727
2025-11-19T13:59:42.406180Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435885339630222:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:42.406189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435885822107581:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.406206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:42.406211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435885822107581:2050] [node 3] ICP22 created new session: [5:7574435885822107681:2050]
2025-11-19T13:59:42.406216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435885822107681:2050] [node 3] ICS09 handshake done sender: [5:7574435885822107663:2118] self: [5:7574435885822107664:0] peer: [3:7574435885339630212:0] socket: 606 qp: -1
2025-11-19T13:59:42.406222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435885822107681:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:42.406230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:42.407430Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435885339630238:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:42.407605Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435883959044979:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:42.407671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435883959044886:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.407686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:42.407692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435883959044886:2050] [node 3] ICP22 created new session: [4:7574435883959044982:2050]
2025-11-19T13:59:42.407699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435883959044982:2050] [node 3] ICS09 handshake done sender: [4:7574435883959044979:2119] self: [4:7574435883959044980:0] peer: [3:7574435885339630235:0] socket: 562 qp: -1
2025-11-19T13:59:42.407704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435883959044982:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:42.407712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:42.408890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435885339630216:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.408910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:42.408916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435885339630216:2111] [node 6] ICP22 created new session: [3:7574435885339630248:2111]
2025-11-19T13:59:42.408922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435885339630248:2111] [node 6] ICS09 handshake done sender: [3:7574435885339630222:2116] self: [3:7574435885339630221:0] peer: [6:7574435887215808756:0] socket: 590 qp: -1
2025-11-19T13:59:42.408928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435885339630248:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:42.408934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:42.409319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435885339630214:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.409333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:42.409338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435885339630214:2110] [node 4] ICP22 created new session: [3:7574435885339630250:2110]
2025-11-19T13:59:42.409344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435885339630250:2110] [node 4] ICS09 handshake done sender: [3:7574435885339630238:2120] self: [3:7574435885339630235:0] peer: [4:7574435883959044980:0] socket: 583 qp: -1
2025-11-19T13:59:42.409349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435885339630250:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:42.409356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:42.411957Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435887215808742:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:42.412124Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435887215808745:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.415095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435887215808684:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.415118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:42.415127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435887215808684:2067] [node 5] ICP22 created new session: [6:7574435887215808759:2067]
2025-11-19T13:59:42.415132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435887215808759:2067] [node 5] ICS09 handshake done sender: [6:7574435887215808742:2117] self: [6:7574435887215808751:0] peer: [5:7574435885822107616:0] socket: 592 qp: -1
2025-11-19T13:59:42.415139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435887215808759:2067] [node 5] ICS10 traffic start
2025-11-19T13:59:42.415148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:42.415244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435887215808661:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.415252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:42.415254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435887215808661:2048] [node 1] ICP22 created new session: [6:7574435887215808761:2048]
2025-11-19T13:59:42.415257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435887215808761:2048] [node 1] ICS09 handshake done sender: [6:7574435887215808745:2120] self: [6:7574435887215808750:0] peer: [1:7574435883418647300:0] socket: 603 qp: -1
2025-11-19T13:59:42.415260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435887215808761:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:42.415264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:42.415328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435887215808661:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:42.415426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435887215808663:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64865) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435887215808744:2119] held: no
2025-11-19T13:59:42.415500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435887215808758:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21304
2025-11-19T13:59:42.417526Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435887215808744:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:42.417682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435887215808663:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.417701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:42.417708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435887215808663:2050] [node 3] ICP22 created new session: [6:7574435887215808763:2050]
2025-11-19T13:59:42.417713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435887215808763:2050] [node 3] ICS09 handshake done sender: [6:7574435887215808744:2119] self: [6:7574435887215808756:0] peer: [3:7574435885339630221:0] socket: 602 qp: -1
2025-11-19T13:59:42.417720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435887215808763:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:42.417732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:42.422185Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:42.422272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435885445059054:2048] [node 1] ICS01 socket: 580 reason# EndOfStream
2025-11-19T13:59:42.422279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435885445058966:2048] [node 1] ICP30 unregister session Session# [2:7574435885445059054:2048] VirtualId# [2:7574435885445059050:0]
2025-11-19T13:59:42.422286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435885445059054:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:42.422321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:42.422340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:42.422726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:42.423113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435885339630231:2048] [node 1] ICS01 socket: 587 reason# EndOfStream
2025-11-19T13:59:42.423137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435885339630146:2048] [node 1] ICP30 unregister session Session# [3:7574435885339630231:2048] VirtualId# [3:7574435885339630227:0]
2025-11-19T13:59:42.423145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435885339630231:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:42.423213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:42.423239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:42.433872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:42.433985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435883959044977:2048] [node 1] ICS01 socket: 604 reason# EndOfStream
2025-11-19T13:59:42.433995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435883959044884:2048] [node 1] ICP30 unregister session Session# [4:7574435883959044977:2048] VirtualId# [4:7574435883959044975:0]
2025-11-19T13:59:42.434004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435883959044977:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:42.434072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:42.434098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:42.436929Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:42.437032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435885822107668:2048] [node 1] ICS01 socket: 589 reason# EndOfStream
2025-11-19T13:59:42.437042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435885822107579:2048] [node 1] ICP30 unregister session Session# [5:7574435885822107668:2048] VirtualId# [5:7574435885822107662:0]
2025-11-19T13:59:42.437051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435885822107668:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:42.437110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:42.437137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:42.437178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:42.437244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435887215808761:2048] [node 1] ICS07 socket disconnect 603 reason# EndOfStream
2025-11-19T13:59:42.437261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435887215808761:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:42.437292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:42.437315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435887215808761:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:42.438607Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:42.438684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435885339630233:2049] [node 2] ICS01 socket: 586 reason# EndOfStream
2025-11-19T13:59:42.438691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435885339630147:2049] [node 2] ICP30 unregister session Session# [3:7574435885339630233:2049] VirtualId# [3:7574435885339630228:0]
2025-11-19T13:59:42.438696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435885339630233:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:42.438725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:42.438743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:42.439214Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435887215808661:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24748) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:42.439229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435887215808761:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:42.439233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435887215808661:2048] [node 1] ICP30 unregister session Session# [6:7574435887215808761:2048] VirtualId# [6:7574435887215808750:0]
2025-11-19T13:59:42.439246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:42.439252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435887215808661:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24748) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:42.439255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24748) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:42.444048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:42.444147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435883959044970:2049] [node 2] ICS01 socket: 582 reason# EndOfStream
2025-11-19T13:59:42.444157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435883959044885:2049] [node 2] ICP30 unregister session Session# [4:7574435883959044970:2049] VirtualId# [4:7574435883959044960:0]
2025-11-19T13:59:42.444165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435883959044970:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:42.444226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:42.444252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:42.448261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:42.448363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435885822107678:2049] [node 2] ICS01 socket: 611 reason# EndOfStream
2025-11-19T13:59:42.448372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435885822107580:2049] [node 2] ICP30 unregister session Session# [5:7574435885822107678:2049] VirtualId# [5:7574435885822107672:0]
2025-11-19T13:59:42.448382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435885822107678:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:42.448440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:42.448467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:42.448580Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:42.448643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435887215808753:2049] [node 2] ICS01 socket: 574 reason# EndOfStream
2025-11-19T13:59:42.448648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435887215808662:2049] [node 2] ICP30 unregister session Session# [6:7574435887215808753:2049] VirtualId# [6:7574435887215808746:0]
2025-11-19T13:59:42.448653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435887215808753:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:42.448686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:42.448709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:42.451853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435887215808661:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:42.464156Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:42.464283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435885822107681:2050] [node 3] ICS01 socket: 606 reason# EndOfStream
2025-11-19T13:59:42.464293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435885822107581:2050] [node 3] ICP30 unregister session Session# [5:7574435885822107681:2050] VirtualId# [5:7574435885822107664:0]
2025-11-19T13:59:42.464302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435885822107681:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:42.464366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:42.464393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:42.464436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:42.464505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435887215808763:2050] [node 3] ICS01 socket: 602 reason# EndOfStream
2025-11-19T13:59:42.464511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435887215808663:2050] [node 3] ICP30 unregister session Session# [6:7574435887215808763:2050] VirtualId# [6:7574435887215808756:0]
2025-11-19T13:59:42.464516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435887215808763:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:42.464553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:42.464575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:42.493092Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:42.493203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435887215808747:2078] [node 4] ICS07 socket disconnect 560 reason# EndOfStream
2025-11-19T13:59:42.493212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435887215808747:2078] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:42.493282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:42.493308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435887215808747:2078] [node 4] ICS15 start handshake
2025-11-19T13:59:42.493692Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T13:59:42.493782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435885822107670:2065] [node 4] ICS01 socket: 568 reason# EndOfStream
2025-11-19T13:59:42.493788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435885822107599:2065] [node 4] ICP30 unregister session Session# [5:7574435885822107670:2065] VirtualId# [5:7574435885822107658:0]
2025-11-19T13:59:42.493796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435885822107670:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:42.493832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:42.493863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:42.494248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435887215808714:2078] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24467) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:42.494262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435887215808747:2078] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:42.494267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435887215808714:2078] [node 4] ICP30 unregister session Session# [6:7574435887215808747:2078] VirtualId# [6:7574435887215808721:0]
2025-11-19T13:59:42.494282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:42.494287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435887215808714:2078] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24467) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:42.494291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24467) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:42.506270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435887215808714:2078] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:42.741818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435883353989805:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:42.741835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435883353989806:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:42.783991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435884799021904:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:42.784006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435884799021905:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:42.819917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435885681079615:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:42.819933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435885681079616:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:42.847950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435884494881662:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:42.847969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435884494881663:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:42.847973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435884494881664:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:42.880599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435886606258642:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:42.880620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435886606258643:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:42.880626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435886606258644:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:42.887908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435885841253971:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:42.887908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435885841253970:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:42.887925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435885841253972:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:42.897684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435883353989852:2092] [node 4] ICP01 ready to work
2025-11-19T13:59:42.897771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435884799021939:2080] [node 5] ICP01 ready to work
2025-11-19T13:59:42.899369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435884494881703:2080] [node 5] ICP01 ready to work
2025-11-19T13:59:42.897552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435885681079656:2087] [node 6] ICP01 ready to work
2025-11-19T13:59:42.898162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435885681079662:2089] [node 5] ICP01 ready to work
2025-11-19T13:59:42.899597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435884799021951:2084] [node 6] ICP01 ready to work
2025-11-19T13:59:42.900049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435885841253995:2069] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.900529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435884799021959:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44327
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.903229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435886606258691:2085] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.904033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435883353989869:2102] [node 6] ICP01 ready to work
2025-11-19T13:59:42.904496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435883353989867:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21514
2025-11-19T13:59:42.905409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435883353989868:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21514
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.907544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435886606258698:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-19T13:59:42.907892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435885841253991:2065] [node 4] ICP01 ready to work
2025-11-19T13:59:42.911989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435883353989881:2113] [node 5] ICP01 ready to work
2025-11-19T13:59:42.914910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435886606258701:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-19T13:59:42.915018Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435886606258698:2098] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.915897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435886606258679:2080] [node 4] ICP01 ready to work
2025-11-19T13:59:42.916050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435884799021952:2086] [node 4] ICP01 ready to work
2025-11-19T13:59:42.925461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435886606258643:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.928666Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883353989867:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.929215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435884799021983:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44327
2025-11-19T13:59:42.929271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435884799021984:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44327
2025-11-19T13:59:42.929297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435884799021985:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44327
2025-11-19T13:59:42.929893Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435884799021984:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.931329Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435886606258701:2099] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.929444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435883353989885:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21514
2025-11-19T13:59:42.932089Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435885841253991:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24135380ns) {Configure:104(24134890ns) {ProcessConfigured:135(23815162ns) {StartInitialHandshake:157(23814888ns) {DropHandshakes:469(540ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(307ns) PrepareNewSessionHandshake:476(914ns) {SwitchToState:178(551ns)}}}}}
2025-11-19T13:59:42.929485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435883353989886:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21514
2025-11-19T13:59:42.931980Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883353989885:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.932008Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883353989886:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.932473Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435884799021983:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.932502Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435884799021985:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.932139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435885681079615:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24229071ns) {Configure:104(24228255ns) {ProcessConfigured:135(23847024ns) {StartInitialHandshake:157(23846117ns) {DropHandshakes:469(600ns) {DropIncomingHandshake:441(109ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(526ns) PrepareNewSessionHandshake:476(704ns) {SwitchToState:178(353ns)}}}}}
2025-11-19T13:59:42.931939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435884494881663:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(32653872ns) {Configure:104(24015787ns) {ProcessConfigured:135(23646764ns) {StartInitialHandshake:157(23645474ns) {DropHandshakes:469(638ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(146ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:476(1260ns) {SwitchToState:178(1069ns)}}}}}
2025-11-19T13:59:42.932884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435885681079677:2097] [node 4] ICP01 ready to work
2025-11-19T13:59:42.932583Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435885841253995:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(32488838ns) {Configure:104(32488322ns) {ProcessConfigured:135(32239463ns) {StartInitialHandshake:157(32238731ns) {DropHandshakes:469(463ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:476(767ns) {SwitchToState:178(28ns)}}}}}
2025-11-19T13:59:42.935325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435884799021942:2082] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.935347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435884799021942:2082] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.933547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435884494881727:2078] [node 6] ICP01 ready to work
2025-11-19T13:59:42.938434Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883353989873:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.938448Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435883353989873:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.939319Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435883353989873:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:42.936644Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435885841253972:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(37636510ns) {Configure:104(37635557ns) {ProcessConfigured:135(37207147ns) {StartInitialHandshake:157(37206517ns) {DropHandshakes:469(488ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(942ns) {SwitchToState:178(427ns)}}}}}
2025-11-19T13:59:42.936501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435886606258721:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-19T13:59:42.939467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435883353989805:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.939488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:42.939495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435883353989805:2048] [node 2] ICP22 created new session: [1:7574435883353989892:2048]
2025-11-19T13:59:42.939502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435883353989892:2048] [node 2] ICS09 handshake done sender: [1:7574435883353989873:2108] self: [1:7574435883353989861:0] peer: [2:7574435884799021987:0] socket: 582 qp: -1
2025-11-19T13:59:42.939509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435883353989892:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:42.939536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:42.940088Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435886606258721:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.943853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435886606258679:2080] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(27870685ns) {Configure:104(27869684ns) {ProcessConfigured:135(27527940ns) {StartInitialHandshake:157(27527385ns) {DropHandshakes:469(548ns) {DropIncomingHandshake:441(107ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(1422ns) {SwitchToState:178(837ns)}}}}}
2025-11-19T13:59:42.943892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435886606258679:2080] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.940604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435885841254048:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14679
2025-11-19T13:59:42.940668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435885841254049:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14679
2025-11-19T13:59:42.944113Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435885841254048:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.944137Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435885841254049:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.944161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435885841253995:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.944170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435885841253972:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.944127Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435884799021952:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(27997190ns) {Configure:104(27996345ns) {ProcessConfigured:135(27645832ns) {StartInitialHandshake:157(27645582ns) {DropHandshakes:469(548ns) {DropIncomingHandshake:441(45ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:476(1192ns) {SwitchToState:178(1026ns)}}}}}
2025-11-19T13:59:42.944545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435885841254050:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14679
2025-11-19T13:59:42.947947Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435885841254050:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.947989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435885841253970:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.951856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435885681079677:2097] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18905660ns) {Configure:104(18904567ns) {ProcessConfigured:135(18524559ns) {StartInitialHandshake:157(18523971ns) {DropHandshakes:469(597ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(172ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(1435ns) {SwitchToState:178(1266ns)}}}}}
2025-11-19T13:59:42.952215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435884494881740:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5984
2025-11-19T13:59:42.952240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435884494881741:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5984
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.952976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435885681079695:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21772
2025-11-19T13:59:42.953052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435885681079694:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21772
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.953183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435885681079696:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21772
2025-11-19T13:59:42.954251Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883353989868:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.954474Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435884494881741:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.954526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435884494881662:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.954655Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883353989876:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.954660Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435883353989876:2109] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.955419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435884494881743:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5984
2025-11-19T13:59:42.955523Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435883353989876:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:42.955508Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885681079696:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.955536Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885681079694:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.955568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435885681079615:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.955660Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435884494881740:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.955685Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435884494881741:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:42.956050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435884494881662:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:42.956066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:42.956074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435884494881662:2048] [node 1] ICP22 created new session: [4:7574435884494881745:2048]
2025-11-19T13:59:42.956080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435884494881745:2048] [node 1] ICS09 handshake done sender: [4:7574435884494881741:2116] self: [4:7574435884494881742:0] peer: [1:7574435883353989859:0] socket: 600 qp: -1
2025-11-19T13:59:42.956086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435884494881745:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:42.956105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:42.956969Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883353989855:2095] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.956982Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435883353989855:2095] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.957700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435883353989852:2092] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.957716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:42.957723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435883353989852:2092] [node 4] ICP22 created new session: [1:7574435883353989897:2092]
2025-11-19T13:59:42.957729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435883353989897:2092] [node 4] ICS09 handshake done sender: [1:7574435883353989876:2109] self: [1:7574435883353989859:0] peer: [4:7574435884494881742:0] socket: 597 qp: -1
2025-11-19T13:59:42.957735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435883353989897:2092] [node 4] ICS10 traffic start
2025-11-19T13:59:42.957745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:42.958573Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435886606258694:2095] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.958596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435886606258694:2095] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.958830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435884799021904:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.958874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435884799021904:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(26328289ns) {IncomingHandshake:261(26327873ns) {DropIncomingHandshake:441(70ns) PrepareNewSessionHandshake:476(322ns) {SwitchToState:178(115ns)} GenerateSessionVirtualId:988(203ns) IssueIncomingHandshakeReply:193(26324380ns) {SwitchToState:178(1193ns)}}}
2025-11-19T13:59:42.958698Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435884494881729:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.958722Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435884494881729:2107] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.958750Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885681079695:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.958751Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885681079661:2091] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.958759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435885681079661:2091] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.958779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435885681079616:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.958953Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435884799021942:2082] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:42.959372Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435884799021959:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.959393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435884799021904:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2644) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435884799021983:2115] held: no
2025-11-19T13:59:42.959920Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883353989871:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.959936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435883353989871:2106] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.960186Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435883353989855:2095] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:42.964517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435883353989806:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.964541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:42.964551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435883353989806:2049] [node 3] ICP22 created new session: [1:7574435883353989901:2049]
2025-11-19T13:59:42.964558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435883353989901:2049] [node 3] ICS09 handshake done sender: [1:7574435883353989855:2095] self: [1:7574435883353989854:0] peer: [3:7574435885681079698:0] socket: 564 qp: -1
2025-11-19T13:59:42.964567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435883353989901:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:42.964574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:42.967844Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435884799021957:2093] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.967868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435884799021957:2093] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:42.967989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435886606258644:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(35753251ns) {Configure:104(35752475ns) {ProcessConfigured:135(35433591ns) {StartInitialHandshake:157(35433299ns) {DropHandshakes:469(334ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:476(1070ns) {SwitchToState:178(713ns)}}}}}
2025-11-19T13:59:42.968043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435886606258644:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.973682Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435884494881729:2107] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:42.973836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435884494881703:2080] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.973857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:42.973866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435884494881703:2080] [node 5] ICP22 created new session: [4:7574435884494881749:2080]
2025-11-19T13:59:42.973873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435884494881749:2080] [node 5] ICS09 handshake done sender: [4:7574435884494881729:2107] self: [4:7574435884494881705:0] peer: [5:7574435886606258723:0] socket: 586 qp: -1
2025-11-19T13:59:42.973881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435884494881749:2080] [node 5] ICS10 traffic start
2025-11-19T13:59:42.973891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:42.975976Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435886606258694:2095] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:42.976122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435886606258691:2085] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.976137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:42.976147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435886606258691:2085] [node 6] ICP22 created new session: [5:7574435886606258727:2085]
2025-11-19T13:59:42.976153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435886606258727:2085] [node 6] ICS09 handshake done sender: [5:7574435886606258694:2095] self: [5:7574435886606258693:0] peer: [6:7574435885841254051:0] socket: 575 qp: -1
2025-11-19T13:59:42.976160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435886606258727:2085] [node 6] ICS10 traffic start
2025-11-19T13:59:42.976169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.978515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435884494881752:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5984
2025-11-19T13:59:42.978562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435884494881753:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5984
2025-11-19T13:59:42.978705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435886606258643:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10157) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435886606258698:2098] held: no
2025-11-19T13:59:42.979023Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435884494881752:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.979048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435884494881703:2080] [node 5] ICP18 handshake (actor [4:7574435884494881752:2118]) is too old
2025-11-19T13:59:42.976989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435884799021939:2080] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.977014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:42.977022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435884799021939:2080] [node 5] ICP22 created new session: [2:7574435884799021991:2080]
2025-11-19T13:59:42.977030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435884799021991:2080] [node 5] ICS09 handshake done sender: [2:7574435884799021942:2082] self: [2:7574435884799021941:0] peer: [5:7574435886606258718:0] socket: 566 qp: -1
2025-11-19T13:59:42.977038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435884799021991:2080] [node 5] ICS10 traffic start
2025-11-19T13:59:42.977048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:42.980036Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435885681079661:2091] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:42.980168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435885681079656:2087] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.980182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:42.980188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435885681079656:2087] [node 6] ICP22 created new session: [3:7574435885681079701:2087]
2025-11-19T13:59:42.980195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435885681079701:2087] [node 6] ICS09 handshake done sender: [3:7574435885681079661:2091] self: [3:7574435885681079660:0] peer: [6:7574435885841254052:0] socket: 577 qp: -1
2025-11-19T13:59:42.980202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435885681079701:2087] [node 6] ICS10 traffic start
2025-11-19T13:59:42.980225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:42.981259Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435883353989871:2106] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:42.981397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435883353989869:2102] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.981415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:42.981423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435883353989869:2102] [node 6] ICP22 created new session: [1:7574435883353989904:2102]
2025-11-19T13:59:42.981429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435883353989904:2102] [node 6] ICS09 handshake done sender: [1:7574435883353989871:2106] self: [1:7574435883353989870:0] peer: [6:7574435885841254053:0] socket: 581 qp: -1
2025-11-19T13:59:42.981435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435883353989904:2102] [node 6] ICS10 traffic start
2025-11-19T13:59:42.981454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:42.982265Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435884494881662:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(26175135ns) {HandleHandshakeStatus:329(26174663ns) {DropIncomingHandshake:441(220ns) DropOutgoingHandshake:456(1726ns) SwitchToState:178(117ns) ProcessPendingSessionEvents:495(22215517ns) {ForwardSessionEventToSession:612(22202136ns) ForwardSessionEventToSession:612(2385ns) ForwardSessionEventToSession:612(1426ns) ForwardSessionEventToSession:612(917ns) ForwardSessionEventToSession:612(606ns) ForwardSessionEventToSession:612(994ns) ForwardSessionEventToSession:612(1208ns) ForwardSessionEventToSession:612(694ns)}}}
2025-11-19T13:59:42.987948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435885841253970:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2644) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435885841254050:2117] held: no
2025-11-19T13:59:42.988839Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885681079674:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.988863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435885681079674:2102] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.992791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435884799021996:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44327
2025-11-19T13:59:42.993495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435884799021997:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44327
2025-11-19T13:59:42.991858Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435884494881753:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.991902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435884494881663:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:42.996224Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435884799021957:2093] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:42.996839Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435886606258679:2080] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24599) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435886606258721:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:42.996822Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435884799021997:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:42.996849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435884799021905:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:42.996871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:42.996879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435884799021905:2049] [node 3] ICP22 created new session: [2:7574435884799021999:2049]
2025-11-19T13:59:42.996886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435884799021999:2049] [node 3] ICS09 handshake done sender: [2:7574435884799021957:2093] self: [2:7574435884799021940:0] peer: [3:7574435885681079699:0] socket: 570 qp: -1
2025-11-19T13:59:42.996894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435884799021999:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:42.996903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:42.997897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435883353989906:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21514
2025-11-19T13:59:42.999797Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435884799021983:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:43.000080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435886606258691:2085] [node 6] ICP35 event processing took too much time PendingConnection:278(23925589ns) {HandleHandshakeStatus:329(23924686ns) {DropIncomingHandshake:441(143ns) DropOutgoingHandshake:456(235ns) SwitchToState:178(317ns) ProcessPendingSessionEvents:495(13194541ns) {ForwardSessionEventToSession:612(13178725ns) ForwardSessionEventToSession:612(2399ns) ForwardSessionEventToSession:612(1208ns) ForwardSessionEventToSession:612(748ns) ForwardSessionEventToSession:612(686ns) ForwardSessionEventToSession:612(1104ns) ForwardSessionEventToSession:612(1026ns) ForwardSessionEventToSession:612(715ns) ForwardSessionEventToSession:612(752ns) ForwardSessionEventToSession:612(1169ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.000530Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435884799021989:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.000543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435884799021989:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.006919Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435885681079674:2102] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.007479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890136221351:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14679
2025-11-19T13:59:43.007520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890136221352:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14679
2025-11-19T13:59:43.007559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890136221353:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14679
2025-11-19T13:59:43.007596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890136221354:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14679
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.007892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435888789849051:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5984
2025-11-19T13:59:43.007066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435885681079662:2089] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.007083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:43.007092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435885681079662:2089] [node 5] ICP22 created new session: [3:7574435889976047001:2089]
2025-11-19T13:59:43.007098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435889976047001:2089] [node 5] ICS09 handshake done sender: [3:7574435885681079674:2102] self: [3:7574435885681079663:0] peer: [5:7574435886606258725:0] socket: 578 qp: -1
2025-11-19T13:59:43.007106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435889976047001:2089] [node 5] ICS10 traffic start
2025-11-19T13:59:43.007129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:43.007635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435889976047004:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21772
2025-11-19T13:59:43.007740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435885681079656:2087] [node 6] ICP35 event processing took too much time PendingConnection:278(27538327ns) {HandleHandshakeStatus:329(27537729ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(155ns) SwitchToState:178(217ns) ProcessPendingSessionEvents:495(18080916ns) {ForwardSessionEventToSession:612(18067351ns) ForwardSessionEventToSession:612(1174ns) ForwardSessionEventToSession:612(858ns) ForwardSessionEventToSession:612(1023ns) ForwardSessionEventToSession:612(772ns) ForwardSessionEventToSession:612(1269ns) ForwardSessionEventToSession:612(603ns) ForwardSessionEventToSession:612(838ns) ForwardSessionEventToSession:612(588ns) ForwardSessionEventToSession:612(927ns)}}}
2025-11-19T13:59:43.007772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435889976047005:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21772
2025-11-19T13:59:43.007829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435889976047006:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21772
2025-11-19T13:59:43.008952Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435890136221351:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.008980Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435890136221353:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.008999Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435885841254050:2117] [node 1] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-19T13:59:43.009069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890136221350:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14679
2025-11-19T13:59:43.011526Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435885841254049:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:43.011681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435885841253970:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.011693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:43.011702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435885841253970:2048] [node 1] ICP22 created new session: [6:7574435890136221355:2048]
2025-11-19T13:59:43.011709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435890136221355:2048] [node 1] ICS09 handshake done sender: [6:7574435885841254050:2117] self: [6:7574435885841254053:0] peer: [1:7574435883353989870:0] socket: 599 qp: -1
2025-11-19T13:59:43.011716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435890136221355:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:43.011724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:43.008255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435884799021904:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.008274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:43.008282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435884799021904:2048] [node 1] ICP22 created new session: [2:7574435889093989297:2048]
2025-11-19T13:59:43.008287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435889093989297:2048] [node 1] ICS09 handshake done sender: [2:7574435884799021983:2115] self: [2:7574435884799021987:0] peer: [1:7574435883353989861:0] socket: 583 qp: -1
2025-11-19T13:59:43.008296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435889093989297:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:43.008305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:43.008259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435886606258731:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-19T13:59:43.012054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435885841253991:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.012074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435885841253971:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.012097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435885841253972:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.012107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:43.012112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435885841253972:2050] [node 3] ICP22 created new session: [6:7574435890136221359:2050]
2025-11-19T13:59:43.012117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435890136221359:2050] [node 3] ICS09 handshake done sender: [6:7574435885841254049:2116] self: [6:7574435885841254052:0] peer: [3:7574435885681079660:0] socket: 598 qp: -1
2025-11-19T13:59:43.012123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435890136221359:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:43.012142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:43.012318Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435885841254048:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:43.008939Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435889976047005:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.012637Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435885681079616:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10157) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435885681079695:2116] held: no
2025-11-19T13:59:43.012747Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435885681079694:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:43.016095Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435884799021969:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.016110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435884799021969:2102] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.017099Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435886606258698:2098] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:43.017409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435886606258643:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.017428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:43.017437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435886606258643:2049] [node 2] ICP22 created new session: [5:7574435890901226028:2049]
2025-11-19T13:59:43.017444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435890901226028:2049] [node 2] ICS09 handshake done sender: [5:7574435886606258698:2098] self: [5:7574435886606258718:0] peer: [2:7574435884799021941:0] socket: 574 qp: -1
2025-11-19T13:59:43.017452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435890901226028:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:43.017462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:43.017823Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435884799021969:2102] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:43.018279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435884799021951:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.018299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:43.018306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435884799021951:2084] [node 6] ICP22 created new session: [2:7574435889093989301:2084]
2025-11-19T13:59:43.018313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435889093989301:2084] [node 6] ICS09 handshake done sender: [2:7574435884799021969:2102] self: [2:7574435884799021956:0] peer: [6:7574435890136221358:0] socket: 591 qp: -1
2025-11-19T13:59:43.018319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435889093989301:2084] [node 6] ICS10 traffic start
2025-11-19T13:59:43.018326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:43.015954Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435885681079695:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:43.018751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435885681079615:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.018764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:43.018770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435885681079615:2048] [node 1] ICP22 created new session: [3:7574435889976047007:2048]
2025-11-19T13:59:43.018775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435889976047007:2048] [node 1] ICS09 handshake done sender: [3:7574435885681079694:2115] self: [3:7574435885681079698:0] peer: [1:7574435883353989854:0] socket: 567 qp: -1
2025-11-19T13:59:43.018782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435889976047007:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:43.018788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:43.018935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435885681079662:2089] [node 5] ICP18 handshake (actor [3:7574435889976047005:2119]) is too old
2025-11-19T13:59:43.018946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435885681079616:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.018953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:43.018959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435885681079616:2049] [node 2] ICP22 created new session: [3:7574435889976047009:2049]
2025-11-19T13:59:43.018979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435889976047009:2049] [node 2] ICS09 handshake done sender: [3:7574435885681079695:2116] self: [3:7574435885681079699:0] peer: [2:7574435884799021940:0] socket: 601 qp: -1
2025-11-19T13:59:43.018985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435889976047009:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:43.018993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:43.023160Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435884494881739:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.023185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435884494881739:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.020279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435883353989806:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:43.023791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435888789849052:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5984
2025-11-19T13:59:43.024484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435885841253995:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.024506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:43.024516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435885841253995:2069] [node 5] ICP22 created new session: [6:7574435890136221361:2069]
2025-11-19T13:59:43.024523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435890136221361:2069] [node 5] ICS09 handshake done sender: [6:7574435885841254048:2115] self: [6:7574435885841254051:0] peer: [5:7574435886606258693:0] socket: 588 qp: -1
2025-11-19T13:59:43.024530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435890136221361:2069] [node 5] ICS10 traffic start
2025-11-19T13:59:43.024538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:43.026692Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435888789849051:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.026740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435884494881664:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.027231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435886606258644:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3157) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435886606258701:2099] held: no
2025-11-19T13:59:43.027889Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435884799021989:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.028113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435885841253971:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10157) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435890136221353:2121] held: no
2025-11-19T13:59:43.028206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890136221363:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14679
2025-11-19T13:59:43.028444Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435890136221353:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:43.028534Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435885681079693:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.028548Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435885681079693:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.028751Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435884494881753:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:43.029054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435884799021952:2086] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.029079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:43.029088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435884799021952:2086] [node 4] ICP22 created new session: [2:7574435889093989304:2086]
2025-11-19T13:59:43.029095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435889093989304:2086] [node 4] ICS09 handshake done sender: [2:7574435884799021989:2118] self: [2:7574435884799021980:0] peer: [4:7574435884494881754:0] socket: 615 qp: -1
2025-11-19T13:59:43.029101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435889093989304:2086] [node 4] ICS10 traffic start
2025-11-19T13:59:43.029109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.032175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435890901226030:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-19T13:59:43.032238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435890901226031:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-19T13:59:43.032458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435890901226032:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-19T13:59:43.032692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435885841253971:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.032711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:43.032717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435885841253971:2049] [node 2] ICP22 created new session: [6:7574435890136221364:2049]
2025-11-19T13:59:43.032725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435890136221364:2049] [node 2] ICS09 handshake done sender: [6:7574435890136221353:2121] self: [6:7574435890136221358:0] peer: [2:7574435884799021956:0] socket: 624 qp: -1
2025-11-19T13:59:43.032733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435890136221364:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:43.032741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.035969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890136221366:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14679
2025-11-19T13:59:43.036334Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435884494881739:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:43.036562Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435890136221351:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:43.036640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435885841253991:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.036657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:43.036664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435885841253991:2065] [node 4] ICP22 created new session: [6:7574435890136221367:2065]
2025-11-19T13:59:43.036670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435890136221367:2065] [node 4] ICS09 handshake done sender: [6:7574435890136221351:2119] self: [6:7574435890136221357:0] peer: [4:7574435884494881738:0] socket: 621 qp: -1
2025-11-19T13:59:43.036678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435890136221367:2065] [node 4] ICS10 traffic start
2025-11-19T13:59:43.036686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:43.037848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435884494881727:2078] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.037873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:43.037880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435884494881727:2078] [node 6] ICP22 created new session: [4:7574435888789849055:2078]
2025-11-19T13:59:43.037886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435888789849055:2078] [node 6] ICS09 handshake done sender: [4:7574435884494881739:2114] self: [4:7574435884494881738:0] peer: [6:7574435890136221357:0] socket: 611 qp: -1
2025-11-19T13:59:43.037893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435888789849055:2078] [node 6] ICS10 traffic start
2025-11-19T13:59:43.037901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:43.038775Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435885681079693:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:43.038983Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435890901226030:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.039019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435886606258642:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.039120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435885681079677:2097] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.039141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:43.039148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435885681079677:2097] [node 4] ICP22 created new session: [3:7574435889976047012:2097]
2025-11-19T13:59:43.039156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435889976047012:2097] [node 4] ICS09 handshake done sender: [3:7574435885681079693:2114] self: [3:7574435885681079692:0] peer: [4:7574435888789849053:0] socket: 622 qp: -1
2025-11-19T13:59:43.039163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435889976047012:2097] [node 4] ICS10 traffic start
2025-11-19T13:59:43.039170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:43.039521Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435886606258701:2099] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:43.039910Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435883353989883:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.039933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435883353989883:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.040116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435886606258644:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.040131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:43.040138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435886606258644:2050] [node 3] ICP22 created new session: [5:7574435890901226034:2050]
2025-11-19T13:59:43.040144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435890901226034:2050] [node 3] ICS09 handshake done sender: [5:7574435886606258701:2099] self: [5:7574435886606258725:0] peer: [3:7574435885681079663:0] socket: 580 qp: -1
2025-11-19T13:59:43.040151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435890901226034:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:43.040160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:43.041216Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435886606258721:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:43.044270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435886606258679:2080] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.044285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:43.044291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435886606258679:2080] [node 4] ICP22 created new session: [5:7574435890901226036:2080]
2025-11-19T13:59:43.044298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435890901226036:2080] [node 4] ICS09 handshake done sender: [5:7574435886606258721:2115] self: [5:7574435886606258723:0] peer: [4:7574435884494881705:0] socket: 587 qp: -1
2025-11-19T13:59:43.044306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435890901226036:2080] [node 4] ICS10 traffic start
2025-11-19T13:59:43.044315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:43.049564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435884494881663:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.049596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:43.049607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435884494881663:2049] [node 2] ICP22 created new session: [4:7574435888789849057:2049]
2025-11-19T13:59:43.049615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435888789849057:2049] [node 2] ICS09 handshake done sender: [4:7574435884494881753:2119] self: [4:7574435884494881754:0] peer: [2:7574435884799021980:0] socket: 617 qp: -1
2025-11-19T13:59:43.049621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435888789849057:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:43.049644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:43.049748Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435884494881663:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(20840027ns) {HandleHandshakeStatus:329(20839495ns) {DropIncomingHandshake:441(1600ns) DropOutgoingHandshake:456(2230ns) SwitchToState:178(242ns) ProcessPendingSessionEvents:495(34592ns) {ForwardSessionEventToSession:612(23543ns) ForwardSessionEventToSession:612(724ns) ForwardSessionEventToSession:612(934ns) ForwardSessionEventToSession:612(977ns) ForwardSessionEventToSession:612(518ns) ForwardSessionEventToSession:612(1430ns) ForwardSessionEventToSession:612(900ns) ForwardSessionEventToSession:612(721ns) ForwardSessionEventToSession:612(763ns) ForwardSessionEventToSession:612(798ns)}}}
2025-11-19T13:59:43.052321Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435884494881664:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3157) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435888789849051:2120] held: no
2025-11-19T13:59:43.052413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435884494881663:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.054470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435890901226038:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24651
2025-11-19T13:59:43.052939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435888789849059:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5984
2025-11-19T13:59:43.054719Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435888789849051:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:43.055260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435884494881664:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.055281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:43.055289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435884494881664:2050] [node 3] ICP22 created new session: [4:7574435888789849060:2050]
2025-11-19T13:59:43.055295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435888789849060:2050] [node 3] ICS09 handshake done sender: [4:7574435888789849051:2120] self: [4:7574435888789849053:0] peer: [3:7574435885681079692:0] socket: 628 qp: -1
2025-11-19T13:59:43.055302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435888789849060:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:43.055314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:43.056531Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435883353989883:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:43.056777Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435890901226030:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:43.057497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435883353989881:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.057522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:43.057529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435883353989881:2113] [node 5] ICP22 created new session: [1:7574435887648957204:2113]
2025-11-19T13:59:43.057536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435887648957204:2113] [node 5] ICS09 handshake done sender: [1:7574435883353989883:2114] self: [1:7574435883353989882:0] peer: [5:7574435890901226033:0] socket: 608 qp: -1
2025-11-19T13:59:43.057543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435887648957204:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:43.057551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:43.058181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435886606258642:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.058200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:43.058207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435886606258642:2048] [node 1] ICP22 created new session: [5:7574435890901226039:2048]
2025-11-19T13:59:43.058213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435890901226039:2048] [node 1] ICS09 handshake done sender: [5:7574435890901226030:2119] self: [5:7574435890901226033:0] peer: [1:7574435883353989882:0] socket: 619 qp: -1
2025-11-19T13:59:43.058219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435890901226039:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:43.058231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:43.078842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:43.078933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435889093989297:2048] [node 1] ICS01 socket: 583 reason# EndOfStream
2025-11-19T13:59:43.078942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435884799021904:2048] [node 1] ICP30 unregister session Session# [2:7574435889093989297:2048] VirtualId# [2:7574435884799021987:0]
2025-11-19T13:59:43.078950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435889093989297:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.078998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:43.079017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:43.085157Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:43.085248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435889976047007:2048] [node 1] ICS01 socket: 567 reason# EndOfStream
2025-11-19T13:59:43.085258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435885681079615:2048] [node 1] ICP30 unregister session Session# [3:7574435889976047007:2048] VirtualId# [3:7574435885681079698:0]
2025-11-19T13:59:43.085266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435889976047007:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.085316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:43.085338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:43.092225Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:43.092325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435884494881745:2048] [node 1] ICS07 socket disconnect 600 reason# EndOfStream
2025-11-19T13:59:43.092334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435884494881745:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.092621Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:43.092377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:43.092403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435884494881745:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:43.092683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435890136221355:2048] [node 1] ICS01 socket: 599 reason# EndOfStream
2025-11-19T13:59:43.092693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435885841253970:2048] [node 1] ICP30 unregister session Session# [6:7574435890136221355:2048] VirtualId# [6:7574435885841254053:0]
2025-11-19T13:59:43.092699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435890136221355:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.092727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:43.092749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:43.093269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435884494881662:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2644) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:43.093283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435884494881745:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:43.093287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435884494881662:2048] [node 1] ICP30 unregister session Session# [4:7574435884494881745:2048] VirtualId# [4:7574435884494881742:0]
2025-11-19T13:59:43.093302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:43.093307Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435884494881662:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2644) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:43.093309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2644) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:43.095502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:43.095602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435890901226039:2048] [node 1] ICS01 socket: 619 reason# EndOfStream
2025-11-19T13:59:43.095612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435886606258642:2048] [node 1] ICP30 unregister session Session# [5:7574435890901226039:2048] VirtualId# [5:7574435890901226033:0]
2025-11-19T13:59:43.095618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435890901226039:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.095678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:43.095710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:43.098494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:43.098580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435889976047009:2049] [node 2] ICS01 socket: 601 reason# EndOfStream
2025-11-19T13:59:43.098589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435885681079616:2049] [node 2] ICP30 unregister session Session# [3:7574435889976047009:2049] VirtualId# [3:7574435885681079699:0]
2025-11-19T13:59:43.098595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435889976047009:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.098630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:43.098649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:43.103903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435884494881662:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:43.119099Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:43.119199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435890901226028:2049] [node 2] ICS01 socket: 574 reason# EndOfStream
2025-11-19T13:59:43.119210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435886606258643:2049] [node 2] ICP30 unregister session Session# [5:7574435890901226028:2049] VirtualId# [5:7574435886606258718:0]
2025-11-19T13:59:43.119218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435890901226028:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.119275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:43.119305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:43.120090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:43.120167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435890136221364:2049] [node 2] ICS01 socket: 624 reason# EndOfStream
2025-11-19T13:59:43.120173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435885841253971:2049] [node 2] ICP30 unregister session Session# [6:7574435890136221364:2049] VirtualId# [6:7574435890136221358:0]
2025-11-19T13:59:43.120179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435890136221364:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.120226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:43.120252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:43.120289Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:43.120348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435888789849057:2049] [node 2] ICS01 socket: 617 reason# EndOfStream
2025-11-19T13:59:43.120353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435884494881663:2049] [node 2] ICP30 unregister session Session# [4:7574435888789849057:2049] VirtualId# [4:7574435884494881754:0]
2025-11-19T13:59:43.120358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435888789849057:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.120388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:43.120410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:43.143560Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:43.143653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435890901226034:2050] [node 3] ICS01 socket: 580 reason# EndOfStream
2025-11-19T13:59:43.143662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435886606258644:2050] [node 3] ICP30 unregister session Session# [5:7574435890901226034:2050] VirtualId# [5:7574435886606258725:0]
2025-11-19T13:59:43.143669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435890901226034:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.143732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:43.143768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:43.143774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T13:59:43.143854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435888789849060:2050] [node 3] ICS01 socket: 628 reason# EndOfStream
2025-11-19T13:59:43.143863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435884494881664:2050] [node 3] ICP30 unregister session Session# [4:7574435888789849060:2050] VirtualId# [4:7574435888789849053:0]
2025-11-19T13:59:43.143869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435888789849060:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.143922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:43.143945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:43.143980Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:43.144040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435890136221359:2050] [node 3] ICS01 socket: 598 reason# EndOfStream
2025-11-19T13:59:43.144045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435885841253972:2050] [node 3] ICP30 unregister session Session# [6:7574435890136221359:2050] VirtualId# [6:7574435885841254052:0]
2025-11-19T13:59:43.144049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435890136221359:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.144074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:43.144097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:43.163126Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:43.163215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435890136221367:2065] [node 4] ICS01 socket: 621 reason# EndOfStream
2025-11-19T13:59:43.163224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435885841253991:2065] [node 4] ICP30 unregister session Session# [6:7574435890136221367:2065] VirtualId# [6:7574435890136221357:0]
2025-11-19T13:59:43.163231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435890136221367:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.163294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:43.163318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:43.223905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435887567889220:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:43.223923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435887567889221:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:43.259862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435891756823426:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:43.259876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435891756823427:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:43.292018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435890720296057:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:43.292039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435890720296058:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:43.324129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435890413019506:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:43.324161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435890413019507:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:43.324166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435890413019508:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:43.339363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435890235989951:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:43.339376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435890235989952:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:43.339380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435890235989953:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:43.377865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435890781390502:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:43.377890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435890781390503:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:43.377895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435890781390504:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:43.435588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435891756823487:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:43.435843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435890413019534:2073] [node 5] ICP01 ready to work
2025-11-19T13:59:43.436106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435890235989966:2059] [node 6] ICP01 ready to work
2025-11-19T13:59:43.436202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435890235989967:2061] [node 4] ICP01 ready to work
2025-11-19T13:59:43.437228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435890413019543:2080] [node 6] ICP01 ready to work
2025-11-19T13:59:43.438016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435887567889290:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.439722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435887567889294:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:43.442383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435890781390527:2064] [node 5] ICP01 ready to work
2025-11-19T13:59:43.444241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435890781390534:2071] [node 4] ICP01 ready to work
2025-11-19T13:59:43.447106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435887567889285:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:43.447392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435891756823490:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50487
2025-11-19T13:59:43.447573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435891756823493:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:43.455929Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435890235989966:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19692587ns) {Configure:104(19690970ns) {ProcessConfigured:135(18281527ns) {StartInitialHandshake:157(18280170ns) {DropHandshakes:469(525ns) {DropIncomingHandshake:441(88ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(342ns) PrepareNewSessionHandshake:476(1367ns) {SwitchToState:178(843ns)}}}}}
2025-11-19T13:59:43.454801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435890720296125:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.456385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435890781390503:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(18923349ns) {Configure:104(18922852ns) {ProcessConfigured:135(18669559ns) {StartInitialHandshake:157(18669235ns) {DropHandshakes:469(191ns) {DropIncomingHandshake:441(21ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(1154ns) {SwitchToState:178(626ns)}}}}}
2025-11-19T13:59:43.458028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890781390568:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35654
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.458182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435890413019507:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20413345ns) {Configure:104(20412875ns) {ProcessConfigured:135(20191237ns) {StartInitialHandshake:157(20190805ns) {DropHandshakes:469(212ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:476(970ns) {SwitchToState:178(233ns)}}}}}
2025-11-19T13:59:43.456536Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435890235989967:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(20292434ns) {Configure:104(20291926ns) {ProcessConfigured:135(18694518ns) {StartInitialHandshake:157(18693871ns) {DropHandshakes:469(337ns) {DropIncomingHandshake:441(97ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(117ns) PrepareNewSessionHandshake:476(468ns) {SwitchToState:178(396ns)}}}}}
2025-11-19T13:59:43.458363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435890235990010:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16441
2025-11-19T13:59:43.458408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435890235990011:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16441
2025-11-19T13:59:43.458418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435890235990012:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16441
2025-11-19T13:59:43.458685Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435890235990010:2096] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.458722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435890235989967:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.458831Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435890781390568:2103] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.458851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435890781390527:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.458938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435890413019538:2077] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.458944Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435890413019538:2077] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.459609Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435890413019538:2077] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:43.459722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435890413019534:2073] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.459739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:43.459759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435890413019534:2073] [node 5] ICP22 created new session: [4:7574435890413019568:2073]
2025-11-19T13:59:43.459766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435890413019568:2073] [node 5] ICS09 handshake done sender: [4:7574435890413019538:2077] self: [4:7574435890413019537:0] peer: [5:7574435890235990013:0] socket: 570 qp: -1
2025-11-19T13:59:43.459775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435890413019568:2073] [node 5] ICS10 traffic start
2025-11-19T13:59:43.459797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.461521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435887567889298:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10254
2025-11-19T13:59:43.461492Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435890235990005:2092] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.461503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435890235990005:2092] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.463084Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435890235990005:2092] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:43.463227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435890235989966:2059] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.463240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:43.463246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435890235989966:2059] [node 6] ICP22 created new session: [5:7574435890235990018:2059]
2025-11-19T13:59:43.463253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435890235990018:2059] [node 6] ICS09 handshake done sender: [5:7574435890235990005:2092] self: [5:7574435890235989968:0] peer: [6:7574435890781390569:0] socket: 589 qp: -1
2025-11-19T13:59:43.463258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435890235990018:2059] [node 6] ICS10 traffic start
2025-11-19T13:59:43.463274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.467261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435887567889299:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10254
2025-11-19T13:59:43.471881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435891756823487:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(36198066ns) {Configure:104(36195990ns) {ProcessConfigured:135(35230650ns) {StartInitialHandshake:157(35229920ns) {DropHandshakes:469(718ns) {DropIncomingHandshake:441(209ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(131ns) PrepareNewSessionHandshake:476(1530ns) {SwitchToState:178(723ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.472631Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435890781390504:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(34433774ns) {Configure:104(34433125ns) {ProcessConfigured:135(34212024ns) {StartInitialHandshake:157(34211484ns) {DropHandshakes:469(190ns) {DropIncomingHandshake:441(45ns) DropOutgoingHandshake:456(28ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:476(578ns) {SwitchToState:178(166ns)}}}}}
2025-11-19T13:59:43.472903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435891756823501:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50487
2025-11-19T13:59:43.472955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435891756823502:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50487
2025-11-19T13:59:43.472984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435891756823503:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50487
2025-11-19T13:59:43.470577Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435890235989953:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(30955475ns) {Configure:104(30954581ns) {ProcessConfigured:135(30642309ns) {StartInitialHandshake:157(30641899ns) {DropHandshakes:469(553ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:476(669ns) {SwitchToState:178(354ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.475924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435887567889285:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(28746796ns) {Configure:104(28745475ns) {ProcessConfigured:135(28421855ns) {StartInitialHandshake:157(28421476ns) {DropHandshakes:469(498ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:476(986ns) {SwitchToState:178(464ns)}}}}}
2025-11-19T13:59:43.475964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435887567889301:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10254
2025-11-19T13:59:43.476057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435887567889298:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.476449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435890413019586:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10840
2025-11-19T13:59:43.476470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435890413019587:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10840
2025-11-19T13:59:43.479856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435891756823493:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(32229251ns) {Configure:104(32228454ns) {ProcessConfigured:135(31738850ns) {StartInitialHandshake:157(31738297ns) {DropHandshakes:469(628ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(95ns)} GenerateSessionVirtualId:988(179ns) PrepareNewSessionHandshake:476(966ns) {SwitchToState:178(501ns)}}}}}
2025-11-19T13:59:43.479865Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435891756823503:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.479925Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435891756823501:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.480454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435890413019588:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10840
2025-11-19T13:59:43.480057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435891756823502:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.480516Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435887567889301:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.487220Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435891756823490:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.487999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435887567889300:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10254
2025-11-19T13:59:43.488336Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435890413019587:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.488495Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435887567889299:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.488521Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435887567889300:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.488748Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435890235990011:2097] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.488774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435890235989951:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.496061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435891756823426:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(60277981ns) {Configure:104(60276243ns) {ProcessConfigured:135(59122986ns) {StartInitialHandshake:157(59122294ns) {DropHandshakes:469(273ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(14ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:476(1057ns) {SwitchToState:178(123ns)}}}}}
2025-11-19T13:59:43.496120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435891756823426:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.496292Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435890413019543:2080] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(59006405ns) {Configure:104(59006022ns) {ProcessConfigured:135(58760507ns) {StartInitialHandshake:157(58760301ns) {DropHandshakes:469(308ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(110ns) PrepareNewSessionHandshake:476(1615ns) {SwitchToState:178(130ns)}}}}}
2025-11-19T13:59:43.496578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890781390582:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35654
2025-11-19T13:59:43.496729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435887567889284:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.496741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435887567889284:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.498029Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435887567889284:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.498534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435890413019591:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10840
2025-11-19T13:59:43.498123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435887567889220:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.498145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:43.498152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435887567889220:2048] [node 2] ICP22 created new session: [1:7574435887567889308:2048]
2025-11-19T13:59:43.498158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435887567889308:2048] [node 2] ICS09 handshake done sender: [1:7574435887567889284:2110] self: [1:7574435887567889283:0] peer: [2:7574435891756823510:0] socket: 572 qp: -1
2025-11-19T13:59:43.498165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435887567889308:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:43.498174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.500334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435891756823511:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50487
2025-11-19T13:59:43.501126Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435891756823490:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:43.501888Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435890413019591:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.501927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435890413019506:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.501283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435891756823426:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.501306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:43.501313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435891756823426:2048] [node 1] ICP22 created new session: [2:7574435891756823512:2048]
2025-11-19T13:59:43.501320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435891756823512:2048] [node 1] ICS09 handshake done sender: [2:7574435891756823490:2108] self: [2:7574435891756823510:0] peer: [1:7574435887567889283:0] socket: 578 qp: -1
2025-11-19T13:59:43.501327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435891756823512:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:43.501347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:43.502187Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435887567889296:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.502208Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435887567889296:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.502811Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435887567889296:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:43.502894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435887567889294:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.502910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:43.502916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435887567889294:2108] [node 4] ICP22 created new session: [1:7574435887567889312:2108]
2025-11-19T13:59:43.502922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435887567889312:2108] [node 4] ICS09 handshake done sender: [1:7574435887567889296:2113] self: [1:7574435887567889295:0] peer: [4:7574435890413019592:0] socket: 611 qp: -1
2025-11-19T13:59:43.502927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435887567889312:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:43.502941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.503263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435890413019593:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10840
2025-11-19T13:59:43.504285Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435890781390568:2103] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:43.504983Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435890413019591:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:43.505065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435890413019506:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.505078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:43.505085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435890413019506:2048] [node 1] ICP22 created new session: [4:7574435890413019594:2048]
2025-11-19T13:59:43.505090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435890413019594:2048] [node 1] ICS09 handshake done sender: [4:7574435890413019591:2118] self: [4:7574435890413019592:0] peer: [1:7574435887567889295:0] socket: 612 qp: -1
2025-11-19T13:59:43.505095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435890413019594:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:43.505110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:43.505479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435890720296128:2106] [node 6] ICP01 ready to work
2025-11-19T13:59:43.505868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435890720296129:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:43.504423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435890781390527:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.504437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:43.504443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435890781390527:2064] [node 5] ICP22 created new session: [6:7574435890781390583:2064]
2025-11-19T13:59:43.504449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435890781390583:2064] [node 5] ICS09 handshake done sender: [6:7574435890781390568:2103] self: [6:7574435890781390569:0] peer: [5:7574435890235989968:0] socket: 590 qp: -1
2025-11-19T13:59:43.504455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435890781390583:2064] [node 5] ICS10 traffic start
2025-11-19T13:59:43.504461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:43.505057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435891756823495:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:43.507808Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435887567889292:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.507828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435887567889292:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.515032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435890720296134:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49008
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.515509Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435890235989966:2059] [node 6] ICP35 event processing took too much time PendingConnection:278(52246854ns) {HandleHandshakeStatus:329(52245413ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(189ns) SwitchToState:178(305ns) ProcessPendingSessionEvents:495(29944764ns) {ForwardSessionEventToSession:612(29933770ns) ForwardSessionEventToSession:612(2768ns) ForwardSessionEventToSession:612(1342ns) ForwardSessionEventToSession:612(572ns) ForwardSessionEventToSession:612(571ns) ForwardSessionEventToSession:612(887ns) ForwardSessionEventToSession:612(634ns) ForwardSessionEventToSession:612(662ns)}}}
2025-11-19T13:59:43.515108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435890720296135:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49008
2025-11-19T13:59:43.515143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435890720296136:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49008
2025-11-19T13:59:43.520772Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435890720296134:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.521586Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435890781390527:2064] [node 5] ICP35 event processing took too much time PendingConnection:278(17130804ns) {HandleHandshakeStatus:329(17130266ns) {DropIncomingHandshake:441(160ns) DropOutgoingHandshake:456(812ns) SwitchToState:178(69ns) ProcessPendingSessionEvents:495(17046541ns) {ForwardSessionEventToSession:612(17019424ns) ForwardSessionEventToSession:612(2324ns) ForwardSessionEventToSession:612(1088ns) ForwardSessionEventToSession:612(15001ns) ForwardSessionEventToSession:612(514ns) ForwardSessionEventToSession:612(668ns) ForwardSessionEventToSession:612(673ns) ForwardSessionEventToSession:612(785ns) ForwardSessionEventToSession:612(616ns) ForwardSessionEventToSession:612(1172ns)}}}
2025-11-19T13:59:43.521816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435890413019534:2073] [node 5] ICP18 handshake (actor [4:7574435890413019587:2114]) is too old
2025-11-19T13:59:43.521843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:293: Proxy [4:7574435890413019534:2073] [node 5] ICP35 event processing took too much time StateWork:293(33442903ns) {IncomingHandshake:261(33441818ns)}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.521938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435890720296057:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.522065Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435887567889287:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.522077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435887567889287:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.522392Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435890720296135:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.528142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435891756823495:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23003667ns) {Configure:104(23002510ns) {ProcessConfigured:135(22632109ns) {StartInitialHandshake:157(22630847ns) {DropHandshakes:469(477ns) {DropIncomingHandshake:441(90ns) DropOutgoingHandshake:456(38ns)} GenerateSessionVirtualId:988(178ns) PrepareNewSessionHandshake:476(1279ns) {SwitchToState:178(448ns)}}}}}
2025-11-19T13:59:43.528478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435890720296128:2106] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(22929833ns) {Configure:104(22929476ns) {ProcessConfigured:135(22666881ns) {StartInitialHandshake:157(22666443ns) {DropHandshakes:469(147ns) {DropIncomingHandshake:441(21ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(147ns) PrepareNewSessionHandshake:476(767ns) {SwitchToState:178(320ns)}}}}}
2025-11-19T13:59:43.528698Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435890720296129:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(22789337ns) {Configure:104(22789150ns) {ProcessConfigured:135(22581489ns) {StartInitialHandshake:157(22581307ns) {DropHandshakes:469(140ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(15ns)} GenerateSessionVirtualId:988(42ns) PrepareNewSessionHandshake:476(613ns) {SwitchToState:178(436ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.531834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890781390585:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35654
2025-11-19T13:59:43.531902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890781390586:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35654
2025-11-19T13:59:43.532962Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435887567889292:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.533461Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435890781390585:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.533516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435890781390502:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.533142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435890235990038:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16441
2025-11-19T13:59:43.533189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435890235990039:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16441
2025-11-19T13:59:43.533209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435890235990040:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16441
2025-11-19T13:59:43.533283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435890235990041:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16441
2025-11-19T13:59:43.533655Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435890235990010:2096] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:43.535269Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435890413019506:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(30172460ns) {HandleHandshakeStatus:329(30172045ns) {DropIncomingHandshake:441(160ns) DropOutgoingHandshake:456(1238ns) SwitchToState:178(68ns) ProcessPendingSessionEvents:495(13615686ns) {ForwardSessionEventToSession:612(13603067ns) ForwardSessionEventToSession:612(2039ns) ForwardSessionEventToSession:612(968ns) ForwardSessionEventToSession:612(627ns) ForwardSessionEventToSession:612(643ns) ForwardSessionEventToSession:612(1015ns) ForwardSessionEventToSession:612(609ns) ForwardSessionEventToSession:612(613ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(992ns)}}}
2025-11-19T13:59:43.535740Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435887567889287:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:43.535824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435887567889290:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.535842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:43.535849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435887567889290:2107] [node 5] ICP22 created new session: [1:7574435887567889317:2107]
2025-11-19T13:59:43.535855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435887567889317:2107] [node 5] ICS09 handshake done sender: [1:7574435887567889292:2112] self: [1:7574435887567889291:0] peer: [5:7574435890235990037:0] socket: 580 qp: -1
2025-11-19T13:59:43.535861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435887567889317:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:43.535863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435887567889221:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.535869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:43.535878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:43.535887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435887567889221:2049] [node 3] ICP22 created new session: [1:7574435887567889319:2049]
2025-11-19T13:59:43.535893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435887567889319:2049] [node 3] ICS09 handshake done sender: [1:7574435887567889287:2111] self: [1:7574435887567889286:0] peer: [3:7574435890720296137:0] socket: 573 qp: -1
2025-11-19T13:59:43.535899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435887567889319:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:43.535905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:43.536039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435890235989967:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.536054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:43.536059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435890235989967:2061] [node 4] ICP22 created new session: [5:7574435890235990042:2061]
2025-11-19T13:59:43.536064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435890235990042:2061] [node 4] ICS09 handshake done sender: [5:7574435890235990010:2096] self: [5:7574435890235990013:0] peer: [4:7574435890413019537:0] socket: 571 qp: -1
2025-11-19T13:59:43.536069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435890235990042:2061] [node 4] ICS10 traffic start
2025-11-19T13:59:43.536074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:43.536124Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435890413019586:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.539289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890781390589:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35654
2025-11-19T13:59:43.547049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435887567889302:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.547072Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435887567889302:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.543940Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435890235990011:2097] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:43.549440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435890720296057:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3624) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435890720296134:2113] held: no
2025-11-19T13:59:43.549789Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435890235990039:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.549807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435890235989951:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.549825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:43.549843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435890235989951:2048] [node 1] ICP22 created new session: [5:7574435890235990044:2048]
2025-11-19T13:59:43.549850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435890235990044:2048] [node 1] ICS09 handshake done sender: [5:7574435890235990011:2097] self: [5:7574435890235990037:0] peer: [1:7574435887567889291:0] socket: 591 qp: -1
2025-11-19T13:59:43.549857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435890235990044:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:43.549867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:43.550061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435890235989966:2059] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:43.551022Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435890781390589:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.551079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435890781390534:2071] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.553140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435890720296142:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49008
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.553844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890781390591:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35654
2025-11-19T13:59:43.554075Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435890413019590:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.554088Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435890413019590:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.554062Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435890720296130:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.554073Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435890720296130:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.555113Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435890720296130:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.555399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435890720296125:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.555418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:43.555424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435890720296125:2110] [node 5] ICP22 created new session: [3:7574435890720296144:2110]
2025-11-19T13:59:43.555431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435890720296144:2110] [node 5] ICS09 handshake done sender: [3:7574435890720296130:2112] self: [3:7574435890720296126:0] peer: [5:7574435890235990046:0] socket: 601 qp: -1
2025-11-19T13:59:43.555438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435890720296144:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:43.555448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:43.555562Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435890720296134:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:43.553695Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435890235989951:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:43.553737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435890235989953:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.555476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435890235990048:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16441
2025-11-19T13:59:43.555950Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435890781390591:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.555993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435890781390504:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.556240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435890720296057:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.556254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:43.556260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435890720296057:2048] [node 1] ICP22 created new session: [3:7574435890720296146:2048]
2025-11-19T13:59:43.556267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435890720296146:2048] [node 1] ICS09 handshake done sender: [3:7574435890720296134:2113] self: [3:7574435890720296137:0] peer: [1:7574435887567889286:0] socket: 574 qp: -1
2025-11-19T13:59:43.556273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435890720296146:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:43.556278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:43.557196Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435890235990040:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.557235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435890235989952:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.557466Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435890235990039:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:43.557550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435890235989953:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.557567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:43.557572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435890235989953:2050] [node 3] ICP22 created new session: [5:7574435890235990050:2050]
2025-11-19T13:59:43.557579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435890235990050:2050] [node 3] ICS09 handshake done sender: [5:7574435890235990039:2119] self: [5:7574435890235990046:0] peer: [3:7574435890720296126:0] socket: 619 qp: -1
2025-11-19T13:59:43.557584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435890235990050:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:43.557599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:43.562090Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435890781390586:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.562152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435890781390503:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.562369Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435890413019588:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.562409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435890413019507:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.562672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890781390594:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35654
2025-11-19T13:59:43.562637Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435887567889290:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(26775556ns) {HandleHandshakeStatus:329(26774698ns) {DropIncomingHandshake:441(3539ns) DropOutgoingHandshake:456(366ns) SwitchToState:178(205ns) ProcessPendingSessionEvents:495(15450254ns) {ForwardSessionEventToSession:612(15437983ns) ForwardSessionEventToSession:612(1580ns) ForwardSessionEventToSession:612(998ns) ForwardSessionEventToSession:612(566ns) ForwardSessionEventToSession:612(688ns) ForwardSessionEventToSession:612(701ns) ForwardSessionEventToSession:612(612ns) ForwardSessionEventToSession:612(785ns) ForwardSessionEventToSession:612(555ns) ForwardSessionEventToSession:612(950ns)}}}
2025-11-19T13:59:43.562807Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435891756823515:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.562821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435891756823515:2119] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.563424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435891756823517:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50487
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.564597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890781390595:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35654
2025-11-19T13:59:43.564819Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435891756823517:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.564944Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435887567889302:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:43.565155Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435890781390585:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:43.565285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435887567889285:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.565309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:43.565315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435887567889285:2109] [node 6] ICP22 created new session: [1:7574435887567889322:2109]
2025-11-19T13:59:43.565322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435887567889322:2109] [node 6] ICS09 handshake done sender: [1:7574435887567889302:2118] self: [1:7574435887567889297:0] peer: [6:7574435890781390587:0] socket: 616 qp: -1
2025-11-19T13:59:43.565328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435887567889322:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:43.565336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:43.565693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435890781390502:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.565708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:43.565713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435890781390502:2048] [node 1] ICP22 created new session: [6:7574435890781390596:2048]
2025-11-19T13:59:43.565717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435890781390596:2048] [node 1] ICS09 handshake done sender: [6:7574435890781390585:2117] self: [6:7574435890781390587:0] peer: [1:7574435887567889297:0] socket: 617 qp: -1
2025-11-19T13:59:43.565722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435890781390596:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:43.565731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.566767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435890720296150:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49008
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.568605Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435890413019590:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:43.568630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435890413019600:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10840
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.568670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435890413019601:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10840
2025-11-19T13:59:43.568786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435890413019543:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.568812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:43.568823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435890413019543:2080] [node 6] ICP22 created new session: [4:7574435890413019602:2080]
2025-11-19T13:59:43.568829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435890413019602:2080] [node 6] ICS09 handshake done sender: [4:7574435890413019590:2117] self: [4:7574435890413019550:0] peer: [6:7574435890781390590:0] socket: 626 qp: -1
2025-11-19T13:59:43.568835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435890413019602:2080] [node 6] ICS10 traffic start
2025-11-19T13:59:43.568840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:43.569017Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435890781390589:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:43.567272Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435890720296138:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.567284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435890720296138:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.567317Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435891756823496:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.567327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435891756823496:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.567550Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435891756823504:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.567555Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435891756823504:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.568100Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435891756823504:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:43.568336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435891756823493:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.568352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:43.568358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435891756823493:2110] [node 4] ICP22 created new session: [2:7574435891756823520:2110]
2025-11-19T13:59:43.568363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435891756823520:2110] [node 4] ICS09 handshake done sender: [2:7574435891756823504:2116] self: [2:7574435891756823494:0] peer: [4:7574435890413019598:0] socket: 608 qp: -1
2025-11-19T13:59:43.569135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435890781390534:2071] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.569149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:43.569154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435890781390534:2071] [node 4] ICP22 created new session: [6:7574435890781390599:2071]
2025-11-19T13:59:43.569159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435890781390599:2071] [node 4] ICS09 handshake done sender: [6:7574435890781390589:2119] self: [6:7574435890781390590:0] peer: [4:7574435890413019550:0] socket: 628 qp: -1
2025-11-19T13:59:43.569163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435890781390599:2071] [node 4] ICS10 traffic start
2025-11-19T13:59:43.569168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:43.568369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435891756823520:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:43.568375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:43.569904Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435891756823515:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:43.570166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435891756823495:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.570185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:43.570190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435891756823495:2109] [node 6] ICP22 created new session: [2:7574435891756823523:2109]
2025-11-19T13:59:43.570195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435891756823523:2109] [node 6] ICS09 handshake done sender: [2:7574435891756823515:2119] self: [2:7574435891756823514:0] peer: [6:7574435890781390593:0] socket: 621 qp: -1
2025-11-19T13:59:43.570200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435891756823523:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:43.570205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:43.570292Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435890720296136:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.570321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435890720296058:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.571102Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435890413019600:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.571847Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435890720296138:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:43.572473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435890720296128:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.572489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:43.572494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435890720296128:2106] [node 6] ICP22 created new session: [3:7574435890720296153:2106]
2025-11-19T13:59:43.572500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435890720296153:2106] [node 6] ICS09 handshake done sender: [3:7574435890720296138:2116] self: [3:7574435890720296132:0] peer: [6:7574435890781390592:0] socket: 576 qp: -1
2025-11-19T13:59:43.572504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435890720296153:2106] [node 6] ICS10 traffic start
2025-11-19T13:59:43.572511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.573052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435890235990052:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16441
2025-11-19T13:59:43.573352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435890720296149:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49008
2025-11-19T13:59:43.573806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435890720296125:2110] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:43.573987Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435891756823496:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:43.574094Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435890235990040:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:43.574143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435890235989952:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.574156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:43.574161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435890235989952:2049] [node 2] ICP22 created new session: [5:7574435890235990053:2049]
2025-11-19T13:59:43.574166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435890235990053:2049] [node 2] ICS09 handshake done sender: [5:7574435890235990040:2120] self: [5:7574435890235990049:0] peer: [2:7574435891756823488:0] socket: 620 qp: -1
2025-11-19T13:59:43.574172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435890235990053:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:43.574179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:43.576262Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435891756823498:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.576281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435891756823498:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.576997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435891756823487:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.577010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:43.577016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435891756823487:2107] [node 5] ICP22 created new session: [2:7574435891756823526:2107]
2025-11-19T13:59:43.577021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435891756823526:2107] [node 5] ICS09 handshake done sender: [2:7574435891756823496:2111] self: [2:7574435891756823488:0] peer: [5:7574435890235990049:0] socket: 603 qp: -1
2025-11-19T13:59:43.577027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435891756823526:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:43.577034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:43.577683Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435890413019588:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:43.577849Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435890413019507:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14277) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435890413019588:2115] held: no
2025-11-19T13:59:43.577902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435890413019508:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:43.578491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435890413019507:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.578505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:43.578510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435890413019507:2049] [node 2] ICP22 created new session: [4:7574435890413019605:2049]
2025-11-19T13:59:43.578515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435890413019605:2049] [node 2] ICS09 handshake done sender: [4:7574435890413019588:2115] self: [4:7574435890413019598:0] peer: [2:7574435891756823494:0] socket: 609 qp: -1
2025-11-19T13:59:43.578521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435890413019605:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:43.578528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:43.584093Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435890720296139:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.584108Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435890720296139:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:43.585545Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435890720296139:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.585736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435890720296129:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.585756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:43.585765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435890720296129:2107] [node 4] ICP22 created new session: [3:7574435890720296157:2107]
2025-11-19T13:59:43.585771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435890720296157:2107] [node 4] ICS09 handshake done sender: [3:7574435890720296139:2117] self: [3:7574435890720296133:0] peer: [4:7574435890413019604:0] socket: 602 qp: -1
2025-11-19T13:59:43.585775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435890720296157:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:43.585783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:43.585934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435890720296156:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49008
2025-11-19T13:59:43.586263Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435890720296136:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:43.586321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435890720296058:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.586332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:43.586337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435890720296058:2049] [node 2] ICP22 created new session: [3:7574435890720296159:2049]
2025-11-19T13:59:43.586342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435890720296159:2049] [node 2] ICS09 handshake done sender: [3:7574435890720296136:2115] self: [3:7574435890720296151:0] peer: [2:7574435891756823497:0] socket: 618 qp: -1
2025-11-19T13:59:43.586347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435890720296159:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:43.586352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:43.585616Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435891756823498:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:43.586071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435891756823427:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:43.586083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:43.586089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435891756823427:2049] [node 3] ICP22 created new session: [2:7574435891756823529:2049]
2025-11-19T13:59:43.586094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435891756823529:2049] [node 3] ICS09 handshake done sender: [2:7574435891756823498:2112] self: [2:7574435891756823497:0] peer: [3:7574435890720296151:0] socket: 604 qp: -1
2025-11-19T13:59:43.586098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435891756823529:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:43.586103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.588019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435890413019607:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10840
2025-11-19T13:59:43.588143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435890781390503:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14277) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435890781390586:2118] held: no
2025-11-19T13:59:43.592043Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435890413019600:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:43.592202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435890413019508:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.592229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:43.592237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435890413019508:2050] [node 3] ICP22 created new session: [4:7574435890413019608:2050]
2025-11-19T13:59:43.592244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435890413019608:2050] [node 3] ICS09 handshake done sender: [4:7574435890413019600:2120] self: [4:7574435890413019604:0] peer: [3:7574435890720296133:0] socket: 629 qp: -1
2025-11-19T13:59:43.592253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435890413019608:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:43.592263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:43.593084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435890235989952:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(18916757ns) {HandleHandshakeStatus:329(18916457ns) {DropIncomingHandshake:441(322ns) DropOutgoingHandshake:456(1346ns) SwitchToState:178(79ns) ProcessPendingSessionEvents:495(18854547ns) {ForwardSessionEventToSession:612(18844129ns) ForwardSessionEventToSession:612(1788ns) ForwardSessionEventToSession:612(559ns) ForwardSessionEventToSession:612(769ns) ForwardSessionEventToSession:612(556ns) ForwardSessionEventToSession:612(1072ns) ForwardSessionEventToSession:612(547ns) ForwardSessionEventToSession:612(537ns) ForwardSessionEventToSession:612(673ns) ForwardSessionEventToSession:612(570ns)}}}
2025-11-19T13:59:43.592801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435891756823495:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(22599359ns) {HandleHandshakeStatus:329(22598784ns) {DropIncomingHandshake:441(3101ns) DropOutgoingHandshake:456(89ns) SwitchToState:178(76ns) ProcessPendingSessionEvents:495(18889628ns) {ForwardSessionEventToSession:612(18875865ns) ForwardSessionEventToSession:612(1964ns) ForwardSessionEventToSession:612(1330ns) ForwardSessionEventToSession:612(914ns) ForwardSessionEventToSession:612(578ns) ForwardSessionEventToSession:612(1653ns) ForwardSessionEventToSession:612(632ns) ForwardSessionEventToSession:612(670ns) ForwardSessionEventToSession:612(895ns) ForwardSessionEventToSession:612(1063ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:43.599631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890781390601:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35654
2025-11-19T13:59:43.599783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435890781390602:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35654
2025-11-19T13:59:43.599986Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435890720296150:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:43.600031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435890720296128:2106] [node 6] ICP18 handshake (actor [3:7574435890720296150:2120]) is too old
2025-11-19T13:59:43.600260Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435890781390591:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:43.600356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435890781390504:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.600397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:43.600405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435890781390504:2050] [node 3] ICP22 created new session: [6:7574435890781390603:2050]
2025-11-19T13:59:43.600411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435890781390603:2050] [node 3] ICS09 handshake done sender: [6:7574435890781390591:2120] self: [6:7574435890781390592:0] peer: [3:7574435890720296132:0] socket: 587 qp: -1
2025-11-19T13:59:43.600419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435890781390603:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:43.600431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:43.600810Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435890781390586:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:43.607861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435890781390503:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:43.607885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:43.607892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435890781390503:2049] [node 2] ICP22 created new session: [6:7574435890781390605:2049]
2025-11-19T13:59:43.607898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435890781390605:2049] [node 2] ICS09 handshake done sender: [6:7574435890781390586:2118] self: [6:7574435890781390593:0] peer: [2:7574435891756823514:0] socket: 622 qp: -1
2025-11-19T13:59:43.607906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435890781390605:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:43.607922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:43.628278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:43.628397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435891756823512:2048] [node 1] ICS01 socket: 578 reason# EndOfStream
2025-11-19T13:59:43.628410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435891756823426:2048] [node 1] ICP30 unregister session Session# [2:7574435891756823512:2048] VirtualId# [2:7574435891756823510:0]
2025-11-19T13:59:43.628419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435891756823512:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.628455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:43.628477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:43.628927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:43.628992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435890720296146:2048] [node 1] ICS01 socket: 574 reason# EndOfStream
2025-11-19T13:59:43.628998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435890720296057:2048] [node 1] ICP30 unregister session Session# [3:7574435890720296146:2048] VirtualId# [3:7574435890720296137:0]
2025-11-19T13:59:43.629002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435890720296146:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.629027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:43.629044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:43.639860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:43.640022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435890235990044:2048] [node 1] ICS01 socket: 591 reason# EndOfStream
2025-11-19T13:59:43.640035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435890235989951:2048] [node 1] ICP30 unregister session Session# [5:7574435890235990044:2048] VirtualId# [5:7574435890235990037:0]
2025-11-19T13:59:43.640043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435890235990044:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.640096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:43.640122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:43.639912Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:43.640032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435890781390596:2048] [node 1] ICS01 socket: 617 reason# EndOfStream
2025-11-19T13:59:43.640042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435890781390502:2048] [node 1] ICP30 unregister session Session# [6:7574435890781390596:2048] VirtualId# [6:7574435890781390587:0]
2025-11-19T13:59:43.640049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435890781390596:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.640103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:43.640139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:43.643179Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:43.643282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435890413019594:2048] [node 1] ICS07 socket disconnect 612 reason# EndOfStream
2025-11-19T13:59:43.643289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435890413019594:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.643337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:43.643357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435890413019594:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:43.643916Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435890413019506:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3624) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:43.643935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435890413019594:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:43.643941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435890413019506:2048] [node 1] ICP30 unregister session Session# [4:7574435890413019594:2048] VirtualId# [4:7574435890413019592:0]
2025-11-19T13:59:43.643966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:43.643975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435890413019506:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3624) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:43.643978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3624) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:43.646512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:43.646587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435890720296159:2049] [node 2] ICS01 socket: 618 reason# EndOfStream
2025-11-19T13:59:43.646595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435890720296058:2049] [node 2] ICP30 unregister session Session# [3:7574435890720296159:2049] VirtualId# [3:7574435890720296151:0]
2025-11-19T13:59:43.646601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435890720296159:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.646634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:43.646653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:43.659815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435890413019506:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:43.680074Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:43.680187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435890781390605:2049] [node 2] ICS01 socket: 622 reason# EndOfStream
2025-11-19T13:59:43.680197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435890781390503:2049] [node 2] ICP30 unregister session Session# [6:7574435890781390605:2049] VirtualId# [6:7574435890781390593:0]
2025-11-19T13:59:43.680205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435890781390605:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.680252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:43.680278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:43.680313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:43.680373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435890413019605:2049] [node 2] ICS01 socket: 609 reason# EndOfStream
2025-11-19T13:59:43.680377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435890413019507:2049] [node 2] ICP30 unregister session Session# [4:7574435890413019605:2049] VirtualId# [4:7574435890413019598:0]
2025-11-19T13:59:43.680381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435890413019605:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.680407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:43.680426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:43.680535Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:43.680593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435890235990053:2049] [node 2] ICS07 socket disconnect 620 reason# EndOfStream
2025-11-19T13:59:43.680597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435890235990053:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.680622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:43.680640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435890235990053:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:43.681355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435890235989952:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14277) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:43.681367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435890235990053:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:43.681371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435890235989952:2049] [node 2] ICP30 unregister session Session# [5:7574435890235990053:2049] VirtualId# [5:7574435890235990049:0]
2025-11-19T13:59:43.681385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:43.681391Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435890235989952:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14277) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:43.681394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14277) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:43.695823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435890235989952:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:43.707972Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:43.708073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435890235990050:2050] [node 3] ICS07 socket disconnect 619 reason# EndOfStream
2025-11-19T13:59:43.708081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435890235990050:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.708138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:43.708161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435890235990050:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:43.708477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:43.708538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435890781390603:2050] [node 3] ICS01 socket: 587 reason# EndOfStream
2025-11-19T13:59:43.708543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435890781390504:2050] [node 3] ICP30 unregister session Session# [6:7574435890781390603:2050] VirtualId# [6:7574435890781390592:0]
2025-11-19T13:59:43.708548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435890781390603:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.708574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:43.708596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:43.708763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435890235989953:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28863) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:43.708774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435890235990050:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:43.708779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435890235989953:2050] [node 3] ICP30 unregister session Session# [5:7574435890235990050:2050] VirtualId# [5:7574435890235990046:0]
2025-11-19T13:59:43.708792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:43.708798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435890235989953:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28863) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:43.708800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28863) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:43.719796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435890235989953:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:59:43.767633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:43.767755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435890781390599:2071] [node 4] ICS01 socket: 628 reason# EndOfStream
2025-11-19T13:59:43.767769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435890781390534:2071] [node 4] ICP30 unregister session Session# [6:7574435890781390599:2071] VirtualId# [6:7574435890781390590:0]
2025-11-19T13:59:43.767780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435890781390599:2071] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.767850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:43.767876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:43.812687Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:59:43.812809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435890781390583:2064] [node 5] ICS07 socket disconnect 590 reason# EndOfStream
2025-11-19T13:59:43.812818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435890781390583:2064] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:43.812872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:59:43.812891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435890781390583:2064] [node 5] ICS15 start handshake
2025-11-19T13:59:43.813440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435890781390527:2064] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14656) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:43.813453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435890781390583:2064] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:43.813457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435890781390527:2064] [node 5] ICP30 unregister session Session# [6:7574435890781390583:2064] VirtualId# [6:7574435890781390569:0]
2025-11-19T13:59:43.813475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
2025-11-19T13:59:43.813482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435890781390527:2064] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14656) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:43.813486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14656) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:43.934234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435887852728363:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:43.934253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435887852728364:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:43.963992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435890366073053:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:43.964009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435890366073054:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:44.031917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435893665194186:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:44.031944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435893665194187:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:44.083874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435892159450855:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:44.083899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435892159450856:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:44.083904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435892159450857:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:44.127552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435896067278943:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:44.127570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435896067278944:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:44.127575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435896067278945:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:44.176031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435894110779347:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:44.176047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435894110779348:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:44.176053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435894110779349:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:44.223990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435893665194233:2093] [node 4] ICP01 ready to work
2025-11-19T13:59:44.247367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435896067278987:2078] [node 4] ICP01 ready to work
2025-11-19T13:59:44.247452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435896067278988:2081] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.252339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435894661040414:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:44.252436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435894661040415:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:44.252520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435894661040416:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:44.253824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435894661040412:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34126
2025-11-19T13:59:44.253852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435894661040413:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34126
2025-11-19T13:59:44.254298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435893665194252:2095] [node 5] ICP01 ready to work
2025-11-19T13:59:44.254375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435893665194253:2097] [node 6] ICP01 ready to work
2025-11-19T13:59:44.255597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893665194250:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-19T13:59:44.255610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893665194251:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-19T13:59:44.264532Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435896067278943:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17006331ns) {Configure:104(17004908ns) {ProcessConfigured:135(16490744ns) {StartInitialHandshake:157(16489702ns) {DropHandshakes:469(1497ns) {DropIncomingHandshake:441(199ns) DropOutgoingHandshake:456(171ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:476(2286ns) {SwitchToState:178(908ns)}}}}}
2025-11-19T13:59:44.292141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435892159450921:2081] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.292971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435892147695728:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:44.293061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435892147695729:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:44.293132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435892147695730:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:44.293437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435894110779413:2070] [node 4] ICP01 ready to work
2025-11-19T13:59:44.293521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435894110779414:2078] [node 5] ICP01 ready to work
2025-11-19T13:59:44.292291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435892159450922:2088] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.296425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435892159450923:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52275
2025-11-19T13:59:44.296444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435892159450924:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52275
2025-11-19T13:59:44.296454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435892159450925:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52275
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.296694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435894661040430:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34126
2025-11-19T13:59:44.302837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435894110779415:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45366
2025-11-19T13:59:44.302886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435894110779416:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45366
2025-11-19T13:59:44.302898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435894110779417:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45366
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.315800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435892159450936:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52275
2025-11-19T13:59:44.316253Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435892159450925:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.316430Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435894110779417:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.317843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435892147695731:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49175
2025-11-19T13:59:44.317867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435892147695732:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49175
2025-11-19T13:59:44.317903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435892147695733:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49175
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.318321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435896067279021:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25872
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.318795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435894661040431:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34126
2025-11-19T13:59:44.315711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893665194267:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-19T13:59:44.318928Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435892159450923:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.318959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435892159450856:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.319069Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894661040413:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.319097Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894661040418:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.319100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435894661040418:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.319798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435892159450939:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52275
2025-11-19T13:59:44.319910Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435894661040418:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:44.320147Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435892159450923:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:44.320357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435892159450856:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.320383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:44.320392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435892159450856:2049] [node 2] ICP22 created new session: [4:7574435892159450940:2049]
2025-11-19T13:59:44.320399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435892159450940:2049] [node 2] ICS09 handshake done sender: [4:7574435892159450923:2111] self: [4:7574435892159450938:0] peer: [2:7574435894661040417:0] socket: 594 qp: -1
2025-11-19T13:59:44.320407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435892159450940:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:44.320418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:44.320709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435894661040414:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.320726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:44.320731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435894661040414:2107] [node 4] ICP22 created new session: [2:7574435894661040434:2107]
2025-11-19T13:59:44.320735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435894661040434:2107] [node 4] ICS09 handshake done sender: [2:7574435894661040418:2112] self: [2:7574435894661040417:0] peer: [4:7574435892159450938:0] socket: 583 qp: -1
2025-11-19T13:59:44.320740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435894661040434:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:44.320745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:44.323431Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892147695733:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.323889Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893665194267:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.324024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893665194251:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.324253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893665194268:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-19T13:59:44.324151Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894661040412:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.324488Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435892159450924:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.327240Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894661040430:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.327267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435890366073053:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.329586Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893665194268:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.329618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435893665194187:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.330682Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435894110779416:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.330708Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435894110779415:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.330762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435894110779413:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.330808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435894110779414:2078] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.330819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435894110779348:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.330995Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435896067279013:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.331008Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435896067279013:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.331651Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435896067279013:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:44.331785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435896067278988:2081] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.331802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:44.331810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435896067278988:2081] [node 6] ICP22 created new session: [5:7574435896067279023:2081]
2025-11-19T13:59:44.331817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435896067279023:2081] [node 6] ICS09 handshake done sender: [5:7574435896067279013:2112] self: [5:7574435896067279012:0] peer: [6:7574435894110779429:0] socket: 592 qp: -1
2025-11-19T13:59:44.331824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435896067279023:2081] [node 6] ICS10 traffic start
2025-11-19T13:59:44.331834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.334116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893665194272:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.334114Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892147695732:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.334138Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892147695731:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.334162Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892147695720:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.334165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435892147695720:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.334598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435892147695741:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49175
2025-11-19T13:59:44.334519Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894661040424:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.334530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435894661040424:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.335101Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894661040431:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.335127Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894661040420:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.335130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435894661040420:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.335890Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893665194250:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.335916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435893665194186:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.336374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435894661040439:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34126
2025-11-19T13:59:44.336478Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435894661040424:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:44.336616Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435894661040420:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:44.336803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435890366073054:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.336826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:44.336833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435890366073054:2049] [node 3] ICP22 created new session: [2:7574435894661040443:2049]
2025-11-19T13:59:44.336840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435894661040443:2049] [node 3] ICS09 handshake done sender: [2:7574435894661040424:2115] self: [2:7574435894661040423:0] peer: [3:7574435893665194271:0] socket: 582 qp: -1
2025-11-19T13:59:44.336846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435894661040443:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:44.336854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:44.336995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435894661040415:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.337001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:44.337004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435894661040415:2108] [node 6] ICP22 created new session: [2:7574435894661040445:2108]
2025-11-19T13:59:44.337008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435894661040445:2108] [node 6] ICS09 handshake done sender: [2:7574435894661040420:2113] self: [2:7574435894661040419:0] peer: [6:7574435894110779430:0] socket: 584 qp: -1
2025-11-19T13:59:44.337011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435894661040445:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:44.337016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.339902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893665194274:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-19T13:59:44.340068Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435893665194268:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:44.340121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435893665194187:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.340137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:44.340142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435893665194187:2049] [node 2] ICP22 created new session: [3:7574435893665194275:2049]
2025-11-19T13:59:44.340147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435893665194275:2049] [node 2] ICS09 handshake done sender: [3:7574435893665194268:2118] self: [3:7574435893665194271:0] peer: [2:7574435894661040423:0] socket: 615 qp: -1
2025-11-19T13:59:44.340153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435893665194275:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:44.340159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:44.341366Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435892159450927:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.341378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435892159450927:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.341927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435892159450857:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.343315Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435894110779348:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20101) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435894110779415:2111] held: no
2025-11-19T13:59:44.343361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435894110779431:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45366
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.343507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435894110779432:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45366
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.343623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435894110779433:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45366
2025-11-19T13:59:44.347035Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435892147695720:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:44.347193Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892147695722:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.347201Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435892147695722:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.347913Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435892159450927:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:44.347987Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893665194255:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.348000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435893665194255:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.348116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435887852728363:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.348145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:44.348153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435887852728363:2048] [node 2] ICP22 created new session: [1:7574435892147695746:2048]
2025-11-19T13:59:44.348159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435892147695746:2048] [node 2] ICS09 handshake done sender: [1:7574435892147695720:2107] self: [1:7574435892147695719:0] peer: [2:7574435894661040437:0] socket: 576 qp: -1
2025-11-19T13:59:44.348165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435892147695746:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:44.348174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:44.348277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435896067279021:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.348311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435896067278943:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.349234Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435894110779416:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:44.349315Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435894110779415:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:44.349477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435894110779414:2078] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.349492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:44.349498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435894110779414:2078] [node 5] ICP22 created new session: [6:7574435894110779435:2078]
2025-11-19T13:59:44.349503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435894110779435:2078] [node 5] ICS09 handshake done sender: [6:7574435894110779416:2112] self: [6:7574435894110779429:0] peer: [5:7574435896067279012:0] socket: 601 qp: -1
2025-11-19T13:59:44.349509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435894110779435:2078] [node 5] ICS10 traffic start
2025-11-19T13:59:44.349515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:44.349663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435894110779348:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.349670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:44.349676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435894110779348:2049] [node 2] ICP22 created new session: [6:7574435894110779437:2049]
2025-11-19T13:59:44.349680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435894110779437:2049] [node 2] ICS09 handshake done sender: [6:7574435894110779415:2111] self: [6:7574435894110779430:0] peer: [2:7574435894661040419:0] socket: 596 qp: -1
2025-11-19T13:59:44.349684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435894110779437:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:44.349688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:44.349744Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435894110779417:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:44.349976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435894110779413:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.349987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:44.349992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435894110779413:2070] [node 4] ICP22 created new session: [6:7574435894110779439:2070]
2025-11-19T13:59:44.349997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435894110779439:2070] [node 4] ICS09 handshake done sender: [6:7574435894110779417:2113] self: [6:7574435894110779428:0] peer: [4:7574435892159450926:0] socket: 602 qp: -1
2025-11-19T13:59:44.350002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435894110779439:2070] [node 4] ICS10 traffic start
2025-11-19T13:59:44.350007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:44.348646Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893665194272:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.352079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435896067279027:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25872
2025-11-19T13:59:44.352095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435896067279028:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25872
2025-11-19T13:59:44.352170Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435893665194255:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:44.352405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435893665194233:2093] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.352423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:44.352430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435893665194233:2093] [node 4] ICP22 created new session: [3:7574435893665194280:2093]
2025-11-19T13:59:44.352435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435893665194280:2093] [node 4] ICS09 handshake done sender: [3:7574435893665194255:2112] self: [3:7574435893665194254:0] peer: [4:7574435892159450943:0] socket: 587 qp: -1
2025-11-19T13:59:44.352440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435893665194280:2093] [node 4] ICS10 traffic start
2025-11-19T13:59:44.352446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:44.352753Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435892159450936:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.352814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435892159450921:2081] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.352823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:44.352828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435892159450921:2081] [node 6] ICP22 created new session: [4:7574435892159450945:2081]
2025-11-19T13:59:44.352833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435892159450945:2081] [node 6] ICS09 handshake done sender: [4:7574435892159450927:2114] self: [4:7574435892159450926:0] peer: [6:7574435894110779428:0] socket: 599 qp: -1
2025-11-19T13:59:44.352838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435892159450945:2081] [node 6] ICS10 traffic start
2025-11-19T13:59:44.352843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:44.354329Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435894661040412:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:44.354423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435890366073053:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.354436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:44.354443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435890366073053:2048] [node 1] ICP22 created new session: [2:7574435894661040448:2048]
2025-11-19T13:59:44.354448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435894661040448:2048] [node 1] ICS09 handshake done sender: [2:7574435894661040412:2110] self: [2:7574435894661040437:0] peer: [1:7574435892147695719:0] socket: 580 qp: -1
2025-11-19T13:59:44.354452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435894661040448:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:44.354458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.356915Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892147695735:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.356930Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435892147695735:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.357389Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892147695741:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.357665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435892147695749:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49175
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.357883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435896067279029:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25872
2025-11-19T13:59:44.359447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435892159450921:2081] [node 6] ICP18 handshake (actor [4:7574435892159450936:2118]) is too old
2025-11-19T13:59:44.359480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435892159450857:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435892159450924:2112] held: no
2025-11-19T13:59:44.359711Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435892147695722:2108] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.359768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435894110779441:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45366
2025-11-19T13:59:44.359787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435894110779442:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45366
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.360246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435892159450947:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52275
2025-11-19T13:59:44.360424Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435892159450924:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:44.360606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435892159450857:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.360618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:44.360625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435892159450857:2050] [node 3] ICP22 created new session: [4:7574435892159450949:2050]
2025-11-19T13:59:44.360631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435892159450949:2050] [node 3] ICS09 handshake done sender: [4:7574435892159450924:2112] self: [4:7574435892159450943:0] peer: [3:7574435893665194254:0] socket: 597 qp: -1
2025-11-19T13:59:44.360636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435892159450949:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:44.360644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:44.361233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893665194283:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43331
2025-11-19T13:59:44.361451Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435896067279027:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.360546Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435892147695735:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:44.361950Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435896067279021:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.362104Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892147695749:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.362129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435887852728364:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.362145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:44.362151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435887852728364:2049] [node 3] ICP22 created new session: [1:7574435892147695753:2049]
2025-11-19T13:59:44.362156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435892147695753:2049] [node 3] ICS09 handshake done sender: [1:7574435892147695722:2108] self: [1:7574435892147695721:0] peer: [3:7574435893665194273:0] socket: 577 qp: -1
2025-11-19T13:59:44.362162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435892147695753:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:44.362168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:44.362285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435892147695728:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.362292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:44.362296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435892147695728:2109] [node 5] ICP22 created new session: [1:7574435892147695755:2109]
2025-11-19T13:59:44.362299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435892147695755:2109] [node 5] ICS09 handshake done sender: [1:7574435892147695735:2115] self: [1:7574435892147695734:0] peer: [5:7574435896067279026:0] socket: 609 qp: -1
2025-11-19T13:59:44.362303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435892147695755:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:44.362307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:44.362114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435896067278945:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.362141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435896067279030:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25872
2025-11-19T13:59:44.362147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435896067278943:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.362158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:44.362162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435896067278943:2048] [node 1] ICP22 created new session: [5:7574435896067279032:2048]
2025-11-19T13:59:44.362169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435896067279032:2048] [node 1] ICS09 handshake done sender: [5:7574435896067279021:2115] self: [5:7574435896067279026:0] peer: [1:7574435892147695734:0] socket: 610 qp: -1
2025-11-19T13:59:44.362174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435896067279032:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:44.362180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.363936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435894661040450:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34126
2025-11-19T13:59:44.364605Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435896067279028:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.364636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435896067278944:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.365002Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894661040426:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.365010Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435894661040426:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.365574Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435894661040426:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:44.365656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435894661040416:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.365670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:44.365675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435894661040416:2109] [node 5] ICP22 created new session: [2:7574435894661040452:2109]
2025-11-19T13:59:44.365681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435894661040452:2109] [node 5] ICS09 handshake done sender: [2:7574435894661040426:2116] self: [2:7574435894661040425:0] peer: [5:7574435896067279035:0] socket: 626 qp: -1
2025-11-19T13:59:44.365687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435894661040452:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:44.365698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:44.367591Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435894110779441:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.367629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435894110779349:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.368422Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893665194261:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.368437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435893665194261:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.369516Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894661040450:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.369546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435894661040416:2109] [node 5] ICP18 handshake (actor [2:7574435894661040450:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.372236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435892159450951:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52275
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.372460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435896067279036:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25872
2025-11-19T13:59:44.372825Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435896067279030:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.372860Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435896067279028:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:44.372954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435896067278944:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.372984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:44.372992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435896067278944:2049] [node 2] ICP22 created new session: [5:7574435896067279037:2049]
2025-11-19T13:59:44.372998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435896067279037:2049] [node 2] ICS09 handshake done sender: [5:7574435896067279028:2117] self: [5:7574435896067279035:0] peer: [2:7574435894661040425:0] socket: 628 qp: -1
2025-11-19T13:59:44.373009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435896067279037:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:44.373021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:44.373230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435896067278987:2078] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.375622Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435893665194186:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435893665194250:2110] held: no
2025-11-19T13:59:44.375676Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893665194263:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.375680Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435893665194263:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.376257Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435893665194250:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:44.376926Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892147695737:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.376940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435892147695737:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.375289Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435894110779442:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.375333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435894110779347:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.377828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435894110779445:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45366
2025-11-19T13:59:44.385211Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435892147695737:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:44.385467Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435894110779442:2122] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:44.385542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435894110779347:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.385567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:44.385574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435894110779347:2048] [node 1] ICP22 created new session: [6:7574435894110779446:2048]
2025-11-19T13:59:44.385581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435894110779446:2048] [node 1] ICS09 handshake done sender: [6:7574435894110779442:2122] self: [6:7574435894110779444:0] peer: [1:7574435892147695736:0] socket: 600 qp: -1
2025-11-19T13:59:44.385589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435894110779446:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:44.385599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:44.386760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435892147695729:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.386779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:44.386783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435892147695729:2110] [node 6] ICP22 created new session: [1:7574435892147695760:2110]
2025-11-19T13:59:44.386788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435892147695760:2110] [node 6] ICS09 handshake done sender: [1:7574435892147695737:2116] self: [1:7574435892147695736:0] peer: [6:7574435894110779444:0] socket: 632 qp: -1
2025-11-19T13:59:44.386792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435892147695760:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:44.386799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:44.387118Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435892159450951:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.387157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435892159450855:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.387582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435893665194186:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.387605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:44.387612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435893665194186:2048] [node 1] ICP22 created new session: [3:7574435893665194286:2048]
2025-11-19T13:59:44.387619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435893665194286:2048] [node 1] ICS09 handshake done sender: [3:7574435893665194250:2110] self: [3:7574435893665194273:0] peer: [1:7574435892147695721:0] socket: 581 qp: -1
2025-11-19T13:59:44.387625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435893665194286:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:44.387632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:44.387460Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435892159450933:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.387465Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435892159450933:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.388244Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435893665194261:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.388457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435896067279040:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25872
2025-11-19T13:59:44.388368Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435893665194263:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:44.388433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435893665194252:2095] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.388445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:44.388451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435893665194252:2095] [node 5] ICP22 created new session: [3:7574435893665194288:2095]
2025-11-19T13:59:44.388456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435893665194288:2095] [node 5] ICS09 handshake done sender: [3:7574435893665194261:2115] self: [3:7574435893665194260:0] peer: [5:7574435896067279031:0] socket: 625 qp: -1
2025-11-19T13:59:44.388461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435893665194288:2095] [node 5] ICS10 traffic start
2025-11-19T13:59:44.388466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.388660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435896067279041:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25872
2025-11-19T13:59:44.388676Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435896067279027:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:44.388558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435893665194253:2097] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.388569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:44.388572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435893665194253:2097] [node 6] ICP22 created new session: [3:7574435893665194290:2097]
2025-11-19T13:59:44.388575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435893665194290:2097] [node 6] ICS09 handshake done sender: [3:7574435893665194263:2116] self: [3:7574435893665194262:0] peer: [6:7574435894110779443:0] socket: 629 qp: -1
2025-11-19T13:59:44.388578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435893665194290:2097] [node 6] ICS10 traffic start
2025-11-19T13:59:44.388582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:44.388757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435896067278945:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.388771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:44.388776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435896067278945:2050] [node 3] ICP22 created new session: [5:7574435896067279042:2050]
2025-11-19T13:59:44.388781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435896067279042:2050] [node 3] ICS09 handshake done sender: [5:7574435896067279027:2116] self: [5:7574435896067279031:0] peer: [3:7574435893665194260:0] socket: 604 qp: -1
2025-11-19T13:59:44.388787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435896067279042:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:44.388792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:44.389686Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435892159450933:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:44.389777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435892159450922:2088] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.389792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:44.389797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435892159450922:2088] [node 5] ICP22 created new session: [4:7574435892159450954:2088]
2025-11-19T13:59:44.389802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435892159450954:2088] [node 5] ICS09 handshake done sender: [4:7574435892159450933:2117] self: [4:7574435892159450932:0] peer: [5:7574435896067279039:0] socket: 608 qp: -1
2025-11-19T13:59:44.389807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435892159450954:2088] [node 5] ICS10 traffic start
2025-11-19T13:59:44.389813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:44.391909Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892147695739:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.391922Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435892147695739:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.392763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435892159450956:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52275
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.397381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435894110779349:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435894110779441:2121] held: no
2025-11-19T13:59:44.397463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435894110779448:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45366
2025-11-19T13:59:44.397625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435896067278987:2078] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13260) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435896067279030:2119] held: no
2025-11-19T13:59:44.397656Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435896067279030:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:44.397813Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435894110779441:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:44.398336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435896067278987:2078] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.398354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:44.398361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435896067278987:2078] [node 4] ICP22 created new session: [5:7574435896067279044:2078]
2025-11-19T13:59:44.398367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435896067279044:2078] [node 4] ICS09 handshake done sender: [5:7574435896067279030:2119] self: [5:7574435896067279039:0] peer: [4:7574435892159450932:0] socket: 635 qp: -1
2025-11-19T13:59:44.398376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435896067279044:2078] [node 4] ICS10 traffic start
2025-11-19T13:59:44.398389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:44.398743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435894110779349:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.398756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:44.398761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435894110779349:2050] [node 3] ICP22 created new session: [6:7574435894110779449:2050]
2025-11-19T13:59:44.398767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435894110779449:2050] [node 3] ICS09 handshake done sender: [6:7574435894110779441:2121] self: [6:7574435894110779443:0] peer: [3:7574435893665194262:0] socket: 585 qp: -1
2025-11-19T13:59:44.398772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435894110779449:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:44.398778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:44.399095Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435892147695739:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:44.399267Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435892159450951:2121] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:44.399338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435892159450855:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.399351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:44.399356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435892159450855:2048] [node 1] ICP22 created new session: [4:7574435892159450957:2048]
2025-11-19T13:59:44.399361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435892159450957:2048] [node 1] ICS09 handshake done sender: [4:7574435892159450951:2121] self: [4:7574435892159450952:0] peer: [1:7574435892147695738:0] socket: 637 qp: -1
2025-11-19T13:59:44.399366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435892159450957:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:44.399372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:44.400440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435892147695730:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.400459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:44.400465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435892147695730:2111] [node 4] ICP22 created new session: [1:7574435892147695763:2111]
2025-11-19T13:59:44.400470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435892147695763:2111] [node 4] ICS09 handshake done sender: [1:7574435892147695739:2117] self: [1:7574435892147695738:0] peer: [4:7574435892159450952:0] socket: 606 qp: -1
2025-11-19T13:59:44.400474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435892147695763:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:44.400481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:44.408363Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:44.408467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435894661040448:2048] [node 1] ICS01 socket: 580 reason# EndOfStream
2025-11-19T13:59:44.408477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435890366073053:2048] [node 1] ICP30 unregister session Session# [2:7574435894661040448:2048] VirtualId# [2:7574435894661040437:0]
2025-11-19T13:59:44.408485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435894661040448:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.408527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:44.408551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:44.412137Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:44.412214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435893665194286:2048] [node 1] ICS01 socket: 581 reason# EndOfStream
2025-11-19T13:59:44.412222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435893665194186:2048] [node 1] ICP30 unregister session Session# [3:7574435893665194286:2048] VirtualId# [3:7574435893665194273:0]
2025-11-19T13:59:44.412228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435893665194286:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.412257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:44.412273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:44.425365Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:44.425476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435896067279032:2048] [node 1] ICS01 socket: 610 reason# EndOfStream
2025-11-19T13:59:44.425486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435896067278943:2048] [node 1] ICP30 unregister session Session# [5:7574435896067279032:2048] VirtualId# [5:7574435896067279026:0]
2025-11-19T13:59:44.425496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435896067279032:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.425542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:44.425564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:44.425675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:44.425733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435894110779446:2048] [node 1] ICS01 socket: 600 reason# EndOfStream
2025-11-19T13:59:44.425737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435894110779347:2048] [node 1] ICP30 unregister session Session# [6:7574435894110779446:2048] VirtualId# [6:7574435894110779444:0]
2025-11-19T13:59:44.425741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435894110779446:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.425766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:44.425786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:44.425813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:44.425873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435892159450957:2048] [node 1] ICS01 socket: 637 reason# EndOfStream
2025-11-19T13:59:44.425879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435892159450855:2048] [node 1] ICP30 unregister session Session# [4:7574435892159450957:2048] VirtualId# [4:7574435892159450952:0]
2025-11-19T13:59:44.425883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435892159450957:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.425916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:44.425933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:44.427154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:44.427229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435893665194275:2049] [node 2] ICS01 socket: 615 reason# EndOfStream
2025-11-19T13:59:44.427235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435893665194187:2049] [node 2] ICP30 unregister session Session# [3:7574435893665194275:2049] VirtualId# [3:7574435893665194271:0]
2025-11-19T13:59:44.427239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435893665194275:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.427266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:44.427281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:44.436103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:44.436201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435894110779437:2049] [node 2] ICS01 socket: 596 reason# EndOfStream
2025-11-19T13:59:44.436208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435894110779348:2049] [node 2] ICP30 unregister session Session# [6:7574435894110779437:2049] VirtualId# [6:7574435894110779430:0]
2025-11-19T13:59:44.436216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435894110779437:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.436260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:44.436283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:44.436311Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:44.436371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435896067279037:2049] [node 2] ICS01 socket: 628 reason# EndOfStream
2025-11-19T13:59:44.436374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435896067278944:2049] [node 2] ICP30 unregister session Session# [5:7574435896067279037:2049] VirtualId# [5:7574435896067279035:0]
2025-11-19T13:59:44.436377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435896067279037:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.436400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:44.436421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:44.436623Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:44.436684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435892159450940:2049] [node 2] ICS01 socket: 594 reason# EndOfStream
2025-11-19T13:59:44.436688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435892159450856:2049] [node 2] ICP30 unregister session Session# [4:7574435892159450940:2049] VirtualId# [4:7574435892159450938:0]
2025-11-19T13:59:44.436692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435892159450940:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.436717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:44.436734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:44.450689Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:44.450795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435896067279042:2050] [node 3] ICS01 socket: 604 reason# EndOfStream
2025-11-19T13:59:44.450806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435896067278945:2050] [node 3] ICP30 unregister session Session# [5:7574435896067279042:2050] VirtualId# [5:7574435896067279031:0]
2025-11-19T13:59:44.450814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435896067279042:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.450863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:44.450886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:44.450916Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:44.450972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435894110779449:2050] [node 3] ICS01 socket: 585 reason# EndOfStream
2025-11-19T13:59:44.450977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435894110779349:2050] [node 3] ICP30 unregister session Session# [6:7574435894110779449:2050] VirtualId# [6:7574435894110779443:0]
2025-11-19T13:59:44.450982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435894110779449:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.451018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:44.451036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:44.466723Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:44.466828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435894110779439:2070] [node 4] ICS01 socket: 602 reason# EndOfStream
2025-11-19T13:59:44.466839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435894110779413:2070] [node 4] ICP30 unregister session Session# [6:7574435894110779439:2070] VirtualId# [6:7574435894110779428:0]
2025-11-19T13:59:44.466847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435894110779439:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.466900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:44.466922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:44.592555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435892811881580:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:44.592579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435892811881581:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:44.596293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435894134688964:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:44.596308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435894134688965:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:44.620830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435893378908495:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:44.620845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435893378908496:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:44.639818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435895330949195:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:44.639834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435895330949196:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:44.639840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435895330949197:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:44.659903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435893204055737:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:44.659916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435893204055738:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:44.659921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435893204055739:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:44.651932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435895577991605:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:44.651950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435895577991606:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:44.651955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435895577991607:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:44.728112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435892811881642:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:44.728253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435892811881643:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:44.728247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435893204055781:2080] [node 4] ICP01 ready to work
2025-11-19T13:59:44.728345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435893204055782:2085] [node 5] ICP01 ready to work
2025-11-19T13:59:44.738544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435895577991666:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:44.739803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435892811881650:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:44.745452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435894134689029:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.745986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435893378908555:2105] [node 4] ICP01 ready to work
2025-11-19T13:59:44.746092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435893378908556:2106] [node 6] ICP01 ready to work
2025-11-19T13:59:44.746264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435894134689030:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25190
2025-11-19T13:59:44.746285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435894134689031:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25190
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.755880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435895330949196:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20676041ns) {Configure:104(20674403ns) {ProcessConfigured:135(20261915ns) {StartInitialHandshake:157(20261100ns) {DropHandshakes:469(987ns) {DropIncomingHandshake:441(111ns) DropOutgoingHandshake:456(198ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:476(1534ns) {SwitchToState:178(730ns)}}}}}
2025-11-19T13:59:44.756089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435895330949260:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:44.756178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435895330949261:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:44.757957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435895577991670:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40721
2025-11-19T13:59:44.757984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435895577991671:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40721
2025-11-19T13:59:44.758100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435895577991679:2114] [node 4] ICP01 ready to work
2025-11-19T13:59:44.760045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435894134689025:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:44.760206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435894134689040:2114] [node 6] ICP01 ready to work
2025-11-19T13:59:44.767084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435893204055739:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(38683028ns) {Configure:104(38681819ns) {ProcessConfigured:135(38408522ns) {StartInitialHandshake:157(38408007ns) {DropHandshakes:469(604ns) {DropIncomingHandshake:441(123ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(1493ns) {SwitchToState:178(723ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.773379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435892811881658:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4672
2025-11-19T13:59:44.781283Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435895577991671:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.781333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435895577991605:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.779960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435894134689025:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19690154ns) {Configure:104(19689227ns) {ProcessConfigured:135(19371512ns) {StartInitialHandshake:157(19370918ns) {DropHandshakes:469(590ns) {DropIncomingHandshake:441(120ns) DropOutgoingHandshake:456(88ns)} GenerateSessionVirtualId:988(93ns) PrepareNewSessionHandshake:476(1921ns) {SwitchToState:178(934ns)}}}}}
2025-11-19T13:59:44.783407Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894134689031:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.783493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435894134689044:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25190
2025-11-19T13:59:44.783511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435894134688964:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.783706Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435893378908555:2105] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(37571929ns) {Configure:104(37570877ns) {ProcessConfigured:135(37226145ns) {StartInitialHandshake:157(37225234ns) {DropHandshakes:469(750ns) {DropIncomingHandshake:441(87ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(1260ns) {SwitchToState:178(443ns)}}}}}
2025-11-19T13:59:44.783974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435893378908565:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:44.784870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893378908563:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33883
2025-11-19T13:59:44.784933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893378908564:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33883
2025-11-19T13:59:44.785384Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435893378908556:2106] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(38808419ns) {Configure:104(24713436ns) {ProcessConfigured:135(24383922ns) {StartInitialHandshake:157(24383648ns) {DropHandshakes:469(612ns) {DropIncomingHandshake:441(153ns) DropOutgoingHandshake:456(137ns)} GenerateSessionVirtualId:988(267ns) PrepareNewSessionHandshake:476(506ns) {SwitchToState:178(380ns)}}}}}
2025-11-19T13:59:44.785531Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892811881652:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.785542Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435892811881652:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.785935Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892811881647:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.785937Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435892811881647:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.786671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435892811881581:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(58378865ns) {Configure:104(58378433ns) {ProcessConfigured:135(56642278ns) {StartInitialHandshake:157(56641609ns) {DropHandshakes:469(160ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(26ns)} GenerateSessionVirtualId:988(108ns) PrepareNewSessionHandshake:476(650ns) {SwitchToState:178(378ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.789835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435895577991686:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40721
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.791135Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435892811881652:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.791360Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435892811881647:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:44.791467Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893378908563:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.791504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435893378908496:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.791593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435895577991687:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40721
2025-11-19T13:59:44.790709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435895330949260:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(34095017ns) {Configure:104(34093630ns) {ProcessConfigured:135(33878912ns) {StartInitialHandshake:157(33878621ns) {DropHandshakes:469(190ns) {DropIncomingHandshake:441(40ns) DropOutgoingHandshake:456(36ns)} GenerateSessionVirtualId:988(538ns) PrepareNewSessionHandshake:476(790ns) {SwitchToState:178(391ns)}}}}}
2025-11-19T13:59:44.791247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435895330949265:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21859
2025-11-19T13:59:44.791688Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435895577991671:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:44.794923Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893378908564:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.800476Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435894134689040:2114] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26431103ns) {Configure:104(26429183ns) {ProcessConfigured:135(26071150ns) {StartInitialHandshake:157(26070352ns) {DropHandshakes:469(825ns) {DropIncomingHandshake:441(116ns) DropOutgoingHandshake:456(138ns)} GenerateSessionVirtualId:988(344ns) PrepareNewSessionHandshake:476(1243ns) {SwitchToState:178(928ns)}}}}}
2025-11-19T13:59:44.800655Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894134689033:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.800660Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435894134689033:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.802156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435895330949266:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21859
2025-11-19T13:59:44.801696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435893204055815:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34287
2025-11-19T13:59:44.804033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435895330949267:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21859
2025-11-19T13:59:44.804220Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435895330949265:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.805487Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435894134689033:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:44.807778Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894134689030:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.807863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435894134688965:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.807880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:44.807886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435894134688965:2049] [node 3] ICP22 created new session: [2:7574435894134689051:2049]
2025-11-19T13:59:44.807893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435894134689051:2049] [node 3] ICS09 handshake done sender: [2:7574435894134689033:2112] self: [2:7574435894134689032:0] peer: [3:7574435893378908576:0] socket: 595 qp: -1
2025-11-19T13:59:44.807902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435894134689051:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:44.807910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:44.808174Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435895330949267:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.812785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435893204055816:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34287
2025-11-19T13:59:44.811830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435892811881643:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.811855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:44.811868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435892811881643:2108] [node 5] ICP22 created new session: [1:7574435892811881662:2108]
2025-11-19T13:59:44.811876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435892811881662:2108] [node 5] ICS09 handshake done sender: [1:7574435892811881652:2112] self: [1:7574435892811881651:0] peer: [5:7574435895577991685:0] socket: 590 qp: -1
2025-11-19T13:59:44.811885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435892811881662:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:44.811909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:44.812078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435892811881580:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.812087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:44.812092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435892811881580:2048] [node 2] ICP22 created new session: [1:7574435892811881664:2048]
2025-11-19T13:59:44.812096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435892811881664:2048] [node 2] ICS09 handshake done sender: [1:7574435892811881647:2110] self: [1:7574435892811881645:0] peer: [2:7574435894134689045:0] socket: 588 qp: -1
2025-11-19T13:59:44.812100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435892811881664:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:44.812104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:44.812221Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892811881658:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.812597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435892811881580:2048] [node 2] ICP18 handshake (actor [1:7574435892811881658:2114]) is too old
2025-11-19T13:59:44.812642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435892811881666:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4672
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.813095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435895577991605:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.813126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:44.813133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435895577991605:2048] [node 1] ICP22 created new session: [5:7574435895577991688:2048]
2025-11-19T13:59:44.813138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435895577991688:2048] [node 1] ICS09 handshake done sender: [5:7574435895577991671:2110] self: [5:7574435895577991685:0] peer: [1:7574435892811881651:0] socket: 597 qp: -1
2025-11-19T13:59:44.813146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435895577991688:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:44.813155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:44.813683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435895577991690:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40721
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.816597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435894134689048:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25190
2025-11-19T13:59:44.816776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435894134689050:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25190
2025-11-19T13:59:44.816844Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894134689044:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.817698Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435893204055815:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.817784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435893204055782:2085] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.817907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435894134689049:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25190
2025-11-19T13:59:44.816475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435895330949268:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21859
2025-11-19T13:59:44.816536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435895330949196:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.816575Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435895330949266:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.821079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435893204055818:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34287
2025-11-19T13:59:44.817481Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435895577991686:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.817509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435895577991607:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.819973Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435895577991670:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.820020Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435895577991668:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.820023Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435895577991668:2108] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.820889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435895577991693:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40721
2025-11-19T13:59:44.821502Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435893204055816:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.821523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435893204055737:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.821873Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435893204055815:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:44.821702Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435895577991668:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:44.818619Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893378908569:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.818634Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435893378908569:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.821935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435893204055782:2085] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.821952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:44.821957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435893204055782:2085] [node 5] ICP22 created new session: [6:7574435893204055820:2085]
2025-11-19T13:59:44.821963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435893204055820:2085] [node 5] ICS09 handshake done sender: [6:7574435893204055815:2115] self: [6:7574435893204055817:0] peer: [5:7574435895577991667:0] socket: 587 qp: -1
2025-11-19T13:59:44.821969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435893204055820:2085] [node 5] ICS10 traffic start
2025-11-19T13:59:44.821976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:44.823544Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435893378908569:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:44.823892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435893378908565:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.823913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:44.823919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435893378908565:2107] [node 5] ICP22 created new session: [3:7574435893378908579:2107]
2025-11-19T13:59:44.823936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435893378908579:2107] [node 5] ICS09 handshake done sender: [3:7574435893378908569:2114] self: [3:7574435893378908568:0] peer: [5:7574435895577991692:0] socket: 606 qp: -1
2025-11-19T13:59:44.823942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435893378908579:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:44.823949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.825518Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435894134688964:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16402) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435894134689031:2111] held: no
2025-11-19T13:59:44.825569Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894134689048:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.825608Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894134689035:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.825614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435894134689035:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.823126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435895577991666:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.823145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:44.823151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435895577991666:2107] [node 6] ICP22 created new session: [5:7574435895577991696:2107]
2025-11-19T13:59:44.823157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435895577991696:2107] [node 6] ICS09 handshake done sender: [5:7574435895577991668:2108] self: [5:7574435895577991667:0] peer: [6:7574435893204055817:0] socket: 584 qp: -1
2025-11-19T13:59:44.823162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435895577991696:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:44.823183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:44.826505Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892811881654:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.826515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435892811881654:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.827072Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892811881649:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.827077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435892811881649:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.824352Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435895330949268:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.824414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435895330949195:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.824440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435895330949279:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21859
2025-11-19T13:59:44.824502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435895330949197:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.829368Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435892811881654:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:44.829511Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435892811881649:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:44.829582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435892811881650:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.829593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:44.829600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435892811881650:2109] [node 6] ICP22 created new session: [1:7574435892811881671:2109]
2025-11-19T13:59:44.829605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435892811881671:2109] [node 6] ICS09 handshake done sender: [1:7574435892811881654:2113] self: [1:7574435892811881653:0] peer: [6:7574435893204055819:0] socket: 623 qp: -1
2025-11-19T13:59:44.829610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435892811881671:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:44.829618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:44.829696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435892811881642:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.829702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:44.829706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435892811881642:2107] [node 4] ICP22 created new session: [1:7574435892811881673:2107]
2025-11-19T13:59:44.829709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435892811881673:2107] [node 4] ICS09 handshake done sender: [1:7574435892811881649:2111] self: [1:7574435892811881648:0] peer: [4:7574435895330949280:0] socket: 589 qp: -1
2025-11-19T13:59:44.829712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435892811881673:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:44.829715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:44.826014Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435894134689031:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:44.826150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435894134688965:2049] [node 3] ICP18 handshake (actor [2:7574435894134689048:2118]) is too old
2025-11-19T13:59:44.827896Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435894134689035:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:44.828002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435894134689029:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.828011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:44.828017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435894134689029:2108] [node 4] ICP22 created new session: [2:7574435894134689058:2108]
2025-11-19T13:59:44.828022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435894134689058:2108] [node 4] ICS09 handshake done sender: [2:7574435894134689035:2113] self: [2:7574435894134689034:0] peer: [4:7574435895330949278:0] socket: 596 qp: -1
2025-11-19T13:59:44.828027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435894134689058:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:44.828034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.831106Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435895330949279:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.831242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435895330949282:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21859
2025-11-19T13:59:44.832154Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893378908562:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.832173Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435893378908562:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.832001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435894134688964:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.832022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:44.832029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435894134688964:2048] [node 1] ICP22 created new session: [2:7574435894134689060:2048]
2025-11-19T13:59:44.832034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435894134689060:2048] [node 1] ICS09 handshake done sender: [2:7574435894134689031:2111] self: [2:7574435894134689045:0] peer: [1:7574435892811881645:0] socket: 593 qp: -1
2025-11-19T13:59:44.832040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435894134689060:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:44.832047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.833728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435895330949283:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21859
2025-11-19T13:59:44.833882Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435895330949267:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:44.833942Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435895330949266:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:44.833997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435895330949196:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.834012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:44.834017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435895330949196:2049] [node 2] ICP22 created new session: [4:7574435895330949285:2049]
2025-11-19T13:59:44.834022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435895330949285:2049] [node 2] ICS09 handshake done sender: [4:7574435895330949267:2114] self: [4:7574435895330949278:0] peer: [2:7574435894134689034:0] socket: 610 qp: -1
2025-11-19T13:59:44.834027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435895330949285:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:44.834045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:44.834174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435895330949195:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.834181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:44.834187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435895330949195:2048] [node 1] ICP22 created new session: [4:7574435895330949287:2048]
2025-11-19T13:59:44.834190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435895330949287:2048] [node 1] ICS09 handshake done sender: [4:7574435895330949266:2113] self: [4:7574435895330949280:0] peer: [1:7574435892811881648:0] socket: 609 qp: -1
2025-11-19T13:59:44.834194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435895330949287:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:44.834199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.835781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435892811881675:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4672
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.836316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435895330949291:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21859
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.835492Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435893378908496:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26210) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435893378908563:2111] held: no
2025-11-19T13:59:44.835645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893378908581:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33883
2025-11-19T13:59:44.838719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893378908582:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33883
2025-11-19T13:59:44.838779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893378908583:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33883
2025-11-19T13:59:44.838878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893378908586:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33883
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.839438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435893204055823:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34287
2025-11-19T13:59:44.839479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435893204055824:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34287
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.839776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435893204055825:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34287
2025-11-19T13:59:44.839793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435893204055826:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34287
2025-11-19T13:59:44.839827Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435893204055823:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.839847Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435893204055816:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:44.839951Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435893378908562:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:44.839905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435893204055738:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.840087Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893378908583:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.840102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435893378908555:2105] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.840112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:44.840118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435893378908555:2105] [node 4] ICP22 created new session: [3:7574435893378908588:2105]
2025-11-19T13:59:44.840123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435893378908588:2105] [node 4] ICS09 handshake done sender: [3:7574435893378908562:2110] self: [3:7574435893378908557:0] peer: [4:7574435895330949281:0] socket: 604 qp: -1
2025-11-19T13:59:44.840127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435893378908588:2105] [node 4] ICS10 traffic start
2025-11-19T13:59:44.840135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:44.840102Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894134689046:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.840115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435894134689046:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.840610Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435895577991686:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:44.840648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435893204055737:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.840663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:44.840669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435893204055737:2048] [node 1] ICP22 created new session: [6:7574435893204055828:2048]
2025-11-19T13:59:44.840675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435893204055828:2048] [node 1] ICS09 handshake done sender: [6:7574435893204055816:2116] self: [6:7574435893204055819:0] peer: [1:7574435892811881653:0] socket: 625 qp: -1
2025-11-19T13:59:44.840680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435893204055828:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:44.840700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:44.841182Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435895577991607:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23428) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435895577991686:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.841248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435895577991698:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40721
2025-11-19T13:59:44.843659Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893378908582:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.843689Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435893378908563:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:44.844381Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435895577991698:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.844395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435895577991607:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.844405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:44.844412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435895577991607:2050] [node 3] ICP22 created new session: [5:7574435895577991699:2050]
2025-11-19T13:59:44.844417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435895577991699:2050] [node 3] ICS09 handshake done sender: [5:7574435895577991686:2117] self: [5:7574435895577991692:0] peer: [3:7574435893378908568:0] socket: 613 qp: -1
2025-11-19T13:59:44.844423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435895577991699:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:44.844430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:44.844548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435895577991606:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.843926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435893378908495:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.846909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435893378908592:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33883
2025-11-19T13:59:44.847174Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435894134689043:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.847189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435894134689043:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.845100Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435895330949268:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:44.846471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435895330949197:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.846482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:44.846487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435895330949197:2050] [node 3] ICP22 created new session: [4:7574435895330949292:2050]
2025-11-19T13:59:44.846492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435895330949292:2050] [node 3] ICS09 handshake done sender: [4:7574435895330949268:2115] self: [4:7574435895330949281:0] peer: [3:7574435893378908557:0] socket: 611 qp: -1
2025-11-19T13:59:44.846498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435895330949292:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:44.846504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:44.845234Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892811881661:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.845246Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435892811881661:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.845719Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435892811881675:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.845965Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435892811881661:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:44.846036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435892811881581:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.846048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:44.846053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435892811881581:2049] [node 3] ICP22 created new session: [1:7574435892811881678:2049]
2025-11-19T13:59:44.846058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435892811881678:2049] [node 3] ICS09 handshake done sender: [1:7574435892811881661:2115] self: [1:7574435892811881646:0] peer: [3:7574435893378908591:0] socket: 624 qp: -1
2025-11-19T13:59:44.846063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435892811881678:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:44.846071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:44.846371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435893204055830:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34287
2025-11-19T13:59:44.847049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435893204055826:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.847072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435893204055739:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.847486Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435894134689046:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:44.847846Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435893204055823:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:44.847957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435894134689040:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.847979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:44.847990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435894134689040:2114] [node 6] ICP22 created new session: [2:7574435894134689063:2114]
2025-11-19T13:59:44.847995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435894134689063:2114] [node 6] ICS09 handshake done sender: [2:7574435894134689046:2117] self: [2:7574435894134689042:0] peer: [6:7574435893204055827:0] socket: 633 qp: -1
2025-11-19T13:59:44.848002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435894134689063:2114] [node 6] ICS10 traffic start
2025-11-19T13:59:44.848009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:44.848478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435893204055738:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.848500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:44.848507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435893204055738:2049] [node 2] ICP22 created new session: [6:7574435893204055832:2049]
2025-11-19T13:59:44.848512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435893204055832:2049] [node 2] ICS09 handshake done sender: [6:7574435893204055823:2118] self: [6:7574435893204055827:0] peer: [2:7574435894134689042:0] socket: 615 qp: -1
2025-11-19T13:59:44.848518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435893204055832:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:44.848526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:44.848544Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435893378908575:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.848556Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435893378908575:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.864323Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435894134689043:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:44.864487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435894134689025:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.864506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:44.864512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435894134689025:2107] [node 5] ICP22 created new session: [2:7574435894134689066:2107]
2025-11-19T13:59:44.864520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435894134689066:2107] [node 5] ICS09 handshake done sender: [2:7574435894134689043:2115] self: [2:7574435894134689041:0] peer: [5:7574435895577991701:0] socket: 586 qp: -1
2025-11-19T13:59:44.864526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435894134689066:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:44.864535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:44.864831Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435895330949270:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.864843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435895330949270:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.865666Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435893378908495:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16402) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435893378908582:2118] held: no
2025-11-19T13:59:44.865695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435893378908496:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.865705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:44.865711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435893378908496:2049] [node 2] ICP22 created new session: [3:7574435893378908593:2049]
2025-11-19T13:59:44.865717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435893378908593:2049] [node 2] ICS09 handshake done sender: [3:7574435893378908563:2111] self: [3:7574435893378908576:0] peer: [2:7574435894134689032:0] socket: 602 qp: -1
2025-11-19T13:59:44.865723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435893378908593:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:44.865755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:44.865858Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435893378908582:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:44.866203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435893378908495:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.866213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:44.866217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435893378908495:2048] [node 1] ICP22 created new session: [3:7574435893378908596:2048]
2025-11-19T13:59:44.866222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435893378908596:2048] [node 1] ICS09 handshake done sender: [3:7574435893378908582:2118] self: [3:7574435893378908591:0] peer: [1:7574435892811881646:0] socket: 631 qp: -1
2025-11-19T13:59:44.866226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435893378908596:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:44.866231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.866710Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435893378908575:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:44.864212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435895577991702:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40721
2025-11-19T13:59:44.864721Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435895577991690:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.864751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435895577991679:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.864790Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435895577991698:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:44.864958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435895577991606:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.864970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:44.864979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435895577991606:2049] [node 2] ICP22 created new session: [5:7574435895577991704:2049]
2025-11-19T13:59:44.864985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435895577991704:2049] [node 2] ICS09 handshake done sender: [5:7574435895577991698:2121] self: [5:7574435895577991701:0] peer: [2:7574435894134689041:0] socket: 603 qp: -1
2025-11-19T13:59:44.864989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435895577991704:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:44.864996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:44.867994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435893378908556:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.868010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:44.868016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435893378908556:2106] [node 6] ICP22 created new session: [3:7574435893378908598:2106]
2025-11-19T13:59:44.868020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435893378908598:2106] [node 6] ICS09 handshake done sender: [3:7574435893378908575:2116] self: [3:7574435893378908561:0] peer: [6:7574435893204055831:0] socket: 641 qp: -1
2025-11-19T13:59:44.868024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435893378908598:2106] [node 6] ICS10 traffic start
2025-11-19T13:59:44.868031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:44.865962Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435893204055825:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.865992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435893204055781:2080] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:44.866548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435893204055835:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34287
2025-11-19T13:59:44.866900Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435893204055826:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:44.866948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435893204055739:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.866958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:44.866963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435893204055739:2050] [node 3] ICP22 created new session: [6:7574435893204055836:2050]
2025-11-19T13:59:44.866968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435893204055836:2050] [node 3] ICS09 handshake done sender: [6:7574435893204055826:2121] self: [6:7574435893204055831:0] peer: [3:7574435893378908561:0] socket: 644 qp: -1
2025-11-19T13:59:44.866974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435893204055836:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:44.866980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:44.867910Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435895330949270:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:44.868331Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435895330949274:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:44.868336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435895330949274:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:44.868903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435895330949261:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.868920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:44.868925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435895330949261:2109] [node 5] ICP22 created new session: [4:7574435895330949295:2109]
2025-11-19T13:59:44.868929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435895330949295:2109] [node 5] ICS09 handshake done sender: [4:7574435895330949270:2116] self: [4:7574435895330949269:0] peer: [5:7574435895577991703:0] socket: 619 qp: -1
2025-11-19T13:59:44.868934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435895330949295:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:44.868951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.870166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435893204055838:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34287
2025-11-19T13:59:44.870426Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435893204055825:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:44.870471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435893204055781:2080] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.870478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:44.870483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435893204055781:2080] [node 4] ICP22 created new session: [6:7574435893204055839:2080]
2025-11-19T13:59:44.870487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435893204055839:2080] [node 4] ICS09 handshake done sender: [6:7574435893204055825:2120] self: [6:7574435893204055834:0] peer: [4:7574435895330949264:0] socket: 643 qp: -1
2025-11-19T13:59:44.870492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435893204055839:2080] [node 4] ICS10 traffic start
2025-11-19T13:59:44.870498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:44.874098Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435895577991679:2114] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25429) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435895577991690:2119] held: no
2025-11-19T13:59:44.874154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435895577991706:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40721
2025-11-19T13:59:44.870260Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435895330949274:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:44.871367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435895330949260:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:44.871379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:44.871385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435895330949260:2108] [node 6] ICP22 created new session: [4:7574435895330949298:2108]
2025-11-19T13:59:44.871389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435895330949298:2108] [node 6] ICS09 handshake done sender: [4:7574435895330949274:2118] self: [4:7574435895330949264:0] peer: [6:7574435893204055834:0] socket: 618 qp: -1
2025-11-19T13:59:44.871394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435895330949298:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:44.871401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:44.874855Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435895577991690:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:44.874918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435895577991679:2114] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:44.874930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:44.874934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435895577991679:2114] [node 4] ICP22 created new session: [5:7574435895577991707:2114]
2025-11-19T13:59:44.874939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435895577991707:2114] [node 4] ICS09 handshake done sender: [5:7574435895577991690:2119] self: [5:7574435895577991703:0] peer: [4:7574435895330949269:0] socket: 626 qp: -1
2025-11-19T13:59:44.874953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435895577991707:2114] [node 4] ICS10 traffic start
2025-11-19T13:59:44.874960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:44.886553Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:44.886639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435894134689060:2048] [node 1] ICS01 socket: 593 reason# EndOfStream
2025-11-19T13:59:44.886647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435894134688964:2048] [node 1] ICP30 unregister session Session# [2:7574435894134689060:2048] VirtualId# [2:7574435894134689045:0]
2025-11-19T13:59:44.886655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435894134689060:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.886687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:44.886705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:44.887256Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:44.887319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435893378908596:2048] [node 1] ICS01 socket: 631 reason# EndOfStream
2025-11-19T13:59:44.887324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435893378908495:2048] [node 1] ICP30 unregister session Session# [3:7574435893378908596:2048] VirtualId# [3:7574435893378908591:0]
2025-11-19T13:59:44.887328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435893378908596:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.887355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:44.887369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:44.915139Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:44.915231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435893204055828:2048] [node 1] ICS01 socket: 625 reason# EndOfStream
2025-11-19T13:59:44.915239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435893204055737:2048] [node 1] ICP30 unregister session Session# [6:7574435893204055828:2048] VirtualId# [6:7574435893204055819:0]
2025-11-19T13:59:44.915247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435893204055828:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.915299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:44.915323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:44.915678Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:44.915740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435895330949287:2048] [node 1] ICS01 socket: 609 reason# EndOfStream
2025-11-19T13:59:44.915765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435895330949195:2048] [node 1] ICP30 unregister session Session# [4:7574435895330949287:2048] VirtualId# [4:7574435895330949280:0]
2025-11-19T13:59:44.915770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435895330949287:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.915801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:44.915823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:44.915806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:44.915897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435895577991688:2048] [node 1] ICS01 socket: 597 reason# EndOfStream
2025-11-19T13:59:44.915906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435895577991605:2048] [node 1] ICP30 unregister session Session# [5:7574435895577991688:2048] VirtualId# [5:7574435895577991685:0]
2025-11-19T13:59:44.915912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435895577991688:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.915988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:44.916015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:44.918353Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:44.918436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435893378908593:2049] [node 2] ICS01 socket: 602 reason# EndOfStream
2025-11-19T13:59:44.918443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435893378908496:2049] [node 2] ICP30 unregister session Session# [3:7574435893378908593:2049] VirtualId# [3:7574435893378908576:0]
2025-11-19T13:59:44.918448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435893378908593:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.918479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:44.918498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:44.934781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:44.934868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435895577991704:2049] [node 2] ICS07 socket disconnect 603 reason# EndOfStream
2025-11-19T13:59:44.934873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435895577991704:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.934924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:44.934945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435895577991704:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:44.935569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435895577991606:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26210) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:44.935584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435895577991704:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:44.935591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435895577991606:2049] [node 2] ICP30 unregister session Session# [5:7574435895577991704:2049] VirtualId# [5:7574435895577991701:0]
2025-11-19T13:59:44.935607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:44.935612Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435895577991606:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26210) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:44.935615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26210) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:44.938911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:44.938990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435893204055832:2049] [node 2] ICS01 socket: 615 reason# EndOfStream
2025-11-19T13:59:44.939009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435893204055738:2049] [node 2] ICP30 unregister session Session# [6:7574435893204055832:2049] VirtualId# [6:7574435893204055827:0]
2025-11-19T13:59:44.939017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435893204055832:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.939076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:44.939107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:44.939277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:44.939346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435895330949285:2049] [node 2] ICS01 socket: 610 reason# EndOfStream
2025-11-19T13:59:44.939352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435895330949196:2049] [node 2] ICP30 unregister session Session# [4:7574435895330949285:2049] VirtualId# [4:7574435895330949278:0]
2025-11-19T13:59:44.939356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435895330949285:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.939395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:44.939419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:44.946535Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:44.946634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435895577991699:2050] [node 3] ICS01 socket: 613 reason# EndOfStream
2025-11-19T13:59:44.946643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435895577991607:2050] [node 3] ICP30 unregister session Session# [5:7574435895577991699:2050] VirtualId# [5:7574435895577991692:0]
2025-11-19T13:59:44.946649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435895577991699:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.946705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:44.946729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:44.949959Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:44.950053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435893204055836:2050] [node 3] ICS01 socket: 644 reason# EndOfStream
2025-11-19T13:59:44.950062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435893204055739:2050] [node 3] ICP30 unregister session Session# [6:7574435893204055836:2050] VirtualId# [6:7574435893204055831:0]
2025-11-19T13:59:44.950071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435893204055836:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.950132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:44.950160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:44.950540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435895577991606:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:44.959493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:44.959590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435893204055839:2080] [node 4] ICS01 socket: 643 reason# EndOfStream
2025-11-19T13:59:44.959599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435893204055781:2080] [node 4] ICP30 unregister session Session# [6:7574435893204055839:2080] VirtualId# [6:7574435893204055834:0]
2025-11-19T13:59:44.959610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435893204055839:2080] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:44.959678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:44.959707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:45.230557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435898904548172:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:45.230571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435898904548173:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:45.283945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435899227346641:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:45.283957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435899227346642:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:45.324379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435898015219552:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:45.324392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435898015219553:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:45.355954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435898622246992:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:45.355969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435898622246993:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:45.355974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435898622246994:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:45.379879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435899210896131:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:45.379892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435899210896132:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:45.379897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435899210896133:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:45.404162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435899269251597:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:45.404180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435899269251598:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:45.404185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435899269251599:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:45.448034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435898622247024:2077] [node 6] ICP01 ready to work
2025-11-19T13:59:45.448241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435898622247028:2080] [node 5] ICP01 ready to work
2025-11-19T13:59:45.448279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435899210896158:2072] [node 4] ICP01 ready to work
2025-11-19T13:59:45.448129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435898015219596:2087] [node 4] ICP01 ready to work
2025-11-19T13:59:45.448655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435899210896170:2078] [node 6] ICP01 ready to work
2025-11-19T13:59:45.449671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435899227346702:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:45.450023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435898015219604:2093] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.451296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435898904548234:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:45.451394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435898904548235:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:45.450767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435899269251618:2066] [node 5] ICP01 ready to work
2025-11-19T13:59:45.451197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435899269251619:2067] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56382
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.454916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435898015219612:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13068
2025-11-19T13:59:45.457717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435899227346710:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:45.458303Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435898015219612:2101] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.459706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435899227346713:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:45.467901Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435899210896131:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(19592487ns) {Configure:104(19591415ns) {ProcessConfigured:135(19064928ns) {StartInitialHandshake:157(19064170ns) {DropHandshakes:469(825ns) {DropIncomingHandshake:441(241ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(283ns) PrepareNewSessionHandshake:476(2097ns) {SwitchToState:178(1373ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.468464Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435898622246994:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(20421218ns) {Configure:104(20419731ns) {ProcessConfigured:135(19575826ns) {StartInitialHandshake:157(19575409ns) {DropHandshakes:469(656ns) {DropIncomingHandshake:441(96ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(422ns) PrepareNewSessionHandshake:476(1180ns) {SwitchToState:178(118ns)}}}}}
2025-11-19T13:59:45.469193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435898622247061:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32357
2025-11-19T13:59:45.471279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435898015219599:2089] [node 5] ICP01 ready to work
2025-11-19T13:59:45.470378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435898622247028:2080] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(22098534ns) {Configure:104(22098085ns) {ProcessConfigured:135(21264797ns) {StartInitialHandshake:157(21264653ns) {DropHandshakes:469(127ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(43ns)} GenerateSessionVirtualId:988(103ns) PrepareNewSessionHandshake:476(769ns) {SwitchToState:178(464ns)}}}}}
2025-11-19T13:59:45.469555Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435899210896132:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20847456ns) {Configure:104(20846721ns) {ProcessConfigured:135(20522247ns) {StartInitialHandshake:157(20521944ns) {DropHandshakes:469(609ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(85ns)} GenerateSessionVirtualId:988(269ns) PrepareNewSessionHandshake:476(842ns) {SwitchToState:178(649ns)}}}}}
2025-11-19T13:59:45.469947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435899210896204:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17767
2025-11-19T13:59:45.470271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435899210896158:2072] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(21946879ns) {Configure:104(21946299ns) {ProcessConfigured:135(21207506ns) {StartInitialHandshake:157(21207297ns) {DropHandshakes:469(171ns) {DropIncomingHandshake:441(27ns) DropOutgoingHandshake:456(28ns)} GenerateSessionVirtualId:988(120ns) PrepareNewSessionHandshake:476(728ns) {SwitchToState:178(360ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.475798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435898904548234:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(24366371ns) {Configure:104(24365568ns) {ProcessConfigured:135(17643258ns) {StartInitialHandshake:157(17642212ns) {DropHandshakes:469(779ns) {DropIncomingHandshake:441(170ns) DropOutgoingHandshake:456(109ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:476(888ns) {SwitchToState:178(683ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.476714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435899210896209:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17767
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.475288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435899227346710:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17498946ns) {Configure:104(17497953ns) {ProcessConfigured:135(17171089ns) {StartInitialHandshake:157(17170178ns) {DropHandshakes:469(882ns) {DropIncomingHandshake:441(147ns) DropOutgoingHandshake:456(131ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(1764ns) {SwitchToState:178(771ns)}}}}}
2025-11-19T13:59:45.475574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435899227346717:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58980
2025-11-19T13:59:45.475587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435899227346718:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58980
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.478765Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435899269251599:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(29278414ns) {Configure:104(29277397ns) {ProcessConfigured:135(28889083ns) {StartInitialHandshake:157(28888750ns) {DropHandshakes:469(747ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:476(1399ns) {SwitchToState:178(740ns)}}}}}
2025-11-19T13:59:45.479274Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435899269251619:2067] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.479428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435899269251599:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.479446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435899269251671:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56382
2025-11-19T13:59:45.479649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435899269251598:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20170466ns) {Configure:104(20169806ns) {ProcessConfigured:135(19812418ns) {StartInitialHandshake:157(19812180ns) {DropHandshakes:469(487ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(302ns) PrepareNewSessionHandshake:476(494ns) {SwitchToState:178(274ns)}}}}}
2025-11-19T13:59:45.476445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435898904548245:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:45.476873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435898904548243:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58686
2025-11-19T13:59:45.476889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435898904548244:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58686
2025-11-19T13:59:45.477963Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435898904548173:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(28732667ns) {Configure:104(28728496ns) {ProcessConfigured:135(19599886ns) {StartInitialHandshake:157(19598868ns) {DropHandshakes:469(571ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(107ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:476(854ns) {SwitchToState:178(342ns)}}}}}
2025-11-19T13:59:45.478049Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435898015219553:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19681820ns) {Configure:104(19681307ns) {ProcessConfigured:135(19308426ns) {StartInitialHandshake:157(19308228ns) {DropHandshakes:469(426ns) {DropIncomingHandshake:441(29ns) DropOutgoingHandshake:456(45ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:476(1076ns) {SwitchToState:178(248ns)}}}}}
2025-11-19T13:59:45.478519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435898015219553:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.481941Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435898015219607:2097] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.481949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435898015219607:2097] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.484056Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435899227346641:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24388000ns) {Configure:104(24386717ns) {ProcessConfigured:135(24040921ns) {StartInitialHandshake:157(24040347ns) {DropHandshakes:469(397ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:476(1545ns) {SwitchToState:178(1180ns)}}}}}
2025-11-19T13:59:45.484116Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435899227346706:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.484120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435899227346706:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:45.485285Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435899269251671:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.485349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435899269251597:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.486280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435899269251639:2063] [node 4] ICP01 ready to work
2025-11-19T13:59:45.485838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435898015219552:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25861224ns) {Configure:104(25860153ns) {ProcessConfigured:135(25560908ns) {StartInitialHandshake:157(25560018ns) {DropHandshakes:469(457ns) {DropIncomingHandshake:441(166ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(381ns) PrepareNewSessionHandshake:476(1202ns) {SwitchToState:178(25ns)}}}}}
2025-11-19T13:59:45.486973Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435898015219607:2097] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:45.485178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435898904548252:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58686
2025-11-19T13:59:45.485442Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435898904548252:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.485463Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435898904548242:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.485465Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435898904548242:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:45.486738Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435898904548242:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:45.485437Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435899227346713:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25677867ns) {Configure:104(25677051ns) {ProcessConfigured:135(25383710ns) {StartInitialHandshake:157(25383146ns) {DropHandshakes:469(520ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:476(1246ns) {SwitchToState:178(622ns)}}}}}
2025-11-19T13:59:45.485525Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435899227346706:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:45.491248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435899227346721:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58980
2025-11-19T13:59:45.491291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435899227346642:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.491306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:45.491312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435899227346642:2049] [node 3] ICP22 created new session: [2:7574435899227346724:2049]
2025-11-19T13:59:45.491319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435899227346724:2049] [node 3] ICS09 handshake done sender: [2:7574435899227346706:2109] self: [2:7574435899227346705:0] peer: [3:7574435898015219630:0] socket: 590 qp: -1
2025-11-19T13:59:45.491326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435899227346724:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:45.491369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:45.491822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435898904548235:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.491849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:45.491856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435898904548235:2108] [node 6] ICP22 created new session: [1:7574435898904548255:2108]
2025-11-19T13:59:45.491862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435898904548255:2108] [node 6] ICS09 handshake done sender: [1:7574435898904548242:2111] self: [1:7574435898904548241:0] peer: [6:7574435899269251676:0] socket: 612 qp: -1
2025-11-19T13:59:45.491867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435898904548255:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:45.491885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.492539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435898015219604:2093] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.492556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:45.492564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435898015219604:2093] [node 6] ICP22 created new session: [3:7574435898015219638:2093]
2025-11-19T13:59:45.492580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435898015219638:2093] [node 6] ICS09 handshake done sender: [3:7574435898015219607:2097] self: [3:7574435898015219606:0] peer: [6:7574435899269251672:0] socket: 589 qp: -1
2025-11-19T13:59:45.492585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435898015219638:2093] [node 6] ICS10 traffic start
2025-11-19T13:59:45.492603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:45.492830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435898015219634:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13068
2025-11-19T13:59:45.492842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435898015219635:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13068
2025-11-19T13:59:45.492852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435898015219636:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13068
2025-11-19T13:59:45.492892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435898015219637:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13068
2025-11-19T13:59:45.493247Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435898015219612:2101] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:45.493332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435898015219553:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.493343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:45.493350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435898015219553:2049] [node 2] ICP22 created new session: [3:7574435898015219641:2049]
2025-11-19T13:59:45.493355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435898015219641:2049] [node 2] ICS09 handshake done sender: [3:7574435898015219612:2101] self: [3:7574435898015219630:0] peer: [2:7574435899227346705:0] socket: 593 qp: -1
2025-11-19T13:59:45.493360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435898015219641:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:45.493366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.494330Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435898904548172:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(45111649ns) {Configure:104(45110943ns) {ProcessConfigured:135(36014095ns) {StartInitialHandshake:157(36013114ns) {DropHandshakes:469(345ns) {DropIncomingHandshake:441(190ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:476(777ns) {SwitchToState:178(434ns)}}}}}
2025-11-19T13:59:45.494342Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435898622247061:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.494417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435898622247070:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32357
2025-11-19T13:59:45.494460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435898622247071:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32357
2025-11-19T13:59:45.494493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435898622246993:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.495175Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435898622247071:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.495205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435898622246992:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.495246Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435899227346704:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.495258Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435899227346704:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:45.496483Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435898622247070:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.496509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435898622246994:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.497390Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435899269251597:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16101) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435899269251671:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.496367Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435898015219636:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.498393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435898622247024:2077] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(50300853ns) {Configure:104(50299764ns) {ProcessConfigured:135(49074802ns) {StartInitialHandshake:157(49073888ns) {DropHandshakes:469(141ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(21ns)} GenerateSessionVirtualId:988(358ns) PrepareNewSessionHandshake:476(459ns) {SwitchToState:178(213ns)}}}}}
2025-11-19T13:59:45.497549Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435898015219615:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.497560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435898015219615:2104] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:45.500069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435899269251678:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56382
2025-11-19T13:59:45.502585Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435898904548247:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.502609Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435898904548247:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.507996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435899269251639:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(21637667ns) {Configure:104(21636696ns) {ProcessConfigured:135(21305791ns) {StartInitialHandshake:157(21305161ns) {DropHandshakes:469(452ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(121ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:476(1734ns) {SwitchToState:178(982ns)}}}}}
2025-11-19T13:59:45.508045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435899269251679:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56382
2025-11-19T13:59:45.509399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435898904548259:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58686
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.511649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435898015219604:2093] [node 6] ICP18 handshake (actor [3:7574435898015219636:2118]) is too old
2025-11-19T13:59:45.511688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435898015219643:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13068
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.512876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435898622247076:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32357
2025-11-19T13:59:45.511958Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435899210896204:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.512069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435899210896210:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17767
2025-11-19T13:59:45.512105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435899210896211:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17767
2025-11-19T13:59:45.515338Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435899210896211:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.515375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435899210896133:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.515639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435898904548260:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58686
2025-11-19T13:59:45.515774Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435899210896210:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.515795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435899210896131:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.515891Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435898904548240:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.515897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435898904548240:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.516501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435899210896214:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17767
2025-11-19T13:59:45.516640Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435899227346704:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:45.516736Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435899210896209:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.516754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435899210896132:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.516970Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435898622247061:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:45.517082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435898622246993:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.517100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:45.517107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435898622246993:2049] [node 2] ICP22 created new session: [4:7574435898622247077:2049]
2025-11-19T13:59:45.517113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435898622247077:2049] [node 2] ICS09 handshake done sender: [4:7574435898622247061:2111] self: [4:7574435898622247072:0] peer: [2:7574435899227346703:0] socket: 594 qp: -1
2025-11-19T13:59:45.517120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435898622247077:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:45.517141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.517580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435899210896216:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17767
2025-11-19T13:59:45.517856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435899227346716:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.517864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435899227346716:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:45.518763Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435898015219629:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.518773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435898015219629:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:45.519341Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435898015219643:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.519498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435898015219552:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.519619Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435899210896170:2078] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(51663979ns) {Configure:104(51662738ns) {ProcessConfigured:135(51257820ns) {StartInitialHandshake:157(51257065ns) {DropHandshakes:469(848ns) {DropIncomingHandshake:441(142ns) DropOutgoingHandshake:456(153ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(436ns) {SwitchToState:178(198ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.519879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435899210896219:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17767
2025-11-19T13:59:45.520678Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435899227346642:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(29358690ns) {HandleHandshakeStatus:329(29357922ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(318ns) SwitchToState:178(776ns) ProcessPendingSessionEvents:495(21202354ns) {ForwardSessionEventToSession:612(21186167ns) ForwardSessionEventToSession:612(3387ns) ForwardSessionEventToSession:612(994ns) ForwardSessionEventToSession:612(1347ns) ForwardSessionEventToSession:612(792ns) ForwardSessionEventToSession:612(846ns) ForwardSessionEventToSession:612(979ns) ForwardSessionEventToSession:612(598ns) ForwardSessionEventToSession:612(854ns) ForwardSessionEventToSession:612(780ns)}}}
2025-11-19T13:59:45.520693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435899227346702:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.520704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:45.520710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435899227346702:2107] [node 4] ICP22 created new session: [2:7574435899227346728:2107]
2025-11-19T13:59:45.520714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435899227346728:2107] [node 4] ICS09 handshake done sender: [2:7574435899227346704:2108] self: [2:7574435899227346703:0] peer: [4:7574435898622247072:0] socket: 588 qp: -1
2025-11-19T13:59:45.520720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435899227346728:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:45.520727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:45.520909Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435899227346642:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:45.521162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435898904548235:2108] [node 6] ICP35 event processing took too much time PendingConnection:278(29320996ns) {HandleHandshakeStatus:329(29320406ns) {DropIncomingHandshake:441(5814ns) DropOutgoingHandshake:456(128ns) SwitchToState:178(98ns) ProcessPendingSessionEvents:495(21083865ns) {ForwardSessionEventToSession:612(21074215ns) ForwardSessionEventToSession:612(1439ns) ForwardSessionEventToSession:612(1035ns) ForwardSessionEventToSession:612(642ns) ForwardSessionEventToSession:612(463ns) ForwardSessionEventToSession:612(941ns) ForwardSessionEventToSession:612(614ns) ForwardSessionEventToSession:612(569ns) ForwardSessionEventToSession:612(600ns) ForwardSessionEventToSession:612(617ns)}}}
2025-11-19T13:59:45.521217Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435898904548240:2110] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:45.520379Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435898015219615:2104] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:45.521129Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435898015219629:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:45.521549Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435899227346716:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:45.521744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435899227346710:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.521754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:45.521758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435899227346710:2110] [node 5] ICP22 created new session: [2:7574435899227346732:2110]
2025-11-19T13:59:45.521763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435899227346732:2110] [node 5] ICS09 handshake done sender: [2:7574435899227346716:2112] self: [2:7574435899227346711:0] peer: [5:7574435899210896215:0] socket: 607 qp: -1
2025-11-19T13:59:45.521767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435899227346732:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:45.521773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.521974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435898622247080:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32357
2025-11-19T13:59:45.521878Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435898904548259:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.521910Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435898904548260:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.521949Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435898904548251:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.521951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435898904548251:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:45.522076Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435898622247070:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:45.522290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435898015219596:2087] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.522303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:45.522307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435898015219596:2087] [node 4] ICP22 created new session: [3:7574435898015219648:2087]
2025-11-19T13:59:45.522312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435898015219648:2087] [node 4] ICS09 handshake done sender: [3:7574435898015219615:2104] self: [3:7574435898015219597:0] peer: [4:7574435898622247074:0] socket: 602 qp: -1
2025-11-19T13:59:45.522316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435898015219648:2087] [node 4] ICS10 traffic start
2025-11-19T13:59:45.522320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:45.522405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435898904548234:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.522422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:45.522429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435898904548234:2107] [node 5] ICP22 created new session: [1:7574435898904548262:2107]
2025-11-19T13:59:45.522434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435898904548262:2107] [node 5] ICS09 handshake done sender: [1:7574435898904548240:2110] self: [1:7574435898904548236:0] peer: [5:7574435899210896213:0] socket: 611 qp: -1
2025-11-19T13:59:45.522441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435898904548262:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:45.522449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:45.528056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435898622246994:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.528092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:45.528100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435898622246994:2050] [node 3] ICP22 created new session: [4:7574435898622247081:2050]
2025-11-19T13:59:45.528107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435898622247081:2050] [node 3] ICS09 handshake done sender: [4:7574435898622247070:2115] self: [4:7574435898622247074:0] peer: [3:7574435898015219597:0] socket: 604 qp: -1
2025-11-19T13:59:45.528116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435898622247081:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:45.528150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:45.529265Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435899269251671:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.530550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435899227346734:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58980
2025-11-19T13:59:45.530675Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435898904548247:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:45.530960Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435898904548251:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.531484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435898015219650:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13068
2025-11-19T13:59:45.534031Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435899269251619:2067] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.534127Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435899269251599:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435899269251619:2067] held: no
2025-11-19T13:59:45.534151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435899269251599:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.534166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:45.534173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435899269251599:2050] [node 3] ICP22 created new session: [6:7574435899269251682:2050]
2025-11-19T13:59:45.534179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435899269251682:2050] [node 3] ICS09 handshake done sender: [6:7574435899269251619:2067] self: [6:7574435899269251672:0] peer: [3:7574435898015219606:0] socket: 591 qp: -1
2025-11-19T13:59:45.534186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435899269251682:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:45.534207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:45.534326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435899269251597:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.534332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:45.534337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435899269251597:2048] [node 1] ICP22 created new session: [6:7574435899269251684:2048]
2025-11-19T13:59:45.534342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435899269251684:2048] [node 1] ICS09 handshake done sender: [6:7574435899269251671:2114] self: [6:7574435899269251676:0] peer: [1:7574435898904548241:0] socket: 616 qp: -1
2025-11-19T13:59:45.534346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435899269251684:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:45.534351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:45.534488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435899269251681:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56382
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.534899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435899227346735:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58980
2025-11-19T13:59:45.535281Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435898015219643:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:45.536210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435898015219552:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.536234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:45.536244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435898015219552:2048] [node 1] ICP22 created new session: [3:7574435898015219651:2048]
2025-11-19T13:59:45.536251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435898015219651:2048] [node 1] ICS09 handshake done sender: [3:7574435898015219643:2120] self: [3:7574435898015219647:0] peer: [1:7574435898904548238:0] socket: 631 qp: -1
2025-11-19T13:59:45.536256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435898015219651:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:45.536264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:45.537992Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435899210896211:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:45.539362Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435899210896210:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:45.539506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435899210896133:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.539520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:45.539525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435899210896133:2050] [node 3] ICP22 created new session: [5:7574435899210896220:2050]
2025-11-19T13:59:45.539530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435899210896220:2050] [node 3] ICS09 handshake done sender: [5:7574435899210896211:2117] self: [5:7574435899210896212:0] peer: [3:7574435898015219628:0] socket: 630 qp: -1
2025-11-19T13:59:45.539534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435899210896220:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:45.539540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.539840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435898622247083:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32357
2025-11-19T13:59:45.539932Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435899269251681:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.541075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435899269251598:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.539993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435898904548173:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.540020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:45.540025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435898904548173:2049] [node 3] ICP22 created new session: [1:7574435898904548269:2049]
2025-11-19T13:59:45.540033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435898904548269:2049] [node 3] ICS09 handshake done sender: [1:7574435898904548251:2115] self: [1:7574435898904548238:0] peer: [3:7574435898015219647:0] socket: 621 qp: -1
2025-11-19T13:59:45.540038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435898904548269:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:45.540046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:45.540182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435898904548245:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.540188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:45.540192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435898904548245:2109] [node 4] ICP22 created new session: [1:7574435898904548271:2109]
2025-11-19T13:59:45.540196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435898904548271:2109] [node 4] ICS09 handshake done sender: [1:7574435898904548247:2114] self: [1:7574435898904548246:0] peer: [4:7574435898622247073:0] socket: 613 qp: -1
2025-11-19T13:59:45.540201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435898904548271:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:45.540207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:45.541935Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435898015219635:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.542019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435898015219599:2089] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.542027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:45.542032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435898015219599:2089] [node 5] ICP22 created new session: [3:7574435898015219653:2089]
2025-11-19T13:59:45.542037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435898015219653:2089] [node 5] ICS09 handshake done sender: [3:7574435898015219629:2114] self: [3:7574435898015219628:0] peer: [5:7574435899210896212:0] socket: 628 qp: -1
2025-11-19T13:59:45.542042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435898015219653:2089] [node 5] ICS10 traffic start
2025-11-19T13:59:45.542049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:45.542159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435898015219599:2089] [node 5] ICP18 handshake (actor [3:7574435898015219635:2117]) is too old
2025-11-19T13:59:45.542875Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435899227346735:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.543094Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435899227346734:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.543113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435899227346641:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.543194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435898622247084:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32357
2025-11-19T13:59:45.543208Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435898622247071:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:45.543684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435898622246992:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.543699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:45.543707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435898622246992:2048] [node 1] ICP22 created new session: [4:7574435898622247086:2048]
2025-11-19T13:59:45.543713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435898622247086:2048] [node 1] ICS09 handshake done sender: [4:7574435898622247071:2116] self: [4:7574435898622247073:0] peer: [1:7574435898904548246:0] socket: 629 qp: -1
2025-11-19T13:59:45.543718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435898622247086:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:45.543724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.545755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435899269251689:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56382
2025-11-19T13:59:45.545941Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435898904548243:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.545979Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435898904548257:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.545982Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435898904548257:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:45.546501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435898904548245:2109] [node 4] ICP18 handshake (actor [1:7574435898904548243:2112]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.546784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435899227346738:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58980
2025-11-19T13:59:45.547341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435899210896131:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.547362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:45.547370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435899210896131:2048] [node 1] ICP22 created new session: [5:7574435899210896222:2048]
2025-11-19T13:59:45.547376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435899210896222:2048] [node 1] ICS09 handshake done sender: [5:7574435899210896210:2116] self: [5:7574435899210896213:0] peer: [1:7574435898904548236:0] socket: 614 qp: -1
2025-11-19T13:59:45.547382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435899210896222:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:45.547400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:45.549199Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435899227346723:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.549212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435899227346723:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.549909Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435899269251689:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.549964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435899269251639:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.549524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435899210896224:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17767
2025-11-19T13:59:45.550129Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435898904548257:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:45.550544Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435898622247084:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.550716Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435899227346718:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.550956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435899227346710:2110] [node 5] ICP18 handshake (actor [2:7574435899227346718:2114]) is too old
2025-11-19T13:59:45.550831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435898015219596:2087] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.551328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435899269251691:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56382
2025-11-19T13:59:45.550824Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435898904548244:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.550838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435898904548172:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.550851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:45.550856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435898904548172:2048] [node 2] ICP22 created new session: [1:7574435898904548274:2048]
2025-11-19T13:59:45.550861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435898904548274:2048] [node 2] ICS09 handshake done sender: [1:7574435898904548257:2117] self: [1:7574435898904548237:0] peer: [2:7574435899227346737:0] socket: 647 qp: -1
2025-11-19T13:59:45.550865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435898904548274:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:45.550872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:45.551655Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435899210896216:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.551706Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435899210896209:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.551849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435898622247089:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32357
2025-11-19T13:59:45.551609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435898904548234:2107] [node 5] ICP18 handshake (actor [1:7574435898904548244:2113]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.551945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435899269251692:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56382
2025-11-19T13:59:45.552974Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435899227346723:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:45.553197Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435898622247075:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.553210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435898622247075:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.555919Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435899269251681:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:45.557083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435899269251693:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56382
2025-11-19T13:59:45.555983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435899227346713:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.556008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:45.556016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435899227346713:2111] [node 6] ICP22 created new session: [2:7574435899227346740:2111]
2025-11-19T13:59:45.556022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435899227346740:2111] [node 6] ICS09 handshake done sender: [2:7574435899227346723:2117] self: [2:7574435899227346715:0] peer: [6:7574435899269251688:0] socket: 643 qp: -1
2025-11-19T13:59:45.556028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435899227346740:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:45.556040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:45.557337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435899210896132:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.557356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:45.557361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435899210896132:2049] [node 2] ICP22 created new session: [5:7574435899210896227:2049]
2025-11-19T13:59:45.557368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435899210896227:2049] [node 2] ICS09 handshake done sender: [5:7574435899210896209:2115] self: [5:7574435899210896215:0] peer: [2:7574435899227346711:0] socket: 610 qp: -1
2025-11-19T13:59:45.557372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435899210896227:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:45.557379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:45.558271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435899210896158:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.558312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435899210896132:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:45.561418Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435898622247069:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.561422Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435898622247075:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:45.561434Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435898622247069:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:45.562526Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435898622247069:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:45.561111Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435899227346734:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:45.562057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435899227346641:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16101) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435899227346734:2118] held: no
2025-11-19T13:59:45.562086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435899227346641:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.562097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:45.562103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435899227346641:2048] [node 1] ICP22 created new session: [2:7574435899227346742:2048]
2025-11-19T13:59:45.562109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435899227346742:2048] [node 1] ICS09 handshake done sender: [2:7574435899227346734:2118] self: [2:7574435899227346737:0] peer: [1:7574435898904548237:0] socket: 649 qp: -1
2025-11-19T13:59:45.562115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435899227346742:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:45.562124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.562781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435899210896230:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17767
2025-11-19T13:59:45.562965Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435898622247089:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.562973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435898622247028:2080] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.562982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435898622247024:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.562984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:45.562991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435898622247028:2080] [node 5] ICP22 created new session: [4:7574435898622247092:2080]
2025-11-19T13:59:45.562994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:45.562997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435898622247092:2080] [node 5] ICS09 handshake done sender: [4:7574435898622247069:2114] self: [4:7574435898622247042:0] peer: [5:7574435899210896229:0] socket: 638 qp: -1
2025-11-19T13:59:45.563016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435898622247024:2077] [node 6] ICP22 created new session: [4:7574435898622247093:2077]
2025-11-19T13:59:45.563019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435898622247092:2080] [node 5] ICS10 traffic start
2025-11-19T13:59:45.563022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435898622247093:2077] [node 6] ICS09 handshake done sender: [4:7574435898622247075:2117] self: [4:7574435898622247043:0] peer: [6:7574435899269251690:0] socket: 635 qp: -1
2025-11-19T13:59:45.563026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435898622247093:2077] [node 6] ICS10 traffic start
2025-11-19T13:59:45.563027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:45.563032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:45.563176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435898622247028:2080] [node 5] ICP18 handshake (actor [4:7574435898622247089:2122]) is too old
2025-11-19T13:59:45.563721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435899269251598:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.563758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:45.563765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435899269251598:2049] [node 2] ICP22 created new session: [6:7574435899269251694:2049]
2025-11-19T13:59:45.563772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435899269251694:2049] [node 2] ICS09 handshake done sender: [6:7574435899269251681:2119] self: [6:7574435899269251688:0] peer: [2:7574435899227346715:0] socket: 648 qp: -1
2025-11-19T13:59:45.563780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435899269251694:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:45.563789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:45.563984Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435899269251692:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.565013Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435899210896216:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:45.565071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435899269251598:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:45.565118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435899269251618:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.565385Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435899269251689:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:45.565661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435899269251639:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.565679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:45.565686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435899269251639:2063] [node 4] ICP22 created new session: [6:7574435899269251697:2063]
2025-11-19T13:59:45.565691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435899269251697:2063] [node 4] ICS09 handshake done sender: [6:7574435899269251689:2120] self: [6:7574435899269251690:0] peer: [4:7574435898622247043:0] socket: 597 qp: -1
2025-11-19T13:59:45.565696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435899269251697:2063] [node 4] ICS10 traffic start
2025-11-19T13:59:45.565703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:45.566613Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435899210896217:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.566622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435899210896217:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:45.567200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435899210896158:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.567214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:45.567221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435899210896158:2072] [node 4] ICP22 created new session: [5:7574435899210896231:2072]
2025-11-19T13:59:45.567227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435899210896231:2072] [node 4] ICS09 handshake done sender: [5:7574435899210896216:2119] self: [5:7574435899210896229:0] peer: [4:7574435898622247042:0] socket: 639 qp: -1
2025-11-19T13:59:45.567232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435899210896231:2072] [node 4] ICS10 traffic start
2025-11-19T13:59:45.567239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.573124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435899269251699:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56382
2025-11-19T13:59:45.572961Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435899210896217:2120] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:45.575608Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435899269251692:2122] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:45.575928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435899269251618:2066] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:45.575945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:45.575952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435899269251618:2066] [node 5] ICP22 created new session: [6:7574435899269251700:2066]
2025-11-19T13:59:45.575957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435899269251700:2066] [node 5] ICS09 handshake done sender: [6:7574435899269251692:2122] self: [6:7574435899269251696:0] peer: [5:7574435899210896200:0] socket: 656 qp: -1
2025-11-19T13:59:45.575964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435899269251700:2066] [node 5] ICS10 traffic start
2025-11-19T13:59:45.575972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:45.575030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435899210896170:2078] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:45.575057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:45.575065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435899210896170:2078] [node 6] ICP22 created new session: [5:7574435899210896234:2078]
2025-11-19T13:59:45.575072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435899210896234:2078] [node 6] ICS09 handshake done sender: [5:7574435899210896217:2120] self: [5:7574435899210896200:0] peer: [6:7574435899269251696:0] socket: 654 qp: -1
2025-11-19T13:59:45.575080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435899210896234:2078] [node 6] ICS10 traffic start
2025-11-19T13:59:45.575091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:45.591359Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:45.591442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435899227346742:2048] [node 1] ICS01 socket: 649 reason# EndOfStream
2025-11-19T13:59:45.591452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435899227346641:2048] [node 1] ICP30 unregister session Session# [2:7574435899227346742:2048] VirtualId# [2:7574435899227346737:0]
2025-11-19T13:59:45.591458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435899227346742:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:45.591493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:45.591511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:45.596604Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:45.596703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435898015219651:2048] [node 1] ICS01 socket: 631 reason# EndOfStream
2025-11-19T13:59:45.596713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435898015219552:2048] [node 1] ICP30 unregister session Session# [3:7574435898015219651:2048] VirtualId# [3:7574435898015219647:0]
2025-11-19T13:59:45.596719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435898015219651:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:45.596764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:45.596786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:45.597171Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:45.597247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435898622247086:2048] [node 1] ICS01 socket: 629 reason# EndOfStream
2025-11-19T13:59:45.597255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435898622246992:2048] [node 1] ICP30 unregister session Session# [4:7574435898622247086:2048] VirtualId# [4:7574435898622247073:0]
2025-11-19T13:59:45.597261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435898622247086:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:45.597307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:45.597328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:45.597404Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:45.597462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435899269251684:2048] [node 1] ICS01 socket: 616 reason# EndOfStream
2025-11-19T13:59:45.597467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435899269251597:2048] [node 1] ICP30 unregister session Session# [6:7574435899269251684:2048] VirtualId# [6:7574435899269251676:0]
2025-11-19T13:59:45.597472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435899269251684:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:45.597496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:45.597517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:45.604407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:45.604506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435899210896222:2048] [node 1] ICS01 socket: 614 reason# EndOfStream
2025-11-19T13:59:45.604517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435899210896131:2048] [node 1] ICP30 unregister session Session# [5:7574435899210896222:2048] VirtualId# [5:7574435899210896213:0]
2025-11-19T13:59:45.604525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435899210896222:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:45.604586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:45.604613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:45.610443Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:45.610532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435898015219641:2049] [node 2] ICS01 socket: 593 reason# EndOfStream
2025-11-19T13:59:45.610541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435898015219553:2049] [node 2] ICP30 unregister session Session# [3:7574435898015219641:2049] VirtualId# [3:7574435898015219630:0]
2025-11-19T13:59:45.610548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435898015219641:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:45.610583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:45.610602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:45.615043Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:45.615145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435898622247077:2049] [node 2] ICS01 socket: 594 reason# EndOfStream
2025-11-19T13:59:45.615154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435898622246993:2049] [node 2] ICP30 unregister session Session# [4:7574435898622247077:2049] VirtualId# [4:7574435898622247072:0]
2025-11-19T13:59:45.615162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435898622247077:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:45.615229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:45.615257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:45.617406Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:45.617486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435899269251694:2049] [node 2] ICS01 socket: 648 reason# EndOfStream
2025-11-19T13:59:45.617494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435899269251598:2049] [node 2] ICP30 unregister session Session# [6:7574435899269251694:2049] VirtualId# [6:7574435899269251688:0]
2025-11-19T13:59:45.617500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435899269251694:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:45.617548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:45.617570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:45.617653Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:45.617714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435899210896227:2049] [node 2] ICS01 socket: 610 reason# EndOfStream
2025-11-19T13:59:45.617719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435899210896132:2049] [node 2] ICP30 unregister session Session# [5:7574435899210896227:2049] VirtualId# [5:7574435899210896215:0]
2025-11-19T13:59:45.617724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435899210896227:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:45.617751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:45.617769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:45.629676Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:45.629773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435899210896220:2050] [node 3] ICS07 socket disconnect 630 reason# EndOfStream
2025-11-19T13:59:45.629781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435899210896220:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:45.629844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:45.629868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435899210896220:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:45.630127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:45.630203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435899269251682:2050] [node 3] ICS01 socket: 591 reason# EndOfStream
2025-11-19T13:59:45.630210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435899269251599:2050] [node 3] ICP30 unregister session Session# [6:7574435899269251682:2050] VirtualId# [6:7574435899269251672:0]
2025-11-19T13:59:45.630216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435899269251682:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:45.630252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:45.630276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:45.630745Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435899210896133:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3123) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:45.630758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435899210896220:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:45.630763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435899210896133:2050] [node 3] ICP30 unregister session Session# [5:7574435899210896220:2050] VirtualId# [5:7574435899210896212:0]
2025-11-19T13:59:45.630777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:45.630783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435899210896133:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3123) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:45.630786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3123) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:45.643829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435899210896133:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:59:45.651722Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:45.651831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435899269251697:2063] [node 4] ICS01 socket: 597 reason# EndOfStream
2025-11-19T13:59:45.651840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435899269251639:2063] [node 4] ICP30 unregister session Session# [6:7574435899269251697:2063] VirtualId# [6:7574435899269251690:0]
2025-11-19T13:59:45.651850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435899269251697:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:45.651908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:45.651933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:45.783847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435899380359513:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:45.783873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435899380359514:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:45.830067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435897685751947:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:45.830085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435897685751948:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:45.856207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435896544515197:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:45.856223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435896544515198:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:45.884075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435899696844724:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:45.884090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435899696844725:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:45.884094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435899696844726:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:45.908288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435900077954755:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:45.908310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435900077954756:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:45.908316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435900077954757:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:45.922028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435899385309772:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:45.922050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435899385309773:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:45.922055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435899385309774:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:45.953396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435899380359574:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.954756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435897685752007:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60497
2025-11-19T13:59:45.954357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435899385309809:2083] [node 5] ICP01 ready to work
2025-11-19T13:59:45.956527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435897685752010:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:45.956714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435897685752011:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:45.958491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435896544515259:2106] [node 4] ICP01 ready to work
2025-11-19T13:59:45.958032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435897685752017:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.958927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435900077954817:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47676
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.960880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435900077954819:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47676
2025-11-19T13:59:45.958584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435896544515260:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:45.959528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435896544515261:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7540
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.968218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435897685752025:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60497
2025-11-19T13:59:45.969188Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435896544515261:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.969744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435899380359583:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:45.973225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435896544515198:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17374303ns) {Configure:104(17373215ns) {ProcessConfigured:135(16976111ns) {StartInitialHandshake:157(16975255ns) {DropHandshakes:469(844ns) {DropIncomingHandshake:441(126ns) DropOutgoingHandshake:456(107ns)} GenerateSessionVirtualId:988(295ns) PrepareNewSessionHandshake:476(814ns) {SwitchToState:178(302ns)}}}}}
2025-11-19T13:59:45.973284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435896544515198:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.974115Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435900077954819:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.974215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435899380359584:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:45.974354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435899696844786:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:45.975864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435899385309822:2087] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.979709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435899385309838:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28738
2025-11-19T13:59:45.979492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435899380359590:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60171
2025-11-19T13:59:45.979550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435899380359591:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60171
2025-11-19T13:59:45.979683Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435899380359514:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18974373ns) {Configure:104(18973477ns) {ProcessConfigured:135(18576733ns) {StartInitialHandshake:157(18576181ns) {DropHandshakes:469(457ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:476(795ns) {SwitchToState:178(130ns)}}}}}
2025-11-19T13:59:45.979854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435899696844789:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:45.981318Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435899380359590:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.981650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435900077954823:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:45.981889Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435899380359578:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.981895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435899380359578:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:45.981718Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435897685752007:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.981770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435897685751947:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.984434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435899385309839:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28738
2025-11-19T13:59:45.984926Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435899385309774:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(23889903ns) {Configure:104(23888602ns) {ProcessConfigured:135(23522339ns) {StartInitialHandshake:157(23521891ns) {DropHandshakes:469(627ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(92ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:476(829ns) {SwitchToState:178(45ns)}}}}}
2025-11-19T13:59:45.982349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435899696844793:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56680
2025-11-19T13:59:45.982407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435899696844794:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56680
2025-11-19T13:59:45.982436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435899696844795:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56680
2025-11-19T13:59:45.985805Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435897685752016:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.985822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435897685752016:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:45.986323Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435897685752025:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.986084Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435899696844794:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.986113Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435899696844795:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.986603Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435899385309839:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.987967Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435899380359591:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.988156Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435899385309838:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.988544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435899385309772:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.988550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435896544515275:2114] [node 6] ICP01 ready to work
2025-11-19T13:59:45.992076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435899696844786:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17652380ns) {Configure:104(17651830ns) {ProcessConfigured:135(17254973ns) {StartInitialHandshake:157(17254431ns) {DropHandshakes:469(528ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(140ns)} GenerateSessionVirtualId:988(76ns) PrepareNewSessionHandshake:476(1033ns) {SwitchToState:178(437ns)}}}}}
2025-11-19T13:59:45.992579Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435899696844793:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.992664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435899696844724:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:45.995759Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435897685752020:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:45.995773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435897685752020:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:45.995389Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435899385309822:2087] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19436914ns) {Configure:104(19436036ns) {ProcessConfigured:135(19090267ns) {StartInitialHandshake:157(19089766ns) {DropHandshakes:469(650ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:476(998ns) {SwitchToState:178(174ns)}}}}}
2025-11-19T13:59:45.995428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435899385309822:2087] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:45.999847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435900077954755:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18142893ns) {Configure:104(18141946ns) {ProcessConfigured:135(17812170ns) {StartInitialHandshake:157(17811740ns) {DropHandshakes:469(555ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(99ns)} GenerateSessionVirtualId:988(138ns) PrepareNewSessionHandshake:476(753ns) {SwitchToState:178(314ns)}}}}}
2025-11-19T13:59:46.000486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435903675326891:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60171
2025-11-19T13:59:46.000541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435903675326892:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60171
2025-11-19T13:59:46.004729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435896544515267:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.004757Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435896544515267:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.004821Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435896544515265:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.004839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435896544515265:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.004898Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435899380359576:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.004916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435899380359576:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.007032Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435899380359578:2109] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:46.007457Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435899380359588:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.007476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435899380359588:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.007530Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435899696844791:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.007545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435899696844791:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.005226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435900077954757:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.005282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435900077954757:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(31090280ns) {Configure:104(31089503ns) {ProcessConfigured:135(30735579ns) {IncomingHandshake:261(30720147ns) {DropIncomingHandshake:441(74ns) PrepareNewSessionHandshake:476(227ns) {SwitchToState:178(75ns)} GenerateSessionVirtualId:988(570ns) IssueIncomingHandshakeReply:193(30714701ns) {SwitchToState:178(760ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.009243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435901980719327:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60497
2025-11-19T13:59:46.009291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435901980719328:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60497
2025-11-19T13:59:46.009569Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435897685752020:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:46.008430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435903991812100:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56680
2025-11-19T13:59:46.008550Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435896544515275:2114] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19946272ns) {Configure:104(19945586ns) {ProcessConfigured:135(19635612ns) {StartInitialHandshake:157(19635137ns) {DropHandshakes:469(578ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(140ns)} GenerateSessionVirtualId:988(99ns) PrepareNewSessionHandshake:476(1046ns) {SwitchToState:178(203ns)}}}}}
2025-11-19T13:59:46.010189Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435900077954817:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.010534Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435901980719327:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.010549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435897685752017:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.010561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:46.010568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435897685752017:2111] [node 4] ICP22 created new session: [2:7574435901980719329:2111]
2025-11-19T13:59:46.010573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435901980719329:2111] [node 4] ICS09 handshake done sender: [2:7574435897685752020:2113] self: [2:7574435897685752019:0] peer: [4:7574435899696844796:0] socket: 601 qp: -1
2025-11-19T13:59:46.010581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435901980719329:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:46.010589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:46.010822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435899696844725:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.010875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435899696844725:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24700596ns) {Configure:104(24699974ns) {ProcessConfigured:135(24350387ns) {IncomingHandshake:261(24340369ns) {DropIncomingHandshake:441(74ns) PrepareNewSessionHandshake:476(365ns) {SwitchToState:178(137ns)} GenerateSessionVirtualId:988(363ns) IssueIncomingHandshakeReply:193(24336744ns) {SwitchToState:178(963ns)}}}}}
2025-11-19T13:59:46.010932Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435903991812100:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.011797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435899696844726:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.011843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435899696844726:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(23836617ns) {Configure:104(23835666ns) {ProcessConfigured:135(23448326ns) {IncomingHandshake:261(23436593ns) {DropIncomingHandshake:441(58ns) PrepareNewSessionHandshake:476(374ns) {SwitchToState:178(128ns)} GenerateSessionVirtualId:988(924ns) IssueIncomingHandshakeReply:193(23430696ns) {SwitchToState:178(1017ns)}}}}}
2025-11-19T13:59:46.012175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435900077954829:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:46.014510Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435899380359583:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(40218431ns) {Configure:104(40217167ns) {ProcessConfigured:135(39842940ns) {StartInitialHandshake:157(39842430ns) {DropHandshakes:469(1275ns) {DropIncomingHandshake:441(127ns) DropOutgoingHandshake:456(769ns)} GenerateSessionVirtualId:988(284ns) PrepareNewSessionHandshake:476(980ns) {SwitchToState:178(68ns)}}}}}
2025-11-19T13:59:46.014533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435899380359513:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.014553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:46.014560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435899380359513:2048] [node 2] ICP22 created new session: [1:7574435903675326895:2048]
2025-11-19T13:59:46.014565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435903675326895:2048] [node 2] ICS09 handshake done sender: [1:7574435899380359578:2109] self: [1:7574435899380359577:0] peer: [2:7574435897685752028:0] socket: 595 qp: -1
2025-11-19T13:59:46.014575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435903675326895:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:46.014594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:46.014815Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435903675326891:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.015572Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435896544515267:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:46.017098Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435896544515265:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:46.017199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435896544515260:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.017212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:46.017220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435896544515260:2107] [node 5] ICP22 created new session: [3:7574435900839482576:2107]
2025-11-19T13:59:46.017226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435900839482576:2107] [node 5] ICS09 handshake done sender: [3:7574435896544515267:2111] self: [3:7574435896544515266:0] peer: [5:7574435900077954821:0] socket: 607 qp: -1
2025-11-19T13:59:46.017233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435900839482576:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:46.017255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:46.017371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435896544515259:2106] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.017377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:46.017381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435896544515259:2106] [node 4] ICP22 created new session: [3:7574435900839482578:2106]
2025-11-19T13:59:46.017386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435900839482578:2106] [node 4] ICS09 handshake done sender: [3:7574435896544515265:2110] self: [3:7574435896544515264:0] peer: [4:7574435899696844797:0] socket: 606 qp: -1
2025-11-19T13:59:46.017390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435900839482578:2106] [node 4] ICS10 traffic start
2025-11-19T13:59:46.017395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:46.022944Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435899380359576:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:46.023413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435899380359574:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.023432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:46.023441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435899380359574:2107] [node 4] ICP22 created new session: [1:7574435903675326900:2107]
2025-11-19T13:59:46.023448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435903675326900:2107] [node 4] ICS09 handshake done sender: [1:7574435899380359576:2108] self: [1:7574435899380359575:0] peer: [4:7574435899696844801:0] socket: 594 qp: -1
2025-11-19T13:59:46.023454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435903675326900:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:46.023477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:46.025876Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435903675326892:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.026277Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435899380359588:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:46.026384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435899380359584:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.026402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:46.026409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435899380359584:2111] [node 6] ICP22 created new session: [1:7574435903675326905:2111]
2025-11-19T13:59:46.026414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435903675326905:2111] [node 6] ICS09 handshake done sender: [1:7574435899380359588:2112] self: [1:7574435899380359587:0] peer: [6:7574435899385309845:0] socket: 611 qp: -1
2025-11-19T13:59:46.026419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435903675326905:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:46.026428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:46.027836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435899696844724:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3051) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435899696844793:2111] held: no
2025-11-19T13:59:46.028853Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435899696844791:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.032143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435899696844789:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.032162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:46.032171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435899696844789:2109] [node 6] ICP22 created new session: [4:7574435903991812103:2109]
2025-11-19T13:59:46.032178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435903991812103:2109] [node 6] ICS09 handshake done sender: [4:7574435899696844791:2110] self: [4:7574435899696844790:0] peer: [6:7574435899385309850:0] socket: 614 qp: -1
2025-11-19T13:59:46.032186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435903991812103:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:46.032195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:46.032490Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435897685752007:2107] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.036319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435900077954829:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(24056102ns) {Configure:104(24055540ns) {ProcessConfigured:135(23711763ns) {StartInitialHandshake:157(23687378ns) {DropHandshakes:469(681ns) {DropIncomingHandshake:441(159ns) DropOutgoingHandshake:456(114ns)} GenerateSessionVirtualId:988(452ns) PrepareNewSessionHandshake:476(1220ns) {SwitchToState:178(750ns)}} IncomingHandshake:261(8506ns) {DropIncomingHandshake:441(47ns) PrepareNewSessionHandshake:476(191ns) {SwitchToState:178(154ns)} GenerateSessionVirtualId:988(247ns) IssueIncomingHandshakeReply:193(329ns)}}}}
2025-11-19T13:59:46.036082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435897685751947:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3051) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435897685752007:2107] held: no
2025-11-19T13:59:46.036112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435897685751947:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.036123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:46.036130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435897685751947:2048] [node 1] ICP22 created new session: [2:7574435901980719332:2048]
2025-11-19T13:59:46.036136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435901980719332:2048] [node 1] ICS09 handshake done sender: [2:7574435897685752007:2107] self: [2:7574435897685752028:0] peer: [1:7574435899380359577:0] socket: 596 qp: -1
2025-11-19T13:59:46.036143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435901980719332:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:46.036150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:46.036510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435901980719334:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60497
2025-11-19T13:59:46.037112Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435897685752016:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:46.037218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435897685751948:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.037231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:46.037236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435897685751948:2049] [node 3] ICP22 created new session: [2:7574435901980719338:2049]
2025-11-19T13:59:46.037241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435901980719338:2049] [node 3] ICS09 handshake done sender: [2:7574435897685752016:2112] self: [2:7574435897685752015:0] peer: [3:7574435896544515273:0] socket: 600 qp: -1
2025-11-19T13:59:46.037246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435901980719338:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:46.037250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:46.040517Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435901980719334:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.040551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435897685751948:2049] [node 3] ICP18 handshake (actor [2:7574435901980719334:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.044257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435900839482583:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7540
2025-11-19T13:59:46.044335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435900839482584:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7540
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.044538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904372922131:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47676
2025-11-19T13:59:46.044593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904372922132:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47676
2025-11-19T13:59:46.044824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904372922133:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47676
2025-11-19T13:59:46.044875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904372922134:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47676
2025-11-19T13:59:46.048122Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435904372922133:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.048167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435900077954755:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.048181Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435904372922134:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.048217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435900077954756:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.048403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435900839482582:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7540
2025-11-19T13:59:46.048534Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435900839482583:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.048550Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435896544515198:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435896544515261:2108] held: no
2025-11-19T13:59:46.048601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435896544515197:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.051101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435903991812105:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56680
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.050695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903680277148:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28738
2025-11-19T13:59:46.050779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903680277149:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28738
2025-11-19T13:59:46.051166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903680277150:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28738
2025-11-19T13:59:46.051552Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435900077954819:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:46.051668Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435896544515261:2108] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.051852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903680277151:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28738
2025-11-19T13:59:46.052200Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435903680277151:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.055934Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435900839482582:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.056210Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435904372922131:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.056233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435900077954757:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.056247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:46.056258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435900077954823:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.056262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435900077954757:2050] [node 3] ICP22 created new session: [5:7574435904372922137:2050]
2025-11-19T13:59:46.056269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435904372922137:2050] [node 3] ICS09 handshake done sender: [5:7574435900077954819:2109] self: [5:7574435900077954821:0] peer: [3:7574435896544515266:0] socket: 608 qp: -1
2025-11-19T13:59:46.056278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435904372922137:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:46.056301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:46.056580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435896544515198:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.056594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:46.056600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435896544515198:2049] [node 2] ICP22 created new session: [3:7574435900839482587:2049]
2025-11-19T13:59:46.056606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435900839482587:2049] [node 2] ICS09 handshake done sender: [3:7574435896544515261:2108] self: [3:7574435896544515273:0] peer: [2:7574435897685752015:0] socket: 602 qp: -1
2025-11-19T13:59:46.056613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435900839482587:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:46.056621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:46.056530Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435897685752017:2111] [node 4] ICP35 event processing took too much time PendingConnection:278(45943538ns) {HandleHandshakeStatus:329(45942484ns) {DropIncomingHandshake:441(175ns) DropOutgoingHandshake:456(211ns) SwitchToState:178(132ns) ProcessPendingSessionEvents:495(28050896ns) {ForwardSessionEventToSession:612(28034215ns) ForwardSessionEventToSession:612(2682ns) ForwardSessionEventToSession:612(996ns) ForwardSessionEventToSession:612(1093ns) ForwardSessionEventToSession:612(909ns) ForwardSessionEventToSession:612(1310ns) ForwardSessionEventToSession:612(795ns) ForwardSessionEventToSession:612(560ns) ForwardSessionEventToSession:612(693ns) ForwardSessionEventToSession:612(1235ns)}}}
2025-11-19T13:59:46.056766Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435897685752024:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.056773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435897685752024:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.056336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435903991812106:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56680
2025-11-19T13:59:46.056678Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435903675326894:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.056698Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435903675326894:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.057114Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435899380359592:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.057124Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435899380359592:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.059873Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435903680277148:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.059890Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435899385309839:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:46.059913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435899385309773:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.059932Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435899385309838:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:46.060903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435899696844789:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(28731440ns) {HandleHandshakeStatus:329(28730462ns) {DropIncomingHandshake:441(183ns) DropOutgoingHandshake:456(334ns) SwitchToState:178(646ns) ProcessPendingSessionEvents:495(12926566ns) {ForwardSessionEventToSession:612(12912524ns) ForwardSessionEventToSession:612(2472ns) ForwardSessionEventToSession:612(1236ns) ForwardSessionEventToSession:612(583ns) ForwardSessionEventToSession:612(856ns) ForwardSessionEventToSession:612(1102ns) ForwardSessionEventToSession:612(547ns) ForwardSessionEventToSession:612(735ns) ForwardSessionEventToSession:612(570ns) ForwardSessionEventToSession:612(1360ns)}}}
2025-11-19T13:59:46.060961Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435899696844798:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.060968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435899696844798:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.059950Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435897685752027:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.059962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435897685752027:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.061769Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435897685752027:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.062103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435899385309772:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.062120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:46.062127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435899385309772:2048] [node 1] ICP22 created new session: [6:7574435903680277154:2048]
2025-11-19T13:59:46.062133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435903680277154:2048] [node 1] ICS09 handshake done sender: [6:7574435899385309838:2107] self: [6:7574435899385309845:0] peer: [1:7574435899380359587:0] socket: 613 qp: -1
2025-11-19T13:59:46.062138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435903680277154:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:46.062146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:46.062334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435899385309774:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.062363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903680277153:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28738
2025-11-19T13:59:46.062618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435899385309822:2087] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.062631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:46.062636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435899385309822:2087] [node 4] ICP22 created new session: [6:7574435903680277157:2087]
2025-11-19T13:59:46.062640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435903680277157:2087] [node 4] ICS09 handshake done sender: [6:7574435899385309839:2108] self: [6:7574435899385309850:0] peer: [4:7574435899696844790:0] socket: 618 qp: -1
2025-11-19T13:59:46.062645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435903680277157:2087] [node 4] ICS10 traffic start
2025-11-19T13:59:46.062658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:46.063033Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435900839482573:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.063046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435900839482573:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.063860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435897685752010:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.063884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:46.063890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435897685752010:2108] [node 6] ICP22 created new session: [2:7574435901980719341:2108]
2025-11-19T13:59:46.063896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435901980719341:2108] [node 6] ICS09 handshake done sender: [2:7574435897685752027:2116] self: [2:7574435897685752026:0] peer: [6:7574435903680277152:0] socket: 646 qp: -1
2025-11-19T13:59:46.063902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435901980719341:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:46.063909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:46.064448Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435903680277151:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:46.061604Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435899696844794:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.065296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903680277159:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28738
2025-11-19T13:59:46.065314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435899385309773:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435903680277151:2119] held: no
2025-11-19T13:59:46.065337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435899385309773:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.065346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:46.065351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435899385309773:2049] [node 2] ICP22 created new session: [6:7574435903680277160:2049]
2025-11-19T13:59:46.065356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435903680277160:2049] [node 2] ICS09 handshake done sender: [6:7574435903680277151:2119] self: [6:7574435903680277152:0] peer: [2:7574435897685752026:0] socket: 652 qp: -1
2025-11-19T13:59:46.065362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435903680277160:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:46.065367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:46.065694Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435900839482573:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:46.065803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435896544515275:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.065815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:46.065819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435896544515275:2114] [node 6] ICP22 created new session: [3:7574435900839482590:2114]
2025-11-19T13:59:46.065824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435900839482590:2114] [node 6] ICS09 handshake done sender: [3:7574435900839482573:2115] self: [3:7574435896544515276:0] peer: [6:7574435903680277156:0] socket: 636 qp: -1
2025-11-19T13:59:46.065828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435900839482590:2114] [node 6] ICS10 traffic start
2025-11-19T13:59:46.065834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:46.066112Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435903680277148:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.066270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435899385309774:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.066286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:46.066293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435899385309774:2050] [node 3] ICP22 created new session: [6:7574435903680277162:2050]
2025-11-19T13:59:46.066298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435903680277162:2050] [node 3] ICS09 handshake done sender: [6:7574435903680277148:2116] self: [6:7574435903680277156:0] peer: [3:7574435896544515276:0] socket: 637 qp: -1
2025-11-19T13:59:46.066304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435903680277162:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:46.066311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:46.066518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904372922140:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47676
2025-11-19T13:59:46.066540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904372922141:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47676
2025-11-19T13:59:46.065352Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435899696844795:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.069209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435903991812108:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56680
2025-11-19T13:59:46.069600Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435899696844798:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:46.070204Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435904372922131:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:46.070164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435899696844726:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.070182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:46.070189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435899696844726:2050] [node 3] ICP22 created new session: [4:7574435903991812109:2050]
2025-11-19T13:59:46.070196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435903991812109:2050] [node 3] ICS09 handshake done sender: [4:7574435899696844795:2113] self: [4:7574435899696844797:0] peer: [3:7574435896544515264:0] socket: 616 qp: -1
2025-11-19T13:59:46.070203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435903991812109:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:46.070210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:46.070348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435899696844786:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.070360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:46.070365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435899696844786:2108] [node 5] ICP22 created new session: [4:7574435903991812111:2108]
2025-11-19T13:59:46.070371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435903991812111:2108] [node 5] ICS09 handshake done sender: [4:7574435899696844798:2114] self: [4:7574435899696844787:0] peer: [5:7574435904372922138:0] socket: 622 qp: -1
2025-11-19T13:59:46.070375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435903991812111:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:46.070381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:46.070345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435900077954823:2111] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.070363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:46.070370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435900077954823:2111] [node 4] ICP22 created new session: [5:7574435904372922142:2111]
2025-11-19T13:59:46.070375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435904372922142:2111] [node 4] ICS09 handshake done sender: [5:7574435904372922131:2116] self: [5:7574435904372922138:0] peer: [4:7574435899696844787:0] socket: 626 qp: -1
2025-11-19T13:59:46.070381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435904372922142:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:46.070391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:46.072100Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435903675326894:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:46.072178Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435899380359592:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:46.072604Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435897685752024:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:46.072732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435899380359514:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.072756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:46.072763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435899380359514:2049] [node 3] ICP22 created new session: [1:7574435903675326909:2049]
2025-11-19T13:59:46.072768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435903675326909:2049] [node 3] ICS09 handshake done sender: [1:7574435899380359592:2115] self: [1:7574435899380359585:0] peer: [3:7574435900839482585:0] socket: 639 qp: -1
2025-11-19T13:59:46.072773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435903675326909:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:46.072782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:46.073088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435897685752011:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.073108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:46.073114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435897685752011:2109] [node 5] ICP22 created new session: [2:7574435901980719344:2109]
2025-11-19T13:59:46.073118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435901980719344:2109] [node 5] ICS09 handshake done sender: [2:7574435897685752024:2114] self: [2:7574435897685752023:0] peer: [5:7574435904372922136:0] socket: 645 qp: -1
2025-11-19T13:59:46.073123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435901980719344:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:46.073130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:46.073411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435899380359583:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.073425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:46.073430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435899380359583:2110] [node 5] ICP22 created new session: [1:7574435903675326911:2110]
2025-11-19T13:59:46.073434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435903675326911:2110] [node 5] ICS09 handshake done sender: [1:7574435903675326894:2118] self: [1:7574435899380359586:0] peer: [5:7574435904372922135:0] socket: 628 qp: -1
2025-11-19T13:59:46.073437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435903675326911:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:46.073443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:46.076070Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435899696844793:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:46.080838Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435904372922133:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:46.081449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435896544515197:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3051) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435900839482583:2117] held: no
2025-11-19T13:59:46.081154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435899696844725:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.081178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:46.081187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435899696844725:2049] [node 2] ICP22 created new session: [4:7574435903991812113:2049]
2025-11-19T13:59:46.081195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435903991812113:2049] [node 2] ICS09 handshake done sender: [4:7574435899696844794:2112] self: [4:7574435899696844796:0] peer: [2:7574435897685752019:0] socket: 615 qp: -1
2025-11-19T13:59:46.081203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435903991812113:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:46.081213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.085588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435899696844724:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.085609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:46.085614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435899696844724:2048] [node 1] ICP22 created new session: [4:7574435903991812115:2048]
2025-11-19T13:59:46.085620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435903991812115:2048] [node 1] ICS09 handshake done sender: [4:7574435899696844793:2111] self: [4:7574435899696844801:0] peer: [1:7574435899380359575:0] socket: 598 qp: -1
2025-11-19T13:59:46.085634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435903991812115:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:46.085641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:46.085469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435900839482592:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7540
2025-11-19T13:59:46.086301Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435900839482583:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:46.086407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435896544515197:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.086417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:46.086423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435896544515197:2048] [node 1] ICP22 created new session: [3:7574435900839482593:2048]
2025-11-19T13:59:46.086429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435900839482593:2048] [node 1] ICS09 handshake done sender: [3:7574435900839482583:2117] self: [3:7574435900839482585:0] peer: [1:7574435899380359585:0] socket: 641 qp: -1
2025-11-19T13:59:46.086434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435900839482593:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:46.086441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:46.087239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435900077954755:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.087261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:46.087269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435900077954755:2048] [node 1] ICP22 created new session: [5:7574435904372922145:2048]
2025-11-19T13:59:46.087276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435904372922145:2048] [node 1] ICS09 handshake done sender: [5:7574435904372922133:2118] self: [5:7574435904372922135:0] peer: [1:7574435899380359586:0] socket: 648 qp: -1
2025-11-19T13:59:46.087283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435904372922145:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:46.087293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.088096Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435900077954756:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20972) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435904372922134:2119] held: no
2025-11-19T13:59:46.088259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904372922144:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47676
2025-11-19T13:59:46.089601Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435904372922134:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:46.089717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435900077954756:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.089741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:46.089747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435900077954756:2049] [node 2] ICP22 created new session: [5:7574435904372922148:2049]
2025-11-19T13:59:46.089753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435904372922148:2049] [node 2] ICS09 handshake done sender: [5:7574435904372922134:2119] self: [5:7574435904372922136:0] peer: [2:7574435897685752023:0] socket: 649 qp: -1
2025-11-19T13:59:46.089759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435904372922148:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:46.089779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.092105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903680277164:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28738
2025-11-19T13:59:46.092183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435900077954755:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:46.092736Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435903680277164:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.092807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435899385309809:2083] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.095979Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435904372922129:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.095992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435904372922129:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.097131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903680277166:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28738
2025-11-19T13:59:46.097279Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435904372922129:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:46.097486Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435903680277164:2122] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:46.097577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435899385309809:2083] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.097604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:46.097611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435899385309809:2083] [node 5] ICP22 created new session: [6:7574435903680277167:2083]
2025-11-19T13:59:46.097618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435903680277167:2083] [node 5] ICS09 handshake done sender: [6:7574435903680277164:2122] self: [6:7574435903680277165:0] peer: [5:7574435904372922128:0] socket: 604 qp: -1
2025-11-19T13:59:46.097624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435903680277167:2083] [node 5] ICS10 traffic start
2025-11-19T13:59:46.097632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:46.098846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435900077954829:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.098867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:46.098873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435900077954829:2113] [node 6] ICP22 created new session: [5:7574435904372922151:2113]
2025-11-19T13:59:46.098879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435904372922151:2113] [node 6] ICS09 handshake done sender: [5:7574435904372922129:2115] self: [5:7574435904372922128:0] peer: [6:7574435903680277165:0] socket: 609 qp: -1
2025-11-19T13:59:46.098884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435904372922151:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:46.098892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:46.126778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:46.126868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435901980719332:2048] [node 1] ICS01 socket: 596 reason# EndOfStream
2025-11-19T13:59:46.126877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435897685751947:2048] [node 1] ICP30 unregister session Session# [2:7574435901980719332:2048] VirtualId# [2:7574435897685752028:0]
2025-11-19T13:59:46.126885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435901980719332:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.126917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:46.126936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:46.127347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:46.127409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435900839482593:2048] [node 1] ICS01 socket: 641 reason# EndOfStream
2025-11-19T13:59:46.127415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435896544515197:2048] [node 1] ICP30 unregister session Session# [3:7574435900839482593:2048] VirtualId# [3:7574435900839482585:0]
2025-11-19T13:59:46.127419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435900839482593:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.127444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:46.127459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:46.144090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:46.144188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435904372922145:2048] [node 1] ICS07 socket disconnect 648 reason# EndOfStream
2025-11-19T13:59:46.144195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435904372922145:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.144253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:46.144278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435904372922145:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:46.144761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435900077954755:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3051) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:46.144773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435904372922145:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:46.144779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435900077954755:2048] [node 1] ICP30 unregister session Session# [5:7574435904372922145:2048] VirtualId# [5:7574435904372922135:0]
2025-11-19T13:59:46.144794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:46.144799Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435900077954755:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3051) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:46.144803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3051) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:46.150757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:46.150819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435903680277154:2048] [node 1] ICS01 socket: 613 reason# EndOfStream
2025-11-19T13:59:46.150824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435899385309772:2048] [node 1] ICP30 unregister session Session# [6:7574435903680277154:2048] VirtualId# [6:7574435899385309845:0]
2025-11-19T13:59:46.150828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435903680277154:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.150853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:46.150874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:46.152861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:46.152935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435900839482587:2049] [node 2] ICS01 socket: 602 reason# EndOfStream
2025-11-19T13:59:46.152942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435896544515198:2049] [node 2] ICP30 unregister session Session# [3:7574435900839482587:2049] VirtualId# [3:7574435896544515273:0]
2025-11-19T13:59:46.152948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435900839482587:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.152976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:46.152992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:46.155127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435900077954755:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:46.150545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:46.150633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435903991812115:2048] [node 1] ICS01 socket: 598 reason# EndOfStream
2025-11-19T13:59:46.150641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435899696844724:2048] [node 1] ICP30 unregister session Session# [4:7574435903991812115:2048] VirtualId# [4:7574435899696844801:0]
2025-11-19T13:59:46.150649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435903991812115:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.150695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:46.150721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:46.160188Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:46.160280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435904372922148:2049] [node 2] ICS01 socket: 649 reason# EndOfStream
2025-11-19T13:59:46.160289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435900077954756:2049] [node 2] ICP30 unregister session Session# [5:7574435904372922148:2049] VirtualId# [5:7574435904372922136:0]
2025-11-19T13:59:46.160296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435904372922148:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.160351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:46.160379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:46.160880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:46.160940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435903680277160:2049] [node 2] ICS01 socket: 652 reason# EndOfStream
2025-11-19T13:59:46.160945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435899385309773:2049] [node 2] ICP30 unregister session Session# [6:7574435903680277160:2049] VirtualId# [6:7574435903680277152:0]
2025-11-19T13:59:46.160950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435903680277160:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.160977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:46.160996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:46.160421Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:46.160486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435903991812113:2049] [node 2] ICS07 socket disconnect 615 reason# EndOfStream
2025-11-19T13:59:46.160490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435903991812113:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.160518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:46.160538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435903991812113:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:46.167628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435899696844725:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20972) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:46.167648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435903991812113:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:46.167653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435899696844725:2049] [node 2] ICP30 unregister session Session# [4:7574435903991812113:2049] VirtualId# [4:7574435899696844796:0]
2025-11-19T13:59:46.167671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:46.167677Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435899696844725:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20972) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:46.167680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20972) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:46.183891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:46.183993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435904372922137:2050] [node 3] ICS01 socket: 608 reason# EndOfStream
2025-11-19T13:59:46.184003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435900077954757:2050] [node 3] ICP30 unregister session Session# [5:7574435904372922137:2050] VirtualId# [5:7574435900077954821:0]
2025-11-19T13:59:46.184010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435904372922137:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.184082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:46.184112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:46.184151Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:46.184220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435903680277162:2050] [node 3] ICS01 socket: 637 reason# EndOfStream
2025-11-19T13:59:46.184225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435899385309774:2050] [node 3] ICP30 unregister session Session# [6:7574435903680277162:2050] VirtualId# [6:7574435903680277156:0]
2025-11-19T13:59:46.184230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435903680277162:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.184259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:46.184282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:46.203919Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:46.204016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435903680277157:2087] [node 4] ICS01 socket: 618 reason# EndOfStream
2025-11-19T13:59:46.204025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435899385309822:2087] [node 4] ICP30 unregister session Session# [6:7574435903680277157:2087] VirtualId# [6:7574435899385309850:0]
2025-11-19T13:59:46.204032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435903680277157:2087] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.204091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:46.204117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:46.455412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435901234709024:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:46.455430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435901234709025:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:46.512863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435902273082891:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:46.512888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435902273082892:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:46.551548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435901931213206:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:46.551565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435901931213207:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:46.591878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435903453501977:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:46.591894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435903453501978:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:46.591901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435903453501979:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:46.636199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435904488144017:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:46.636216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435904488144018:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:46.636222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435904488144019:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:46.669099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435903041847033:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:46.669113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435903041847034:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:46.669117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435903041847035:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:46.720160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435901931213241:2080] [node 5] ICP01 ready to work
2025-11-19T13:59:46.720370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435901931213245:2084] [node 6] ICP01 ready to work
2025-11-19T13:59:46.720178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435901234709085:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:46.721513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435901234709091:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:46.723948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435904488144038:2065] [node 4] ICP01 ready to work
2025-11-19T13:59:46.724306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435901931213247:2077] [node 4] ICP01 ready to work
2025-11-19T13:59:46.726783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435901234709090:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.729752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435902273082954:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57900
2025-11-19T13:59:46.730891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435902273082959:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.731994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435903453502019:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64241
2025-11-19T13:59:46.731952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435901931213262:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62258
2025-11-19T13:59:46.733865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435902273082945:2098] [node 6] ICP01 ready to work
2025-11-19T13:59:46.736743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435903453502037:2088] [node 6] ICP01 ready to work
2025-11-19T13:59:46.736087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435901234709094:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58483
2025-11-19T13:59:46.736291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435903041847063:2072] [node 4] ICP01 ready to work
2025-11-19T13:59:46.740419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435904488144038:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16398465ns) {Configure:104(16397529ns) {ProcessConfigured:135(16067195ns) {StartInitialHandshake:157(16066431ns) {DropHandshakes:469(673ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(62ns) PrepareNewSessionHandshake:476(788ns) {SwitchToState:178(619ns)}}}}}
2025-11-19T13:59:46.741694Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435901931213247:2077] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(17332059ns) {Configure:104(17317202ns) {ProcessConfigured:135(17043694ns) {StartInitialHandshake:157(17043531ns) {DropHandshakes:469(513ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(179ns) PrepareNewSessionHandshake:476(973ns) {SwitchToState:178(635ns)}}}}}
2025-11-19T13:59:46.741075Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435903041847033:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17033631ns) {Configure:104(17032825ns) {ProcessConfigured:135(16632511ns) {StartInitialHandshake:157(16632092ns) {DropHandshakes:469(507ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(184ns) PrepareNewSessionHandshake:476(860ns) {SwitchToState:178(452ns)}}}}}
2025-11-19T13:59:46.747838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435904488144048:2063] [node 6] ICP01 ready to work
2025-11-19T13:59:46.747912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435903453502013:2074] [node 5] ICP01 ready to work
2025-11-19T13:59:46.750905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435903453501979:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17439317ns) {Configure:104(17438526ns) {ProcessConfigured:135(17133922ns) {StartInitialHandshake:157(17133295ns) {DropHandshakes:469(515ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:476(635ns) {SwitchToState:178(284ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.751145Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435901234709085:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(30886660ns) {Configure:104(30885920ns) {ProcessConfigured:135(30431708ns) {StartInitialHandshake:157(30431249ns) {DropHandshakes:469(731ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(40ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:476(1063ns) {SwitchToState:178(94ns)}}}}}
2025-11-19T13:59:46.751535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435902273082945:2098] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17613934ns) {Configure:104(17613053ns) {ProcessConfigured:135(17382147ns) {StartInitialHandshake:157(17381771ns) {DropHandshakes:469(267ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(242ns) PrepareNewSessionHandshake:476(860ns) {SwitchToState:178(170ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.751391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904488144093:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51046
2025-11-19T13:59:46.751702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435902273082969:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57900
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.752147Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435901234709025:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(31402945ns) {Configure:104(31400784ns) {ProcessConfigured:135(31145429ns) {StartInitialHandshake:157(31144911ns) {DropHandshakes:469(222ns) {DropIncomingHandshake:441(33ns) DropOutgoingHandshake:456(44ns)} GenerateSessionVirtualId:988(143ns) PrepareNewSessionHandshake:476(1148ns) {SwitchToState:178(802ns)}}}}}
2025-11-19T13:59:46.752331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435901234709103:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58483
2025-11-19T13:59:46.752342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435901234709104:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58483
2025-11-19T13:59:46.752647Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435901931213206:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(26251136ns) {Configure:104(16942266ns) {ProcessConfigured:135(16579277ns) {StartInitialHandshake:157(16579079ns) {DropHandshakes:469(396ns) {DropIncomingHandshake:441(82ns) DropOutgoingHandshake:456(29ns)} GenerateSessionVirtualId:988(276ns) PrepareNewSessionHandshake:476(626ns) {SwitchToState:178(456ns)}}}}}
2025-11-19T13:59:46.752772Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435901931213262:2095] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.752824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435901931213207:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.753191Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435904488144093:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.753265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435904488144019:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.753375Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435901234709104:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.753502Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435901931213259:2093] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.753509Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435901931213259:2093] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.754220Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435901931213259:2093] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.754326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904488144095:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51046
2025-11-19T13:59:46.754458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435901931213241:2080] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.754478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:46.754485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435901931213241:2080] [node 5] ICP22 created new session: [3:7574435901931213288:2080]
2025-11-19T13:59:46.754492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435901931213288:2080] [node 5] ICS09 handshake done sender: [3:7574435901931213259:2093] self: [3:7574435901931213249:0] peer: [5:7574435904488144094:0] socket: 603 qp: -1
2025-11-19T13:59:46.754499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435901931213288:2080] [node 5] ICS10 traffic start
2025-11-19T13:59:46.754521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:46.755093Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435904488144093:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:46.755276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435903041847034:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(18946165ns) {Configure:104(18945527ns) {ProcessConfigured:135(17855238ns) {StartInitialHandshake:157(17854409ns) {DropHandshakes:469(1266ns) {DropIncomingHandshake:441(168ns) DropOutgoingHandshake:456(134ns)} GenerateSessionVirtualId:988(563ns) PrepareNewSessionHandshake:476(653ns) {SwitchToState:178(256ns)}}}}}
2025-11-19T13:59:46.755520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435902273082966:2104] [node 5] ICP01 ready to work
2025-11-19T13:59:46.755184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435903453502037:2088] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(18392561ns) {Configure:104(18392126ns) {ProcessConfigured:135(18095657ns) {StartInitialHandshake:157(18095261ns) {DropHandshakes:469(462ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:476(636ns) {SwitchToState:178(290ns)}}}}}
2025-11-19T13:59:46.756486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435901234709105:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58483
2025-11-19T13:59:46.765234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435903041847035:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18698415ns) {Configure:104(18697777ns) {ProcessConfigured:135(18249304ns) {StartInitialHandshake:157(18249198ns) {DropHandshakes:469(778ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(189ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(718ns) {SwitchToState:178(381ns)}}}}}
2025-11-19T13:59:46.765116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435901234709091:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(43539943ns) {Configure:104(43539042ns) {ProcessConfigured:135(43284483ns) {StartInitialHandshake:157(43284056ns) {DropHandshakes:469(394ns) {DropIncomingHandshake:441(33ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(94ns) PrepareNewSessionHandshake:476(1371ns) {SwitchToState:178(489ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.765569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435903453502055:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64241
2025-11-19T13:59:46.765655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435903453502056:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64241
2025-11-19T13:59:46.765664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435903453502058:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64241
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.765790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903041847106:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39267
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.766459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435903453502057:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64241
2025-11-19T13:59:46.760031Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435902273082958:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.760071Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435902273082958:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.765943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435904488144048:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17997231ns) {Configure:104(17996344ns) {ProcessConfigured:135(17609251ns) {StartInitialHandshake:157(17608571ns) {DropHandshakes:469(765ns) {DropIncomingHandshake:441(77ns) DropOutgoingHandshake:456(194ns)} GenerateSessionVirtualId:988(245ns) PrepareNewSessionHandshake:476(955ns) {SwitchToState:178(595ns)}}}}}
2025-11-19T13:59:46.771892Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435903453502055:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.771945Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435903453502057:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.771988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435903453501979:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.774620Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435901931213273:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.774637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435901931213273:2104] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.775196Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435901931213261:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.775200Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435901931213261:2094] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.775716Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435901931213273:2104] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.765865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903041847107:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39267
2025-11-19T13:59:46.774977Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435903041847106:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.775042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435903041847035:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.777951Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435901234709094:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.776259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435903453502060:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64241
2025-11-19T13:59:46.778172Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435903453502058:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.778196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435903453501977:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.780066Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435902273082954:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.780870Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435901931213261:2094] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:46.780427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903041847109:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39267
2025-11-19T13:59:46.781060Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435903041847106:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:46.781839Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435902273082969:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.781880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435902273082891:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.781934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435901931213247:2077] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.781954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:46.781961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435901931213247:2077] [node 4] ICP22 created new session: [3:7574435901931213293:2077]
2025-11-19T13:59:46.781969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435901931213293:2077] [node 4] ICS09 handshake done sender: [3:7574435901931213273:2104] self: [3:7574435901931213256:0] peer: [4:7574435903453502059:0] socket: 618 qp: -1
2025-11-19T13:59:46.781976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435901931213293:2077] [node 4] ICS10 traffic start
2025-11-19T13:59:46.781997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:46.781979Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435903041847107:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.782004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435903041847033:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.782506Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435903453502056:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.782612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435903453502062:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64241
2025-11-19T13:59:46.780983Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435901234709100:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.780997Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435901234709100:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.781371Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435901234709103:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.781448Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435901234709105:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.781858Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435901234709100:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:46.782746Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435901234709097:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.782750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435901234709097:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.783228Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435901234709095:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.783231Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435901234709095:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.783733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435901234709085:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.783773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:46.783780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435901234709085:2107] [node 4] ICP22 created new session: [1:7574435901234709113:2107]
2025-11-19T13:59:46.783785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435901234709113:2107] [node 4] ICS09 handshake done sender: [1:7574435901234709100:2113] self: [1:7574435901234709086:0] peer: [4:7574435903453502061:0] socket: 625 qp: -1
2025-11-19T13:59:46.783790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435901234709113:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:46.783807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.784240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435901234709112:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58483
2025-11-19T13:59:46.784718Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435903453502057:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:46.784887Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435903453502058:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:46.784978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435903453501977:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.784992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:46.784997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435903453501977:2048] [node 1] ICP22 created new session: [4:7574435903453502064:2048]
2025-11-19T13:59:46.785003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435903453502064:2048] [node 1] ICS09 handshake done sender: [4:7574435903453502058:2118] self: [4:7574435903453502061:0] peer: [1:7574435901234709086:0] socket: 636 qp: -1
2025-11-19T13:59:46.785009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435903453502064:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:46.785023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:46.786067Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435901234709097:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:46.787209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435903453501979:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.787229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:46.787236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435903453501979:2050] [node 3] ICP22 created new session: [4:7574435903453502067:2050]
2025-11-19T13:59:46.787242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435903453502067:2050] [node 3] ICS09 handshake done sender: [4:7574435903453502057:2117] self: [4:7574435903453502059:0] peer: [3:7574435901931213256:0] socket: 635 qp: -1
2025-11-19T13:59:46.787248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435903453502067:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:46.787255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:46.787776Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435901234709095:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:46.782168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435901931213245:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.782179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:46.782184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435901931213245:2084] [node 6] ICP22 created new session: [3:7574435901931213295:2084]
2025-11-19T13:59:46.782188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435901931213295:2084] [node 6] ICS09 handshake done sender: [3:7574435901931213261:2094] self: [3:7574435901931213250:0] peer: [6:7574435903041847108:0] socket: 610 qp: -1
2025-11-19T13:59:46.782192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435901931213295:2084] [node 6] ICS10 traffic start
2025-11-19T13:59:46.782196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.787951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435901234709024:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.787966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:46.787972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435901234709024:2048] [node 2] ICP22 created new session: [1:7574435901234709118:2048]
2025-11-19T13:59:46.787977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435901234709118:2048] [node 2] ICS09 handshake done sender: [1:7574435901234709097:2112] self: [1:7574435901234709096:0] peer: [2:7574435902273082972:0] socket: 620 qp: -1
2025-11-19T13:59:46.787981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435901234709118:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:46.787986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:46.788065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435901234709090:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.788074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:46.788077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435901234709090:2108] [node 6] ICP22 created new session: [1:7574435901234709120:2108]
2025-11-19T13:59:46.788081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435901234709120:2108] [node 6] ICS09 handshake done sender: [1:7574435901234709095:2111] self: [1:7574435901234709093:0] peer: [6:7574435903041847110:0] socket: 619 qp: -1
2025-11-19T13:59:46.788084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435901234709120:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:46.788088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:46.789562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903041847111:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39267
2025-11-19T13:59:46.789853Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435903453502019:2084] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.789885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435903453501978:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.791845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435903041847093:2064] [node 5] ICP01 ready to work
2025-11-19T13:59:46.796591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435902273082891:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29668) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435902273082969:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.796705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435902273082973:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57900
2025-11-19T13:59:46.796728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435902273082974:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57900
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.796790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435902273082975:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57900
2025-11-19T13:59:46.796882Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435902273082966:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(40782479ns) {Configure:104(40781125ns) {ProcessConfigured:135(40456481ns) {StartInitialHandshake:157(40455095ns) {DropHandshakes:469(517ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:476(1067ns) {SwitchToState:178(381ns)}}}}}
2025-11-19T13:59:46.798528Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435902273082958:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:46.798612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435902273082892:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.798628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:46.798636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435902273082892:2049] [node 3] ICP22 created new session: [2:7574435902273082978:2049]
2025-11-19T13:59:46.798643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435902273082978:2049] [node 3] ICS09 handshake done sender: [2:7574435902273082958:2110] self: [2:7574435902273082957:0] peer: [3:7574435901931213286:0] socket: 606 qp: -1
2025-11-19T13:59:46.798652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435902273082978:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:46.798676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:46.799109Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435902273082969:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.799990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435901931213298:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62258
2025-11-19T13:59:46.800074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435901931213301:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62258
2025-11-19T13:59:46.801991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435901931213299:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62258
2025-11-19T13:59:46.802083Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435901931213298:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.802095Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435901931213262:2095] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:46.802237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435903041847033:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29668) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435903041847107:2112] held: no
2025-11-19T13:59:46.802757Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435903041847107:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.803045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435903041847033:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.803065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:46.803070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435903041847033:2048] [node 1] ICP22 created new session: [6:7574435903041847119:2048]
2025-11-19T13:59:46.803076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435903041847119:2048] [node 1] ICS09 handshake done sender: [6:7574435903041847107:2112] self: [6:7574435903041847110:0] peer: [1:7574435901234709093:0] socket: 637 qp: -1
2025-11-19T13:59:46.803084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435903041847119:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:46.803091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:46.803260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903041847118:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39267
2025-11-19T13:59:46.805034Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435902273082973:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.805070Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435902273082974:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.805099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435902273082892:2049] [node 3] ICP18 handshake (actor [2:7574435902273082973:2116]) is too old
2025-11-19T13:59:46.805793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435903041847035:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.805819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:46.805825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435903041847035:2050] [node 3] ICP22 created new session: [6:7574435903041847122:2050]
2025-11-19T13:59:46.805831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435903041847122:2050] [node 3] ICS09 handshake done sender: [6:7574435903041847106:2111] self: [6:7574435903041847108:0] peer: [3:7574435901931213250:0] socket: 612 qp: -1
2025-11-19T13:59:46.805837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435903041847122:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:46.805853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:46.807934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903041847112:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39267
2025-11-19T13:59:46.808081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435904488144019:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.808121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:46.808130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435904488144019:2050] [node 3] ICP22 created new session: [5:7574435904488144097:2050]
2025-11-19T13:59:46.808138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435904488144097:2050] [node 3] ICS09 handshake done sender: [5:7574435904488144093:2113] self: [5:7574435904488144094:0] peer: [3:7574435901931213249:0] socket: 611 qp: -1
2025-11-19T13:59:46.808147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435904488144097:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:46.808172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:46.808014Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435902273082961:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.808033Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435902273082961:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.805489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435901931213207:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.805531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:46.805540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435901931213207:2049] [node 2] ICP22 created new session: [3:7574435901931213302:2049]
2025-11-19T13:59:46.805548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435901931213302:2049] [node 2] ICS09 handshake done sender: [3:7574435901931213262:2095] self: [3:7574435901931213286:0] peer: [2:7574435902273082957:0] socket: 608 qp: -1
2025-11-19T13:59:46.805555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435901931213302:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:46.805562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:46.805869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435901931213300:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62258
2025-11-19T13:59:46.805900Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435901931213247:2077] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:46.812289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435901931213241:2080] [node 5] ICP18 handshake (actor [3:7574435901931213298:2116]) is too old
2025-11-19T13:59:46.814869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435904488144018:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(86930535ns) {Configure:104(75039276ns) {ProcessConfigured:135(74675975ns) {StartInitialHandshake:157(74675010ns) {DropHandshakes:469(687ns) {DropIncomingHandshake:441(86ns) DropOutgoingHandshake:456(91ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:476(1531ns) {SwitchToState:178(886ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.812287Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435903041847112:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.812333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435903041847063:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.815986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435903453502013:2074] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(66686109ns) {Configure:104(66685201ns) {ProcessConfigured:135(66330582ns) {StartInitialHandshake:157(66330187ns) {DropHandshakes:469(949ns) {DropIncomingHandshake:441(94ns) DropOutgoingHandshake:456(205ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(936ns) {SwitchToState:178(257ns)}}}}}
2025-11-19T13:59:46.816052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435903453502054:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.816063Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435903453502054:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.815307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904488144100:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51046
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.816971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904488144101:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51046
2025-11-19T13:59:46.820466Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435904488144100:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.820507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435904488144017:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.820953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435904488144019:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(63173068ns) {HandleHandshakeStatus:329(63172160ns) {DropIncomingHandshake:441(1411ns) DropOutgoingHandshake:456(13239ns) SwitchToState:178(279ns) ProcessPendingSessionEvents:495(47737ns) {ForwardSessionEventToSession:612(34815ns) ForwardSessionEventToSession:612(745ns) ForwardSessionEventToSession:612(1268ns) ForwardSessionEventToSession:612(781ns) ForwardSessionEventToSession:612(1206ns) ForwardSessionEventToSession:612(1192ns) ForwardSessionEventToSession:612(683ns) ForwardSessionEventToSession:612(1238ns) ForwardSessionEventToSession:612(650ns) ForwardSessionEventToSession:612(1276ns)}}}
2025-11-19T13:59:46.822343Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435901234709106:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.822359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435901234709106:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.823958Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435903453502054:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:46.825428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435903453502037:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.825460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:46.825472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435903453502037:2088] [node 6] ICP22 created new session: [4:7574435903453502074:2088]
2025-11-19T13:59:46.825480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435903453502074:2088] [node 6] ICS09 handshake done sender: [4:7574435903453502054:2114] self: [4:7574435903453502038:0] peer: [6:7574435903041847124:0] socket: 649 qp: -1
2025-11-19T13:59:46.825489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435903453502074:2088] [node 6] ICS10 traffic start
2025-11-19T13:59:46.825504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:46.828926Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435901931213207:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(23403169ns) {HandleHandshakeStatus:329(23402308ns) {DropIncomingHandshake:441(359ns) DropOutgoingHandshake:456(2037ns) SwitchToState:178(83ns) ProcessPendingSessionEvents:495(8963317ns) {ForwardSessionEventToSession:612(8946489ns) ForwardSessionEventToSession:612(2510ns) ForwardSessionEventToSession:612(1262ns) ForwardSessionEventToSession:612(552ns) ForwardSessionEventToSession:612(611ns) ForwardSessionEventToSession:612(2060ns) ForwardSessionEventToSession:612(639ns) ForwardSessionEventToSession:612(1065ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(1472ns)}}}
2025-11-19T13:59:46.830326Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435903041847118:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.830359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435903041847034:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.828152Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435902273082961:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:46.828746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435902273082891:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.828769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:46.828780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435902273082891:2048] [node 1] ICP22 created new session: [2:7574435902273082983:2048]
2025-11-19T13:59:46.828787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435902273082983:2048] [node 1] ICS09 handshake done sender: [2:7574435902273082969:2115] self: [2:7574435902273082972:0] peer: [1:7574435901234709096:0] socket: 626 qp: -1
2025-11-19T13:59:46.828796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435902273082983:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:46.828806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:46.828918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435902273082959:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.828929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:46.828934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435902273082959:2109] [node 4] ICP22 created new session: [2:7574435902273082985:2109]
2025-11-19T13:59:46.828938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435902273082985:2109] [node 4] ICS09 handshake done sender: [2:7574435902273082961:2111] self: [2:7574435902273082960:0] peer: [4:7574435903453502069:0] socket: 607 qp: -1
2025-11-19T13:59:46.828942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435902273082985:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:46.828947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:46.830212Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435901234709112:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.831898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904488144104:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51046
2025-11-19T13:59:46.831937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904488144105:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51046
2025-11-19T13:59:46.832210Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435903041847063:2072] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61946) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435903041847112:2115] held: no
2025-11-19T13:59:46.832622Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435904488144101:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.832668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435904488144038:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.833529Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435901931213300:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.833567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435901931213206:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.835876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435903041847035:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(54703666ns) {HandleHandshakeStatus:329(54702464ns) {DropIncomingHandshake:441(474ns) DropOutgoingHandshake:456(8341ns) SwitchToState:178(70ns) ProcessPendingSessionEvents:495(15077778ns) {ForwardSessionEventToSession:612(15060545ns) ForwardSessionEventToSession:612(2947ns) ForwardSessionEventToSession:612(1513ns) ForwardSessionEventToSession:612(739ns) ForwardSessionEventToSession:612(616ns) ForwardSessionEventToSession:612(1439ns) ForwardSessionEventToSession:612(910ns) ForwardSessionEventToSession:612(841ns) ForwardSessionEventToSession:612(668ns) ForwardSessionEventToSession:612(1234ns)}}}
2025-11-19T13:59:46.837875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435903453501978:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11490) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435903453502019:2084] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.839943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904488144107:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51046
2025-11-19T13:59:46.837457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435901931213305:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62258
2025-11-19T13:59:46.837705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435901931213245:2084] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:46.840210Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435901234709106:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:46.840354Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435904488144100:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:46.840457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435904488144017:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.840483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:46.840490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435904488144017:2048] [node 1] ICP22 created new session: [5:7574435904488144108:2048]
2025-11-19T13:59:46.840496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435904488144108:2048] [node 1] ICS09 handshake done sender: [5:7574435904488144100:2117] self: [5:7574435904488144102:0] peer: [1:7574435901234709092:0] socket: 613 qp: -1
2025-11-19T13:59:46.840504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435904488144108:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:46.840512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.839473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435903453502076:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64241
2025-11-19T13:59:46.841817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903041847127:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39267
2025-11-19T13:59:46.841861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903041847128:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39267
2025-11-19T13:59:46.842253Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435903453502019:2084] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:46.842382Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435903041847112:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:46.842488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435903041847063:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.842504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:46.842512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435903041847063:2072] [node 4] ICP22 created new session: [6:7574435903041847129:2072]
2025-11-19T13:59:46.842518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435903041847129:2072] [node 4] ICS09 handshake done sender: [6:7574435903041847112:2115] self: [6:7574435903041847124:0] peer: [4:7574435903453502038:0] socket: 651 qp: -1
2025-11-19T13:59:46.842525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435903041847129:2072] [node 4] ICS10 traffic start
2025-11-19T13:59:46.842533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:46.842897Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435901234709102:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.842924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435901234709102:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.843116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435901234709091:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.843137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:46.843148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435901234709091:2109] [node 5] ICP22 created new session: [1:7574435901234709125:2109]
2025-11-19T13:59:46.843155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435901234709125:2109] [node 5] ICS09 handshake done sender: [1:7574435901234709106:2118] self: [1:7574435901234709092:0] peer: [5:7574435904488144102:0] socket: 600 qp: -1
2025-11-19T13:59:46.843160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435901234709125:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:46.843167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:46.843982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435903453502037:2088] [node 6] ICP35 event processing took too much time PendingConnection:278(18524471ns) {HandleHandshakeStatus:329(18522957ns) {DropIncomingHandshake:441(7719ns) DropOutgoingHandshake:456(290ns) SwitchToState:178(589ns) ProcessPendingSessionEvents:495(18365939ns) {ForwardSessionEventToSession:612(18344330ns) ForwardSessionEventToSession:612(3670ns) ForwardSessionEventToSession:612(1040ns) ForwardSessionEventToSession:612(2770ns) ForwardSessionEventToSession:612(1026ns) ForwardSessionEventToSession:612(953ns) ForwardSessionEventToSession:612(879ns) ForwardSessionEventToSession:612(798ns) ForwardSessionEventToSession:612(756ns) ForwardSessionEventToSession:612(1517ns)}}}
2025-11-19T13:59:46.845216Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435903453502070:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.845232Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435903453502070:2121] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.845158Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435902273082968:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.845183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435902273082968:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.849453Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435904488144105:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.852840Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435904488144104:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.852884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435904488144018:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.853234Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435902273082976:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.853247Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435902273082976:2119] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.853962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904488144113:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51046
2025-11-19T13:59:46.854070Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435903041847128:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.854115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435903041847093:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:46.854297Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435904488144096:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:46.854302Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435904488144096:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:46.855093Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435904488144096:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.855252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903041847132:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39267
2025-11-19T13:59:46.855426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435904488144048:2063] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.855455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:46.855463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435904488144048:2063] [node 6] ICP22 created new session: [5:7574435904488144115:2063]
2025-11-19T13:59:46.855469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435904488144115:2063] [node 6] ICS09 handshake done sender: [5:7574435904488144096:2115] self: [5:7574435904488144092:0] peer: [6:7574435903041847131:0] socket: 664 qp: -1
2025-11-19T13:59:46.855477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435904488144115:2063] [node 6] ICS10 traffic start
2025-11-19T13:59:46.855485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:46.855904Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435902273082976:2119] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:46.855954Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435903041847128:2122] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:46.856189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435903041847093:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.856206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:46.856211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435903041847093:2064] [node 5] ICP22 created new session: [6:7574435903041847133:2064]
2025-11-19T13:59:46.856217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435903041847133:2064] [node 5] ICS09 handshake done sender: [6:7574435903041847128:2122] self: [6:7574435903041847131:0] peer: [5:7574435904488144092:0] socket: 621 qp: -1
2025-11-19T13:59:46.856223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435903041847133:2064] [node 5] ICS10 traffic start
2025-11-19T13:59:46.856230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:46.856438Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435904488144104:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:46.861519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435904488144018:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.861553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:46.861565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435904488144018:2049] [node 2] ICP22 created new session: [5:7574435904488144117:2049]
2025-11-19T13:59:46.861573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435904488144117:2049] [node 2] ICS09 handshake done sender: [5:7574435904488144104:2119] self: [5:7574435904488144112:0] peer: [2:7574435902273082970:0] socket: 666 qp: -1
2025-11-19T13:59:46.861583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435904488144117:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:46.861594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:46.861829Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435901234709102:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.862111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435901931213306:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62258
2025-11-19T13:59:46.862297Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435901931213300:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:46.862398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435901931213206:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.862420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:46.862427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435901931213206:2048] [node 1] ICP22 created new session: [3:7574435901931213307:2048]
2025-11-19T13:59:46.862434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435901931213307:2048] [node 1] ICS09 handshake done sender: [3:7574435901931213300:2118] self: [3:7574435901931213304:0] peer: [1:7574435901234709087:0] socket: 655 qp: -1
2025-11-19T13:59:46.862441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435901931213307:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:46.862453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.863165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435902273082989:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57900
2025-11-19T13:59:46.863650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435903453501978:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.863675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:46.863687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435903453501978:2049] [node 2] ICP22 created new session: [4:7574435903453502077:2049]
2025-11-19T13:59:46.863693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435903453502077:2049] [node 2] ICS09 handshake done sender: [4:7574435903453502019:2084] self: [4:7574435903453502069:0] peer: [2:7574435902273082960:0] socket: 609 qp: -1
2025-11-19T13:59:46.863700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435903453502077:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:46.863734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:46.863245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435901234709025:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.863280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:46.863287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435901234709025:2049] [node 3] ICP22 created new session: [1:7574435901234709128:2049]
2025-11-19T13:59:46.863293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435901234709128:2049] [node 3] ICS09 handshake done sender: [1:7574435901234709102:2114] self: [1:7574435901234709087:0] peer: [3:7574435901931213304:0] socket: 648 qp: -1
2025-11-19T13:59:46.863298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435901234709128:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:46.863311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:46.864038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435902273082966:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.864056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:46.864063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435902273082966:2104] [node 5] ICP22 created new session: [2:7574435902273082990:2104]
2025-11-19T13:59:46.864069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435902273082990:2104] [node 5] ICS09 handshake done sender: [2:7574435902273082976:2119] self: [2:7574435902273082970:0] peer: [5:7574435904488144112:0] socket: 663 qp: -1
2025-11-19T13:59:46.864075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435902273082990:2104] [node 5] ICS10 traffic start
2025-11-19T13:59:46.864106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:46.869268Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435902273082968:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:46.869452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435902273082945:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.869477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:46.869485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435902273082945:2098] [node 6] ICP22 created new session: [2:7574435902273082993:2098]
2025-11-19T13:59:46.869492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435902273082993:2098] [node 6] ICS09 handshake done sender: [2:7574435902273082968:2114] self: [2:7574435902273082967:0] peer: [6:7574435903041847125:0] socket: 653 qp: -1
2025-11-19T13:59:46.869498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435902273082993:2098] [node 6] ICS10 traffic start
2025-11-19T13:59:46.869519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.869748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435903041847135:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39267
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:46.870073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435904488144119:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51046
2025-11-19T13:59:46.869936Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435903453502070:2121] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:46.870095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435903041847034:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11490) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435903041847118:2120] held: no
2025-11-19T13:59:46.870220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435903453502013:2074] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:46.870237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:46.870245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435903453502013:2074] [node 5] ICP22 created new session: [4:7574435903453502080:2074]
2025-11-19T13:59:46.870251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435903453502080:2074] [node 5] ICS09 handshake done sender: [4:7574435903453502070:2121] self: [4:7574435903453502051:0] peer: [5:7574435904488144106:0] socket: 657 qp: -1
2025-11-19T13:59:46.870256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435903453502080:2074] [node 5] ICS10 traffic start
2025-11-19T13:59:46.870263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:46.870962Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435903041847118:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:46.871053Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435904488144101:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:46.870634Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435901234709091:2109] [node 5] ICP35 event processing took too much time PendingConnection:278(27490641ns) {HandleHandshakeStatus:329(27489724ns) {DropIncomingHandshake:441(9122ns) DropOutgoingHandshake:456(211ns) SwitchToState:178(266ns) ProcessPendingSessionEvents:495(27393655ns) {ForwardSessionEventToSession:612(27381886ns) ForwardSessionEventToSession:612(1485ns) ForwardSessionEventToSession:612(595ns) ForwardSessionEventToSession:612(1352ns) ForwardSessionEventToSession:612(592ns) ForwardSessionEventToSession:612(652ns) ForwardSessionEventToSession:612(977ns) ForwardSessionEventToSession:612(511ns) ForwardSessionEventToSession:612(548ns) ForwardSessionEventToSession:612(1092ns)}}}
2025-11-19T13:59:46.871924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435903453501978:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(29602763ns) {HandleHandshakeStatus:329(29601472ns) {DropIncomingHandshake:441(1006ns) DropOutgoingHandshake:456(66ns) SwitchToState:178(73ns) ProcessPendingSessionEvents:495(573ns)}}
2025-11-19T13:59:46.874516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435904488144038:2065] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61946) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435904488144101:2118] held: no
2025-11-19T13:59:46.875977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435904488144038:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.875999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:46.876007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435904488144038:2065] [node 4] ICP22 created new session: [5:7574435904488144120:2065]
2025-11-19T13:59:46.876012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435904488144120:2065] [node 4] ICS09 handshake done sender: [5:7574435904488144101:2118] self: [5:7574435904488144106:0] peer: [4:7574435903453502051:0] socket: 658 qp: -1
2025-11-19T13:59:46.876019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435904488144120:2065] [node 4] ICS10 traffic start
2025-11-19T13:59:46.876030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:46.876200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435903041847034:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:46.876217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:46.876223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435903041847034:2049] [node 2] ICP22 created new session: [6:7574435903041847136:2049]
2025-11-19T13:59:46.876229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435903041847136:2049] [node 2] ICS09 handshake done sender: [6:7574435903041847118:2120] self: [6:7574435903041847125:0] peer: [2:7574435902273082967:0] socket: 616 qp: -1
2025-11-19T13:59:46.876235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435903041847136:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:46.876243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:46.878761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435902273082966:2104] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:46.900447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:46.900581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435902273082983:2048] [node 1] ICS01 socket: 626 reason# EndOfStream
2025-11-19T13:59:46.900610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435902273082891:2048] [node 1] ICP30 unregister session Session# [2:7574435902273082983:2048] VirtualId# [2:7574435902273082972:0]
2025-11-19T13:59:46.900621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435902273082983:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.900675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:46.900704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:46.901572Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:46.901673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435901931213307:2048] [node 1] ICS01 socket: 655 reason# EndOfStream
2025-11-19T13:59:46.901680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435901931213206:2048] [node 1] ICP30 unregister session Session# [3:7574435901931213307:2048] VirtualId# [3:7574435901931213304:0]
2025-11-19T13:59:46.901686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435901931213307:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.901728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:46.901766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:46.916142Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:46.916255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435903041847119:2048] [node 1] ICS01 socket: 637 reason# EndOfStream
2025-11-19T13:59:46.916267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435903041847033:2048] [node 1] ICP30 unregister session Session# [6:7574435903041847119:2048] VirtualId# [6:7574435903041847110:0]
2025-11-19T13:59:46.916276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435903041847119:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.916352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:46.916378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:46.916982Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:46.917054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435904488144108:2048] [node 1] ICS01 socket: 613 reason# EndOfStream
2025-11-19T13:59:46.917059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435904488144017:2048] [node 1] ICP30 unregister session Session# [5:7574435904488144108:2048] VirtualId# [5:7574435904488144102:0]
2025-11-19T13:59:46.917064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435904488144108:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.917100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:46.917121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:46.924794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:46.924904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435903453502064:2048] [node 1] ICS01 socket: 636 reason# EndOfStream
2025-11-19T13:59:46.924914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435903453501977:2048] [node 1] ICP30 unregister session Session# [4:7574435903453502064:2048] VirtualId# [4:7574435903453502061:0]
2025-11-19T13:59:46.924923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435903453502064:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.924975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:46.925002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:46.931846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:46.931948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435901931213302:2049] [node 2] ICS07 socket disconnect 608 reason# EndOfStream
2025-11-19T13:59:46.931957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435901931213302:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.932000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:46.932033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435901931213302:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:46.935432Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435901931213207:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11490) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:46.935446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435901931213302:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:46.935829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435901931213207:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11490) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:46.935854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435901931213302:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:46.935862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435901931213207:2049] [node 2] ICP30 unregister session Session# [3:7574435901931213302:2049] VirtualId# [3:7574435901931213286:0]
2025-11-19T13:59:46.935894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:46.935903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435901931213207:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:46.935907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11490) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:46.947847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574435901931213207:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:46.970175Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:46.970288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435904488144117:2049] [node 2] ICS01 socket: 666 reason# EndOfStream
2025-11-19T13:59:46.970302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435904488144018:2049] [node 2] ICP30 unregister session Session# [5:7574435904488144117:2049] VirtualId# [5:7574435904488144112:0]
2025-11-19T13:59:46.970310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435904488144117:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.970365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:46.970390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:46.970508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:46.970575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435903041847136:2049] [node 2] ICS01 socket: 616 reason# EndOfStream
2025-11-19T13:59:46.970581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435903041847034:2049] [node 2] ICP30 unregister session Session# [6:7574435903041847136:2049] VirtualId# [6:7574435903041847125:0]
2025-11-19T13:59:46.970586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435903041847136:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.970644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:46.970670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:46.970947Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:46.971036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435903453502077:2049] [node 2] ICS01 socket: 609 reason# EndOfStream
2025-11-19T13:59:46.971045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435903453501978:2049] [node 2] ICP30 unregister session Session# [4:7574435903453502077:2049] VirtualId# [4:7574435903453502069:0]
2025-11-19T13:59:46.971051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435903453502077:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.971095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:46.971116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:46.993355Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T13:59:46.993482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435903453502067:2050] [node 3] ICS01 socket: 635 reason# EndOfStream
2025-11-19T13:59:46.993495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435903453501979:2050] [node 3] ICP30 unregister session Session# [4:7574435903453502067:2050] VirtualId# [4:7574435903453502059:0]
2025-11-19T13:59:46.993511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435903453502067:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:46.993565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:46.993593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:46.999868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:47.000015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435904488144097:2050] [node 3] ICS01 socket: 611 reason# EndOfStream
2025-11-19T13:59:47.000029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435904488144019:2050] [node 3] ICP30 unregister session Session# [5:7574435904488144097:2050] VirtualId# [5:7574435904488144094:0]
2025-11-19T13:59:47.000040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435904488144097:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:47.000112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:47.000142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:47.000190Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:47.000292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435903041847122:2050] [node 3] ICS07 socket disconnect 612 reason# EndOfStream
2025-11-19T13:59:47.000297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435903041847122:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:47.000326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:47.000352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435903041847122:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:47.001731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435903041847035:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13043) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:47.001743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435903041847122:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:47.001748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435903041847035:2050] [node 3] ICP30 unregister session Session# [6:7574435903041847122:2050] VirtualId# [6:7574435903041847108:0]
2025-11-19T13:59:47.001766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:47.001773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435903041847035:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13043) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:47.001776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13043) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:47.012096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435903041847035:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:59:47.030275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:47.030408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435903041847129:2072] [node 4] ICS01 socket: 651 reason# EndOfStream
2025-11-19T13:59:47.030419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435903041847063:2072] [node 4] ICP30 unregister session Session# [6:7574435903041847129:2072] VirtualId# [6:7574435903041847124:0]
2025-11-19T13:59:47.030429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435903041847129:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:47.030486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:47.030514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:47.602417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435908256503968:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:47.602433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435908256503969:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:47.651880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435904869391730:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:47.651905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435904869391731:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:47.705648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435906487965377:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:47.705673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435906487965378:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:47.733436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435904772915260:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:47.733450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435904772915261:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:47.733455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435904772915262:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:47.756030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435906505837883:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:47.756045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435906505837884:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:47.756050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435906505837885:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:47.804080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435907145302727:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:47.804094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435907145302728:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:47.804098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435907145302729:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:47.844742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435906505837912:2070] [node 6] ICP01 ready to work
2025-11-19T13:59:47.844408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435904869391791:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:47.844902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435907145302747:2063] [node 5] ICP01 ready to work
2025-11-19T13:59:47.845294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435907145302753:2067] [node 4] ICP01 ready to work
2025-11-19T13:59:47.845469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435906505837923:2078] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.845689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435908256504031:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:47.845767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435908256504032:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:47.845832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435908256504033:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:47.847171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435908256504044:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
2025-11-19T13:59:47.844221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435904772915307:2087] [node 5] ICP01 ready to work
2025-11-19T13:59:47.844370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435904772915311:2092] [node 6] ICP01 ready to work
2025-11-19T13:59:47.846686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435904772915324:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57471
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.853589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435904869391798:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:47.853274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435906505837940:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19246
2025-11-19T13:59:47.853303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435906505837941:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19246
2025-11-19T13:59:47.853769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435906487965437:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5137
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.859055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435907145302774:2086] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64617
2025-11-19T13:59:47.859114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435907145302775:2087] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64617
2025-11-19T13:59:47.859736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435906487965438:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5137
2025-11-19T13:59:47.859997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435906487965442:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:47.860081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435906487965443:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:47.860159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435906487965444:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.880191Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435906487965438:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.881504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435906487965445:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5137
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.886579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435904772915335:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57471
2025-11-19T13:59:47.886602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435904772915336:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57471
2025-11-19T13:59:47.889013Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435907145302775:2087] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.889043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435907145302753:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:47.888912Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435904772915324:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.889179Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435904772915320:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.889183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435904772915320:2102] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.889739Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435904772915320:2102] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:47.889817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435904772915311:2092] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.889837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:47.889843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435904772915311:2092] [node 6] ICP22 created new session: [4:7574435904772915343:2092]
2025-11-19T13:59:47.889849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435904772915343:2092] [node 6] ICS09 handshake done sender: [4:7574435904772915320:2102] self: [4:7574435904772915319:0] peer: [6:7574435907145302807:0] socket: 614 qp: -1
2025-11-19T13:59:47.889855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435904772915343:2092] [node 6] ICS10 traffic start
2025-11-19T13:59:47.889863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.900446Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435906505837941:2095] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.900689Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435904772915315:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.900705Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435904772915315:2099] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.900371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435908256504048:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
2025-11-19T13:59:47.900514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435906505837963:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19246
2025-11-19T13:59:47.900533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435906505837923:2078] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:47.904067Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435908256504048:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.904178Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435908256504044:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.908439Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435906487965437:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.908474Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435906487965445:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.911924Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435904772915315:2099] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:47.912833Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435904772915335:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.912886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435904772915307:2087] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.912900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:47.912911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435904772915307:2087] [node 5] ICP22 created new session: [4:7574435904772915347:2087]
2025-11-19T13:59:47.912917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435904772915347:2087] [node 5] ICS09 handshake done sender: [4:7574435904772915315:2099] self: [4:7574435904772915314:0] peer: [5:7574435906505837964:0] socket: 612 qp: -1
2025-11-19T13:59:47.912925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435904772915347:2087] [node 5] ICS10 traffic start
2025-11-19T13:59:47.912935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:47.913143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435904772915260:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.915035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435906505837965:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19246
2025-11-19T13:59:47.912481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435906487965377:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:47.915134Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435908256504035:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.915148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435908256504035:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.916461Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435906505837941:2095] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:47.916550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435906505837923:2078] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.916572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:47.916581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435906505837923:2078] [node 4] ICP22 created new session: [5:7574435906505837966:2078]
2025-11-19T13:59:47.916589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435906505837966:2078] [node 4] ICS09 handshake done sender: [5:7574435906505837941:2095] self: [5:7574435906505837964:0] peer: [4:7574435904772915314:0] socket: 625 qp: -1
2025-11-19T13:59:47.916596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435906505837966:2078] [node 4] ICS10 traffic start
2025-11-19T13:59:47.916607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.915806Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435908256504039:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.915816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435908256504039:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.917938Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435908256504035:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:47.916618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435907145302753:2067] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32736) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435907145302775:2087] held: no
2025-11-19T13:59:47.916718Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435907145302774:2086] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.916777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435907145302808:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64617
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.916878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435904869391803:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.917877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435904869391799:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40743
2025-11-19T13:59:47.917947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435904869391800:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40743
2025-11-19T13:59:47.917987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435904869391801:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40743
2025-11-19T13:59:47.918028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435904869391802:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40743
2025-11-19T13:59:47.918225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435904772915336:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.918341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435904772915351:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57471
2025-11-19T13:59:47.918353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435904772915352:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57471
2025-11-19T13:59:47.918368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435904772915262:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.918975Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435906505837963:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.919010Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435906505837940:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.919213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435906505837885:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:47.919253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435906505837968:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19246
2025-11-19T13:59:47.919693Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435904772915335:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.921480Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435906487965447:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.921495Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435906487965447:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.922026Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435906487965449:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.922030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435906487965449:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.923016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435906487965462:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5137
2025-11-19T13:59:47.919293Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435908256504039:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:47.919465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435908256503969:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.919475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:47.919482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435908256503969:2049] [node 3] ICP22 created new session: [1:7574435908256504054:2049]
2025-11-19T13:59:47.919487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435908256504054:2049] [node 3] ICS09 handshake done sender: [1:7574435908256504035:2110] self: [1:7574435908256504034:0] peer: [3:7574435906487965461:0] socket: 619 qp: -1
2025-11-19T13:59:47.919492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435908256504054:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:47.919500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:47.919593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435908256504031:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.919602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:47.919608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435908256504031:2107] [node 4] ICP22 created new session: [1:7574435908256504056:2107]
2025-11-19T13:59:47.919612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435908256504056:2107] [node 4] ICS09 handshake done sender: [1:7574435908256504039:2112] self: [1:7574435908256504038:0] peer: [4:7574435904772915350:0] socket: 621 qp: -1
2025-11-19T13:59:47.919615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435908256504056:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:47.919619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:47.923145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435908256504053:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
2025-11-19T13:59:47.923677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435904772915260:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.923694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:47.923699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435904772915260:2048] [node 1] ICP22 created new session: [4:7574435904772915355:2048]
2025-11-19T13:59:47.923705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435904772915355:2048] [node 1] ICS09 handshake done sender: [4:7574435904772915335:2114] self: [4:7574435904772915350:0] peer: [1:7574435908256504038:0] socket: 632 qp: -1
2025-11-19T13:59:47.923711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435904772915355:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:47.923719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.923524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435907145302747:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:47.923592Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435907145302775:2087] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:47.924640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435907145302753:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.924649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:47.924653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435907145302753:2067] [node 4] ICP22 created new session: [6:7574435907145302812:2067]
2025-11-19T13:59:47.924657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435907145302812:2067] [node 4] ICS09 handshake done sender: [6:7574435907145302775:2087] self: [6:7574435907145302807:0] peer: [4:7574435904772915319:0] socket: 623 qp: -1
2025-11-19T13:59:47.924662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435907145302812:2067] [node 4] ICS10 traffic start
2025-11-19T13:59:47.924668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:47.924756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435907145302810:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64617
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.928208Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435904869391799:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.928241Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435904869391802:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.928258Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435904869391800:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.928276Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435904869391801:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.929498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435904869391730:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.930712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435908256504060:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4463
2025-11-19T13:59:47.930931Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435904772915260:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:47.931085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435904772915354:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57471
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.934077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435906487965463:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5137
2025-11-19T13:59:47.934167Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435906487965445:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:47.935072Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435907145302810:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.935105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435907145302727:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:47.934351Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435908256504037:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.934364Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435908256504037:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.934829Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435908256504060:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.936258Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435904772915336:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:47.935327Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435906505837915:2076] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.935340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435906505837915:2076] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.936212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435906505837972:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19246
2025-11-19T13:59:47.936230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435906505837973:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19246
2025-11-19T13:59:47.936059Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435906487965447:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.936491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435907145302815:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64617
2025-11-19T13:59:47.936863Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435906505837915:2076] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:47.936391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435904772915262:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.936413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:47.936420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435904772915262:2050] [node 3] ICP22 created new session: [4:7574435904772915357:2050]
2025-11-19T13:59:47.936427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435904772915357:2050] [node 3] ICS09 handshake done sender: [4:7574435904772915336:2115] self: [4:7574435904772915353:0] peer: [3:7574435906487965446:0] socket: 633 qp: -1
2025-11-19T13:59:47.936434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435904772915357:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:47.936444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:47.937451Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435906487965449:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:47.937550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435906487965377:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.937567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:47.937573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435906487965377:2048] [node 1] ICP22 created new session: [3:7574435906487965466:2048]
2025-11-19T13:59:47.937578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435906487965466:2048] [node 1] ICS09 handshake done sender: [3:7574435906487965445:2112] self: [3:7574435906487965461:0] peer: [1:7574435908256504034:0] socket: 627 qp: -1
2025-11-19T13:59:47.937584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435906487965466:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:47.937591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:47.937761Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435906487965463:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.937768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435906487965442:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.937781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:47.937785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435906487965442:2109] [node 4] ICP22 created new session: [3:7574435906487965468:2109]
2025-11-19T13:59:47.937789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435906487965468:2109] [node 4] ICS09 handshake done sender: [3:7574435906487965447:2113] self: [3:7574435906487965446:0] peer: [4:7574435904772915353:0] socket: 628 qp: -1
2025-11-19T13:59:47.937792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435906487965468:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:47.937798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:47.937906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435906487965443:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.937912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:47.937916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435906487965443:2110] [node 5] ICP22 created new session: [3:7574435906487965470:2110]
2025-11-19T13:59:47.937921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435906487965470:2110] [node 5] ICS09 handshake done sender: [3:7574435906487965449:2114] self: [3:7574435906487965448:0] peer: [5:7574435906505837970:0] socket: 629 qp: -1
2025-11-19T13:59:47.937924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435906487965470:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:47.937930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:47.938668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435906487965443:2110] [node 5] ICP18 handshake (actor [3:7574435906487965463:2119]) is too old
2025-11-19T13:59:47.936611Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435908256504037:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:47.936753Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435908256504043:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.936757Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435908256504043:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.941321Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435907145302774:2086] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:47.941551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435906505837912:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.941571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:47.941579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435906505837912:2070] [node 6] ICP22 created new session: [5:7574435906505837975:2070]
2025-11-19T13:59:47.941586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435906505837975:2070] [node 6] ICS09 handshake done sender: [5:7574435906505837915:2076] self: [5:7574435906505837914:0] peer: [6:7574435907145302811:0] socket: 610 qp: -1
2025-11-19T13:59:47.941592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435906505837975:2070] [node 6] ICS10 traffic start
2025-11-19T13:59:47.941602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:47.942230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435908256503968:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.942250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:47.942256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435908256503968:2048] [node 2] ICP22 created new session: [1:7574435908256504063:2048]
2025-11-19T13:59:47.942262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435908256504063:2048] [node 2] ICS09 handshake done sender: [1:7574435908256504037:2111] self: [1:7574435908256504036:0] peer: [2:7574435904869391815:0] socket: 620 qp: -1
2025-11-19T13:59:47.942267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435908256504063:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:47.942276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:47.942696Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435908256504043:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:47.942786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435908256504033:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.942799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:47.942805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435908256504033:2109] [node 6] ICP22 created new session: [1:7574435908256504066:2109]
2025-11-19T13:59:47.942810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435908256504066:2109] [node 6] ICS09 handshake done sender: [1:7574435908256504043:2114] self: [1:7574435908256504042:0] peer: [6:7574435907145302814:0] socket: 656 qp: -1
2025-11-19T13:59:47.942815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435908256504066:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:47.942822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:47.945341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435907145302747:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.945359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:47.945365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435907145302747:2063] [node 5] ICP22 created new session: [6:7574435907145302818:2063]
2025-11-19T13:59:47.945370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435907145302818:2063] [node 5] ICS09 handshake done sender: [6:7574435907145302774:2086] self: [6:7574435907145302811:0] peer: [5:7574435906505837914:0] socket: 622 qp: -1
2025-11-19T13:59:47.945376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435907145302818:2063] [node 5] ICS10 traffic start
2025-11-19T13:59:47.945382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:47.945539Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435907145302727:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28433) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435907145302810:2118] held: no
2025-11-19T13:59:47.946920Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435906505837973:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.946944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435906505837885:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4372) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435906505837963:2117] held: no
2025-11-19T13:59:47.947073Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435906505837963:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:47.947220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435906505837885:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.947233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:47.947238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435906505837885:2050] [node 3] ICP22 created new session: [5:7574435906505837977:2050]
2025-11-19T13:59:47.947243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435906505837977:2050] [node 3] ICS09 handshake done sender: [5:7574435906505837963:2117] self: [5:7574435906505837970:0] peer: [3:7574435906487965448:0] socket: 631 qp: -1
2025-11-19T13:59:47.947249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435906505837977:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:47.947255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:47.947352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435906505837883:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:47.947617Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435908256504041:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.947635Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435908256504041:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.948299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435904869391817:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40743
2025-11-19T13:59:47.948441Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435904772915352:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.948474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435904772915261:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:47.949337Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435906505837968:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.950330Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435904869391793:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.950342Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435904869391793:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.951502Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435906505837883:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28433) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435906505837973:2121] held: no
2025-11-19T13:59:47.948653Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435908256504053:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.948710Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435908256504041:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:47.948800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435908256504032:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.948815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:47.948822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435908256504032:2108] [node 5] ICP22 created new session: [1:7574435908256504069:2108]
2025-11-19T13:59:47.948827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435908256504069:2108] [node 5] ICS09 handshake done sender: [1:7574435908256504041:2113] self: [1:7574435908256504040:0] peer: [5:7574435906505837979:0] socket: 637 qp: -1
2025-11-19T13:59:47.948832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435908256504069:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:47.948841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:47.949163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435908256503968:2048] [node 2] ICP18 handshake (actor [1:7574435908256504053:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.952847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435907145302816:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64617
2025-11-19T13:59:47.953009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435907145302817:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64617
2025-11-19T13:59:47.953354Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435907145302816:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.953488Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435907145302810:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:47.953623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435907145302727:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.953637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:47.953645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435907145302727:2048] [node 1] ICP22 created new session: [6:7574435907145302821:2048]
2025-11-19T13:59:47.953651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435907145302821:2048] [node 1] ICS09 handshake done sender: [6:7574435907145302810:2118] self: [6:7574435907145302814:0] peer: [1:7574435908256504042:0] socket: 657 qp: -1
2025-11-19T13:59:47.953657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435907145302821:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:47.953664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:47.953779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435907145302729:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:47.953972Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435906487965453:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.953985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435906487965453:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.955258Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435906487965453:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:47.955350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435906487965444:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.955367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:47.955372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435906487965444:2111] [node 6] ICP22 created new session: [3:7574435906487965474:2111]
2025-11-19T13:59:47.955378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435906487965474:2111] [node 6] ICS09 handshake done sender: [3:7574435906487965453:2116] self: [3:7574435906487965452:0] peer: [6:7574435907145302823:0] socket: 613 qp: -1
2025-11-19T13:59:47.955384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435906487965474:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:47.955392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.956754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435907145302729:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4372) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435907145302816:2120] held: no
2025-11-19T13:59:47.957996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435907145302824:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64617
2025-11-19T13:59:47.957000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435906505837884:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:47.957054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435906505837980:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19246
2025-11-19T13:59:47.957298Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435906505837973:2121] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:47.957360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435906505837883:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.957370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:47.957376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435906505837883:2048] [node 1] ICP22 created new session: [5:7574435906505837982:2048]
2025-11-19T13:59:47.957381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435906505837982:2048] [node 1] ICS09 handshake done sender: [5:7574435906505837973:2121] self: [5:7574435906505837979:0] peer: [1:7574435908256504040:0] socket: 662 qp: -1
2025-11-19T13:59:47.957387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435906505837982:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:47.957395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:47.960373Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435904869391730:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28433) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435904869391800:2113] held: no
2025-11-19T13:59:47.960536Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435904869391805:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.960540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435904869391805:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.961145Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435904869391800:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.961486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435904869391730:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.961502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:47.961507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435904869391730:2048] [node 1] ICP22 created new session: [2:7574435904869391822:2048]
2025-11-19T13:59:47.961514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435904869391822:2048] [node 1] ICS09 handshake done sender: [2:7574435904869391800:2113] self: [2:7574435904869391815:0] peer: [1:7574435908256504036:0] socket: 636 qp: -1
2025-11-19T13:59:47.961520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435904869391822:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:47.961528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:47.961807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435904869391819:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40743
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.962084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435904772915360:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57471
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.962317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435906505837984:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19246
2025-11-19T13:59:47.962432Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435904869391793:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:47.962653Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435904772915352:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:47.962762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435904772915261:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.962777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:47.962783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435904772915261:2049] [node 2] ICP22 created new session: [4:7574435904772915361:2049]
2025-11-19T13:59:47.962790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435904772915361:2049] [node 2] ICS09 handshake done sender: [4:7574435904772915352:2119] self: [4:7574435904772915359:0] peer: [2:7574435904869391792:0] socket: 648 qp: -1
2025-11-19T13:59:47.962795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435904772915361:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:47.962802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.963674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435906487965476:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5137
2025-11-19T13:59:47.963951Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435907145302816:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:47.963961Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435904869391805:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:47.964052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435907145302729:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.964066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:47.964071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435907145302729:2050] [node 3] ICP22 created new session: [6:7574435907145302825:2050]
2025-11-19T13:59:47.964244Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435906505837968:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:47.964076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435907145302825:2050] [node 3] ICS09 handshake done sender: [6:7574435907145302816:2120] self: [6:7574435907145302823:0] peer: [3:7574435906487965452:0] socket: 616 qp: -1
2025-11-19T13:59:47.964082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435907145302825:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:47.964088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:47.964324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435906505837884:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.964336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:47.964343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435906505837884:2049] [node 2] ICP22 created new session: [5:7574435906505837985:2049]
2025-11-19T13:59:47.964349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435906505837985:2049] [node 2] ICS09 handshake done sender: [5:7574435906505837968:2119] self: [5:7574435906505837981:0] peer: [2:7574435904869391804:0] socket: 650 qp: -1
2025-11-19T13:59:47.964356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435906505837985:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:47.964363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:47.964490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435904869391791:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.964510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:47.964515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435904869391791:2107] [node 4] ICP22 created new session: [2:7574435904869391824:2107]
2025-11-19T13:59:47.964520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435904869391824:2107] [node 4] ICS09 handshake done sender: [2:7574435904869391793:2108] self: [2:7574435904869391792:0] peer: [4:7574435904772915359:0] socket: 645 qp: -1
2025-11-19T13:59:47.964524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435904869391824:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:47.964530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:47.965280Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435904869391819:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.965289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435904869391798:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.965298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:47.965301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435904869391798:2109] [node 5] ICP22 created new session: [2:7574435904869391826:2109]
2025-11-19T13:59:47.965305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435904869391826:2109] [node 5] ICS09 handshake done sender: [2:7574435904869391805:2116] self: [2:7574435904869391804:0] peer: [5:7574435906505837981:0] socket: 649 qp: -1
2025-11-19T13:59:47.965309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435904869391826:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:47.965314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.978498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435907145302827:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64617
2025-11-19T13:59:47.979985Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435906487965476:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.980030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435906487965378:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.981453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435906487965478:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5137
2025-11-19T13:59:47.981707Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435906487965476:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:47.980662Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435904869391807:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.980677Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435904869391807:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.981552Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435904869391807:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:47.981764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435906487965378:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.981782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:47.981787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435906487965378:2049] [node 2] ICP22 created new session: [3:7574435906487965479:2049]
2025-11-19T13:59:47.981794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435906487965479:2049] [node 2] ICS09 handshake done sender: [3:7574435906487965476:2120] self: [3:7574435906487965477:0] peer: [2:7574435904869391806:0] socket: 667 qp: -1
2025-11-19T13:59:47.981800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435906487965479:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:47.981810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:47.983700Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435907145302827:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.983776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435907145302728:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:47.983342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435904869391731:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.983365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:47.983371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435904869391731:2049] [node 3] ICP22 created new session: [2:7574435904869391831:2049]
2025-11-19T13:59:47.983376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435904869391831:2049] [node 3] ICS09 handshake done sender: [2:7574435904869391807:2117] self: [2:7574435904869391806:0] peer: [3:7574435906487965477:0] socket: 652 qp: -1
2025-11-19T13:59:47.983382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435904869391831:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:47.983390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:47.989147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435907145302829:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64617
2025-11-19T13:59:47.988074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435904869391809:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:47.988085Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435904869391809:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:47.994465Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435907145302827:2123] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:47.994544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435907145302728:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:47.994566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:47.994572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435907145302728:2049] [node 2] ICP22 created new session: [6:7574435907145302830:2049]
2025-11-19T13:59:47.994578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435907145302830:2049] [node 2] ICS09 handshake done sender: [6:7574435907145302827:2123] self: [6:7574435907145302828:0] peer: [2:7574435904869391808:0] socket: 611 qp: -1
2025-11-19T13:59:47.994584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435907145302830:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:47.994596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:47.994228Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435904869391809:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:47.995816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435904869391803:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:47.995832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:47.995837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435904869391803:2110] [node 6] ICP22 created new session: [2:7574435904869391834:2110]
2025-11-19T13:59:47.995843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435904869391834:2110] [node 6] ICS09 handshake done sender: [2:7574435904869391809:2118] self: [2:7574435904869391808:0] peer: [6:7574435907145302828:0] socket: 609 qp: -1
2025-11-19T13:59:47.995848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435904869391834:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:47.995856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:48.007662Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:48.007784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435906487965466:2048] [node 1] ICS01 socket: 627 reason# EndOfStream
2025-11-19T13:59:48.007795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435906487965377:2048] [node 1] ICP30 unregister session Session# [3:7574435906487965466:2048] VirtualId# [3:7574435906487965461:0]
2025-11-19T13:59:48.007802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435906487965466:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.007828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.007848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.007819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:48.007911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435904869391822:2048] [node 1] ICS01 socket: 636 reason# EndOfStream
2025-11-19T13:59:48.007918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435904869391730:2048] [node 1] ICP30 unregister session Session# [2:7574435904869391822:2048] VirtualId# [2:7574435904869391815:0]
2025-11-19T13:59:48.007925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435904869391822:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.007965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.008009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.016012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:48.016111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435907145302821:2048] [node 1] ICS01 socket: 657 reason# EndOfStream
2025-11-19T13:59:48.016121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435907145302727:2048] [node 1] ICP30 unregister session Session# [6:7574435907145302821:2048] VirtualId# [6:7574435907145302814:0]
2025-11-19T13:59:48.016129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435907145302821:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.016178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.016201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.017715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:48.017791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435906505837982:2048] [node 1] ICS01 socket: 662 reason# EndOfStream
2025-11-19T13:59:48.017799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435906505837883:2048] [node 1] ICP30 unregister session Session# [5:7574435906505837982:2048] VirtualId# [5:7574435906505837979:0]
2025-11-19T13:59:48.017806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435906505837982:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.017846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.017867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.017948Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:48.018009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435904772915355:2048] [node 1] ICS01 socket: 632 reason# EndOfStream
2025-11-19T13:59:48.018013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435904772915260:2048] [node 1] ICP30 unregister session Session# [4:7574435904772915355:2048] VirtualId# [4:7574435904772915350:0]
2025-11-19T13:59:48.018018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435904772915355:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.018047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.018066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.026192Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:48.026321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435906487965479:2049] [node 2] ICS01 socket: 667 reason# EndOfStream
2025-11-19T13:59:48.026330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435906487965378:2049] [node 2] ICP30 unregister session Session# [3:7574435906487965479:2049] VirtualId# [3:7574435906487965477:0]
2025-11-19T13:59:48.026338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435906487965479:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.026386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:48.026409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:48.038483Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:48.038575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435904772915361:2049] [node 2] ICS01 socket: 648 reason# EndOfStream
2025-11-19T13:59:48.038584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435904772915261:2049] [node 2] ICP30 unregister session Session# [4:7574435904772915361:2049] VirtualId# [4:7574435904772915359:0]
2025-11-19T13:59:48.038591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435904772915361:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.038634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:48.038656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:48.038690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:48.038752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435906505837985:2049] [node 2] ICS01 socket: 650 reason# EndOfStream
2025-11-19T13:59:48.038756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435906505837884:2049] [node 2] ICP30 unregister session Session# [5:7574435906505837985:2049] VirtualId# [5:7574435906505837981:0]
2025-11-19T13:59:48.038761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435906505837985:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.038783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:48.038802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:48.039981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:48.040065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435907145302830:2049] [node 2] ICS01 socket: 611 reason# EndOfStream
2025-11-19T13:59:48.040072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435907145302728:2049] [node 2] ICP30 unregister session Session# [6:7574435907145302830:2049] VirtualId# [6:7574435907145302828:0]
2025-11-19T13:59:48.040078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435907145302830:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.040121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:48.040141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:48.054316Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:48.054408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435907145302825:2050] [node 3] ICS01 socket: 616 reason# EndOfStream
2025-11-19T13:59:48.054417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435907145302729:2050] [node 3] ICP30 unregister session Session# [6:7574435907145302825:2050] VirtualId# [6:7574435907145302823:0]
2025-11-19T13:59:48.054427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435907145302825:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.054478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:48.054499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:48.062029Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:48.062137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435906505837977:2050] [node 3] ICS01 socket: 631 reason# EndOfStream
2025-11-19T13:59:48.062146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435906505837885:2050] [node 3] ICP30 unregister session Session# [5:7574435906505837977:2050] VirtualId# [5:7574435906505837970:0]
2025-11-19T13:59:48.062154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435906505837977:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.062213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:48.062239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:48.068197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:48.068287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435907145302812:2067] [node 4] ICS01 socket: 623 reason# EndOfStream
2025-11-19T13:59:48.068297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435907145302753:2067] [node 4] ICP30 unregister session Session# [6:7574435907145302812:2067] VirtualId# [6:7574435907145302807:0]
2025-11-19T13:59:48.068305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435907145302812:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.068362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:48.068386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:48.159930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435912088020426:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:48.159949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435912088020427:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:48.196077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435911195113840:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:48.196091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435911195113841:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:48.228069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435913067683960:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:48.228097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435913067683961:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:48.264706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435910221577989:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:48.264727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435910221577990:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:48.264732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435910221577991:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:48.283934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435912219444096:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:48.283964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435912219444097:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:48.283969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435912219444098:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:48.296531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435912664155244:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:48.296543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435912664155245:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:48.296548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435912664155246:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:48.336145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435912219444148:2098] [node 6] ICP01 ready to work
2025-11-19T13:59:48.336267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435912664155260:2061] [node 5] ICP01 ready to work
2025-11-19T13:59:48.337340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435912088020491:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.338159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435911195113898:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11624
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.338325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435910221578050:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38453
2025-11-19T13:59:48.342706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435911195113901:2106] [node 5] ICP01 ready to work
2025-11-19T13:59:48.342784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435911195113902:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:48.343948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435912088020497:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.352523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435912664155309:2074] [node 4] ICP01 ready to work
2025-11-19T13:59:48.353132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435912664155310:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45594
2025-11-19T13:59:48.355855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435912664155245:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19241017ns) {Configure:104(19237841ns) {ProcessConfigured:135(13915899ns) {StartInitialHandshake:157(13914462ns) {DropHandshakes:469(288ns) {DropIncomingHandshake:441(117ns) DropOutgoingHandshake:456(14ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(1346ns) {SwitchToState:178(711ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.362061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435912219444097:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(22003968ns) {Configure:104(22003142ns) {ProcessConfigured:135(21660640ns) {StartInitialHandshake:157(21659952ns) {DropHandshakes:469(560ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(1145ns) {SwitchToState:178(812ns)}}}}}
2025-11-19T13:59:48.363402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435912219444164:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.367643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435913067684033:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:48.367731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435913067684034:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:48.368404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913067684030:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3933
2025-11-19T13:59:48.368416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913067684031:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3933
2025-11-19T13:59:48.368470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913067684032:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3933
2025-11-19T13:59:48.368895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435912088020497:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(24894909ns) {Configure:104(24894023ns) {ProcessConfigured:135(24586131ns) {StartInitialHandshake:157(24586003ns) {DropHandshakes:469(419ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(46ns) PrepareNewSessionHandshake:476(574ns) {SwitchToState:178(244ns)}}}}}
2025-11-19T13:59:48.369068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435912088020504:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:48.369502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435912088020502:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8501
2025-11-19T13:59:48.369555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435912088020503:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8501
2025-11-19T13:59:48.363816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435912219444098:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(20575205ns) {Configure:104(20573850ns) {ProcessConfigured:135(20175702ns) {StartInitialHandshake:157(20174894ns) {DropHandshakes:469(504ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:476(1236ns) {SwitchToState:178(584ns)}}}}}
2025-11-19T13:59:48.365367Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435912219444164:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.371626Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435912664155310:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.371958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435912664155260:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.379800Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435912219444150:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.379828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435912219444150:2099] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.381506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435912664155324:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45594
2025-11-19T13:59:48.376570Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435912088020503:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.376581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435913067684027:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:48.376740Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913067684031:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.376765Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913067684032:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.381389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913067684041:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3933
2025-11-19T13:59:48.384093Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435912664155324:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.384133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435912664155246:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.381623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435911195113840:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(38798469ns) {Configure:104(38797890ns) {ProcessConfigured:135(38412618ns) {StartInitialHandshake:157(38411589ns) {DropHandshakes:469(431ns) {DropIncomingHandshake:441(77ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(82ns) PrepareNewSessionHandshake:476(1239ns) {SwitchToState:178(1000ns)}}}}}
2025-11-19T13:59:48.382141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435911195113910:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:48.382740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435911195113913:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11624
2025-11-19T13:59:48.382789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435911195113914:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11624
2025-11-19T13:59:48.382800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435911195113915:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11624
2025-11-19T13:59:48.384315Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435911195113913:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.388116Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435912088020502:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.388076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435912219444172:2113] [node 4] ICP01 ready to work
2025-11-19T13:59:48.389221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435910221578060:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:48.389356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435910221578062:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.388875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435911195113901:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(43936062ns) {Configure:104(43935410ns) {ProcessConfigured:135(43627241ns) {StartInitialHandshake:157(43626637ns) {DropHandshakes:469(495ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:476(872ns) {SwitchToState:178(491ns)}}}}}
2025-11-19T13:59:48.388964Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435911195113915:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.393623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435912664155326:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45594
2025-11-19T13:59:48.395110Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435912664155324:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:48.392360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435910221578061:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38453
2025-11-19T13:59:48.392579Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913067684036:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.392593Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435913067684036:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.394869Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435913067684036:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:48.395217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435912664155246:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.395237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:48.395244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435912664155246:2050] [node 3] ICP22 created new session: [6:7574435912664155327:2050]
2025-11-19T13:59:48.395251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435912664155327:2050] [node 3] ICS09 handshake done sender: [6:7574435912664155324:2116] self: [6:7574435912664155325:0] peer: [3:7574435913067684035:0] socket: 636 qp: -1
2025-11-19T13:59:48.395259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435912664155327:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:48.395268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:48.400099Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435911195113914:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.404045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435913067684033:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.404076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:48.404085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435913067684033:2110] [node 6] ICP22 created new session: [3:7574435913067684047:2110]
2025-11-19T13:59:48.404095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435913067684047:2110] [node 6] ICS09 handshake done sender: [3:7574435913067684036:2115] self: [3:7574435913067684035:0] peer: [6:7574435912664155325:0] socket: 632 qp: -1
2025-11-19T13:59:48.404102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435913067684047:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:48.404113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:48.404336Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913067684041:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.407371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913067684046:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3933
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.409410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435912088020508:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8501
2025-11-19T13:59:48.409998Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435911195113898:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.410036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435911195113840:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.411965Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435912088020508:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.415852Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435912088020486:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.415878Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435912088020486:2106] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.420075Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913067684046:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.420124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435913067683961:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.420916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435912664155330:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45594
2025-11-19T13:59:48.421498Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435912664155310:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:48.421591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435912664155260:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.421608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:48.421618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435912664155260:2061] [node 5] ICP22 created new session: [6:7574435912664155331:2061]
2025-11-19T13:59:48.421624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435912664155331:2061] [node 5] ICS09 handshake done sender: [6:7574435912664155310:2108] self: [6:7574435912664155323:0] peer: [5:7574435912219444149:0] socket: 619 qp: -1
2025-11-19T13:59:48.421633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435912664155331:2061] [node 5] ICS10 traffic start
2025-11-19T13:59:48.421644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.420676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435912219444175:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
2025-11-19T13:59:48.420702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435912219444176:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
2025-11-19T13:59:48.421287Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435912219444150:2099] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:48.423678Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435912219444176:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.423690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435912219444148:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.423716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:48.423723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435912219444148:2098] [node 6] ICP22 created new session: [5:7574435912219444178:2098]
2025-11-19T13:59:48.423728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435912219444178:2098] [node 6] ICS09 handshake done sender: [5:7574435912219444150:2099] self: [5:7574435912219444149:0] peer: [6:7574435912664155323:0] socket: 613 qp: -1
2025-11-19T13:59:48.423734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435912219444178:2098] [node 6] ICS10 traffic start
2025-11-19T13:59:48.423758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:48.424039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435912219444172:2113] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(35893825ns) {Configure:104(35892485ns) {ProcessConfigured:135(35521904ns) {StartInitialHandshake:157(35521277ns) {DropHandshakes:469(1014ns) {DropIncomingHandshake:441(42ns) DropOutgoingHandshake:456(433ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:476(801ns) {SwitchToState:178(291ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.423992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435911195113927:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11624
2025-11-19T13:59:48.424081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435910221577989:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(51659923ns) {Configure:104(51658940ns) {ProcessConfigured:135(51356007ns) {StartInitialHandshake:157(51355316ns) {DropHandshakes:469(680ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(137ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:476(1048ns) {SwitchToState:178(686ns)}}}}}
2025-11-19T13:59:48.425049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913067684030:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.425077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435913067683960:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.425446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435910221578069:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38453
2025-11-19T13:59:48.425499Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435910221578050:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.425516Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435910221578061:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.425535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435910221577989:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.425543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435910221577990:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.425962Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435911195113927:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.426031Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435910221578069:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.427796Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435911195113912:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.427812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435911195113912:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.431650Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435912088020493:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.431665Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435912088020493:2109] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.433306Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435912088020490:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.433318Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435912088020490:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.433523Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435912088020486:2106] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:48.432596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435912219444098:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.433022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435910221578073:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38453
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.436000Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435911195113909:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.436014Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435911195113909:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.436652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435911195113929:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11624
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.437286Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435911195113909:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:48.437386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435911195113902:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.437407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:48.437413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435911195113902:2107] [node 4] ICP22 created new session: [2:7574435911195113931:2107]
2025-11-19T13:59:48.437420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435911195113931:2107] [node 4] ICS09 handshake done sender: [2:7574435911195113909:2111] self: [2:7574435911195113908:0] peer: [4:7574435910221578071:0] socket: 641 qp: -1
2025-11-19T13:59:48.437425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435911195113931:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:48.437435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:48.437545Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435911195113898:2105] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:48.437808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435911195113840:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.437818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:48.437823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435911195113840:2048] [node 1] ICP22 created new session: [2:7574435911195113933:2048]
2025-11-19T13:59:48.437827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435911195113933:2048] [node 1] ICS09 handshake done sender: [2:7574435911195113898:2105] self: [2:7574435911195113926:0] peer: [1:7574435912088020485:0] socket: 614 qp: -1
2025-11-19T13:59:48.437831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435911195113933:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:48.437836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:48.438667Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435912219444175:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.436978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435910221578074:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38453
2025-11-19T13:59:48.441582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435912219444172:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.448244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913067684053:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3933
2025-11-19T13:59:48.448407Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435911195113912:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:48.448774Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435912088020493:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:48.451818Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913067684038:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.451836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435913067684038:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.451994Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435913067684046:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.452503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435911195113841:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.452533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:48.452541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435911195113841:2049] [node 3] ICP22 created new session: [2:7574435911195113938:2049]
2025-11-19T13:59:48.452549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435911195113938:2049] [node 3] ICS09 handshake done sender: [2:7574435911195113912:2112] self: [2:7574435911195113911:0] peer: [3:7574435913067684051:0] socket: 642 qp: -1
2025-11-19T13:59:48.452556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435911195113938:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:48.452566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.453252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435912664155334:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45594
2025-11-19T13:59:48.452920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435912219444184:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
2025-11-19T13:59:48.453880Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435912664155334:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.453924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435912664155245:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.454317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435910221577990:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26669) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435910221578061:2113] held: no
2025-11-19T13:59:48.454295Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435911195113921:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.454303Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435911195113921:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.455110Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435911195113921:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:48.454040Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435912219444184:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.454075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435912219444097:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.455466Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435911195113917:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.455480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435911195113917:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.455573Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435910221578064:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.455581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435910221578064:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.456142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435913067683961:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.456165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:48.456171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435913067683961:2049] [node 2] ICP22 created new session: [3:7574435913067684054:2049]
2025-11-19T13:59:48.456178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435913067684054:2049] [node 2] ICS09 handshake done sender: [3:7574435913067684046:2119] self: [3:7574435913067684051:0] peer: [2:7574435911195113911:0] socket: 643 qp: -1
2025-11-19T13:59:48.456186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435913067684054:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:48.456194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.459687Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435910221578061:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:48.459726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435911195113901:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.459426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435913067683961:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:48.459651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435912219444186:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
2025-11-19T13:59:48.460142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:48.460151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435911195113901:2106] [node 5] ICP22 created new session: [2:7574435911195113941:2106]
2025-11-19T13:59:48.460157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435911195113941:2106] [node 5] ICS09 handshake done sender: [2:7574435911195113921:2117] self: [2:7574435911195113905:0] peer: [5:7574435912219444185:0] socket: 640 qp: -1
2025-11-19T13:59:48.460164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435911195113941:2106] [node 5] ICS10 traffic start
2025-11-19T13:59:48.460173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:48.460613Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435910221578050:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.461690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435910221577989:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.461704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:48.461708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435910221577989:2048] [node 1] ICP22 created new session: [4:7574435910221578079:2048]
2025-11-19T13:59:48.461713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435910221578079:2048] [node 1] ICS09 handshake done sender: [4:7574435910221578050:2108] self: [4:7574435910221578070:0] peer: [1:7574435912088020492:0] socket: 618 qp: -1
2025-11-19T13:59:48.461717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435910221578079:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:48.461723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:48.461815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435910221578076:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38453
2025-11-19T13:59:48.461979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435910221578078:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38453
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.462126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435912664155336:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45594
2025-11-19T13:59:48.460077Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435912219444184:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:48.460609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435912219444097:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.460620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:48.460626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435912219444097:2049] [node 2] ICP22 created new session: [5:7574435912219444187:2049]
2025-11-19T13:59:48.460631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435912219444187:2049] [node 2] ICS09 handshake done sender: [5:7574435912219444184:2118] self: [5:7574435912219444185:0] peer: [2:7574435911195113905:0] socket: 661 qp: -1
2025-11-19T13:59:48.460636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435912219444187:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:48.460642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:48.461544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435912219444189:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.465518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435912088020491:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.465534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:48.465542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435912088020491:2107] [node 4] ICP22 created new session: [1:7574435912088020515:2107]
2025-11-19T13:59:48.465547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435912088020515:2107] [node 4] ICS09 handshake done sender: [1:7574435912088020493:2109] self: [1:7574435912088020492:0] peer: [4:7574435910221578070:0] socket: 616 qp: -1
2025-11-19T13:59:48.465552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435912088020515:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:48.465559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:48.465798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435912088020513:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8501
2025-11-19T13:59:48.466051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435912088020426:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.466070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:48.466076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435912088020426:2048] [node 2] ICP22 created new session: [1:7574435912088020519:2048]
2025-11-19T13:59:48.466081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435912088020519:2048] [node 2] ICS09 handshake done sender: [1:7574435912088020486:2106] self: [1:7574435912088020485:0] peer: [2:7574435911195113926:0] socket: 612 qp: -1
2025-11-19T13:59:48.466086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435912088020519:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:48.466092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:48.466228Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435912088020490:2108] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.470600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435912664155337:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45594
2025-11-19T13:59:48.470626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435912664155338:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45594
2025-11-19T13:59:48.470811Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435910221578064:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:48.471128Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435911195113917:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:48.471519Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435912664155334:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:48.471421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435910221578060:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.471438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:48.471445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435910221578060:2111] [node 5] ICP22 created new session: [4:7574435910221578081:2111]
2025-11-19T13:59:48.471451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435910221578081:2111] [node 5] ICS09 handshake done sender: [4:7574435910221578064:2114] self: [4:7574435910221578063:0] peer: [5:7574435912219444183:0] socket: 646 qp: -1
2025-11-19T13:59:48.471457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435910221578081:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:48.471465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:48.471623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435912664155245:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.471636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:48.471642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435912664155245:2049] [node 2] ICP22 created new session: [6:7574435912664155339:2049]
2025-11-19T13:59:48.471647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435912664155339:2049] [node 2] ICS09 handshake done sender: [6:7574435912664155334:2119] self: [6:7574435912664155335:0] peer: [2:7574435911195113916:0] socket: 662 qp: -1
2025-11-19T13:59:48.471652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435912664155339:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:48.471660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.472760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435912088020427:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.472781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:48.472787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435912088020427:2049] [node 3] ICP22 created new session: [1:7574435912088020521:2049]
2025-11-19T13:59:48.472792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435912088020521:2049] [node 3] ICS09 handshake done sender: [1:7574435912088020490:2108] self: [1:7574435912088020489:0] peer: [3:7574435913067684052:0] socket: 615 qp: -1
2025-11-19T13:59:48.472796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435912088020521:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:48.472802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:48.473974Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435913067684038:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:48.474345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913067684058:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3933
2025-11-19T13:59:48.473408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435910221577990:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.473427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:48.473435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435910221577990:2049] [node 2] ICP22 created new session: [4:7574435910221578083:2049]
2025-11-19T13:59:48.473441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435910221578083:2049] [node 2] ICS09 handshake done sender: [4:7574435910221578061:2113] self: [4:7574435910221578071:0] peer: [2:7574435911195113908:0] socket: 645 qp: -1
2025-11-19T13:59:48.473447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435910221578083:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:48.473469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:48.474767Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435910221578076:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.474814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435910221577991:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.475439Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913067684044:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.475447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435913067684044:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.476052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435913067684034:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.476066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:48.476070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435913067684034:2111] [node 5] ICP22 created new session: [3:7574435913067684059:2111]
2025-11-19T13:59:48.476075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435913067684059:2111] [node 5] ICS09 handshake done sender: [3:7574435913067684038:2116] self: [3:7574435913067684037:0] peer: [5:7574435912219444182:0] socket: 624 qp: -1
2025-11-19T13:59:48.476079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435913067684059:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:48.476085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:48.476449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435911195113910:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.476467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:48.476472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435911195113910:2108] [node 6] ICP22 created new session: [2:7574435911195113944:2108]
2025-11-19T13:59:48.476477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435911195113944:2108] [node 6] ICS09 handshake done sender: [2:7574435911195113917:2116] self: [2:7574435911195113916:0] peer: [6:7574435912664155335:0] socket: 638 qp: -1
2025-11-19T13:59:48.476481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435911195113944:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:48.476488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:48.477173Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435913067684044:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:48.477220Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435913067684030:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:48.477280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435913067684027:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.477296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:48.477300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435913067684027:2109] [node 4] ICP22 created new session: [3:7574435913067684062:2109]
2025-11-19T13:59:48.477305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435913067684062:2109] [node 4] ICS09 handshake done sender: [3:7574435913067684044:2118] self: [3:7574435913067684040:0] peer: [4:7574435910221578085:0] socket: 621 qp: -1
2025-11-19T13:59:48.477309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435913067684062:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:48.477314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:48.477564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435913067683960:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.477574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:48.477578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435913067683960:2048] [node 1] ICP22 created new session: [3:7574435913067684064:2048]
2025-11-19T13:59:48.477582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435913067684064:2048] [node 1] ICS09 handshake done sender: [3:7574435913067684030:2112] self: [3:7574435913067684052:0] peer: [1:7574435912088020489:0] socket: 617 qp: -1
2025-11-19T13:59:48.477585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435913067684064:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:48.477590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.480655Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435912664155336:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.480689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435912664155309:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.478840Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435912219444175:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:48.478898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435912219444098:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23823) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435912219444176:2116] held: no
2025-11-19T13:59:48.478929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435912219444191:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
2025-11-19T13:59:48.479072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435912219444192:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
2025-11-19T13:59:48.479106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435912219444172:2113] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.479115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:48.479119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435912219444172:2113] [node 4] ICP22 created new session: [5:7574435912219444193:2113]
2025-11-19T13:59:48.479124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435912219444193:2113] [node 4] ICS09 handshake done sender: [5:7574435912219444175:2115] self: [5:7574435912219444183:0] peer: [4:7574435910221578063:0] socket: 629 qp: -1
2025-11-19T13:59:48.479128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435912219444193:2113] [node 4] ICS10 traffic start
2025-11-19T13:59:48.479134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:48.481024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435910221578078:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.481109Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435910221577991:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23823) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435910221578076:2120] held: no
2025-11-19T13:59:48.481143Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435910221578066:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.481146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435910221578066:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.481920Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435912088020513:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.482166Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435912664155338:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.482189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435912664155244:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.482274Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435912088020506:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.482279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435912088020506:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.483070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435912664155343:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45594
2025-11-19T13:59:48.483167Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435912088020506:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:48.483397Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435912664155338:2122] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:48.483462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435912664155244:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.483476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:48.483481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435912664155244:2048] [node 1] ICP22 created new session: [6:7574435912664155344:2048]
2025-11-19T13:59:48.483486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435912664155344:2048] [node 1] ICS09 handshake done sender: [6:7574435912664155338:2122] self: [6:7574435912664155342:0] peer: [1:7574435912088020505:0] socket: 679 qp: -1
2025-11-19T13:59:48.483491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435912664155344:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:48.483498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:48.484564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435912088020504:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.484581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:48.484586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435912088020504:2111] [node 6] ICP22 created new session: [1:7574435912088020525:2111]
2025-11-19T13:59:48.484591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435912088020525:2111] [node 6] ICS09 handshake done sender: [1:7574435912088020506:2116] self: [1:7574435912088020505:0] peer: [6:7574435912664155342:0] socket: 677 qp: -1
2025-11-19T13:59:48.484595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435912088020525:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:48.484603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.488193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435910221578086:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38453
2025-11-19T13:59:48.489451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435910221578060:2111] [node 5] ICP18 handshake (actor [4:7574435910221578078:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.489731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435912664155346:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45594
2025-11-19T13:59:48.490183Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435912088020501:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.490190Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435912088020501:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.490972Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435910221578066:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:48.491168Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435912664155336:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:48.491298Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435910221578076:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:48.491393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435910221578062:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.491413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:48.491418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435910221578062:2112] [node 6] ICP22 created new session: [4:7574435910221578088:2112]
2025-11-19T13:59:48.491423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435910221578088:2112] [node 6] ICS09 handshake done sender: [4:7574435910221578066:2115] self: [4:7574435910221578065:0] peer: [6:7574435912664155341:0] socket: 671 qp: -1
2025-11-19T13:59:48.491428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435910221578088:2112] [node 6] ICS10 traffic start
2025-11-19T13:59:48.491435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:48.491779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435912664155309:2074] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.491793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:48.491798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435912664155309:2074] [node 4] ICP22 created new session: [6:7574435912664155347:2074]
2025-11-19T13:59:48.491803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435912664155347:2074] [node 4] ICS09 handshake done sender: [6:7574435912664155336:2120] self: [6:7574435912664155341:0] peer: [4:7574435910221578065:0] socket: 672 qp: -1
2025-11-19T13:59:48.491807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435912664155347:2074] [node 4] ICS10 traffic start
2025-11-19T13:59:48.491813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:48.492759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435910221577991:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.492770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:48.492775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435910221577991:2050] [node 3] ICP22 created new session: [4:7574435910221578090:2050]
2025-11-19T13:59:48.492780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435910221578090:2050] [node 3] ICS09 handshake done sender: [4:7574435910221578076:2120] self: [4:7574435910221578085:0] peer: [3:7574435913067684040:0] socket: 625 qp: -1
2025-11-19T13:59:48.492784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435910221578090:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:48.492790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.493391Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435912088020501:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:48.494699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435912088020497:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.494720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:48.494725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435912088020497:2110] [node 5] ICP22 created new session: [1:7574435912088020528:2110]
2025-11-19T13:59:48.494730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435912088020528:2110] [node 5] ICS09 handshake done sender: [1:7574435912088020501:2113] self: [1:7574435912088020498:0] peer: [5:7574435912219444197:0] socket: 676 qp: -1
2025-11-19T13:59:48.494735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435912088020528:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:48.494741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:48.489369Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435912219444192:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.489385Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435912219444176:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:48.489843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435912219444098:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.489852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:48.489857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435912219444098:2050] [node 3] ICP22 created new session: [5:7574435912219444195:2050]
2025-11-19T13:59:48.489862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435912219444195:2050] [node 3] ICS09 handshake done sender: [5:7574435912219444176:2116] self: [5:7574435912219444182:0] peer: [3:7574435913067684037:0] socket: 650 qp: -1
2025-11-19T13:59:48.489867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435912219444195:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:48.489873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:48.489976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435912219444096:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.493116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435912219444198:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56361
2025-11-19T13:59:48.493557Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435912219444192:2122] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:48.493820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435912219444096:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.493830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:48.493835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435912219444096:2048] [node 1] ICP22 created new session: [5:7574435912219444199:2048]
2025-11-19T13:59:48.493839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435912219444199:2048] [node 1] ICS09 handshake done sender: [5:7574435912219444192:2122] self: [5:7574435912219444197:0] peer: [1:7574435912088020498:0] socket: 639 qp: -1
2025-11-19T13:59:48.493844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435912219444199:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:48.493850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:48.502760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:48.502843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435911195113933:2048] [node 1] ICS01 socket: 614 reason# EndOfStream
2025-11-19T13:59:48.502852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435911195113840:2048] [node 1] ICP30 unregister session Session# [2:7574435911195113933:2048] VirtualId# [2:7574435911195113926:0]
2025-11-19T13:59:48.502860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435911195113933:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.502900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.502919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.503454Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:48.503516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435913067684064:2048] [node 1] ICS01 socket: 617 reason# EndOfStream
2025-11-19T13:59:48.503521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435913067683960:2048] [node 1] ICP30 unregister session Session# [3:7574435913067684064:2048] VirtualId# [3:7574435913067684052:0]
2025-11-19T13:59:48.503526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435913067684064:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.503553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.503569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.515843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:48.515929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435912219444199:2048] [node 1] ICS01 socket: 639 reason# EndOfStream
2025-11-19T13:59:48.515937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435912219444096:2048] [node 1] ICP30 unregister session Session# [5:7574435912219444199:2048] VirtualId# [5:7574435912219444197:0]
2025-11-19T13:59:48.515943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435912219444199:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.514186Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:48.514273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435910221578079:2048] [node 1] ICS01 socket: 618 reason# EndOfStream
2025-11-19T13:59:48.514283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435910221577989:2048] [node 1] ICP30 unregister session Session# [4:7574435910221578079:2048] VirtualId# [4:7574435910221578070:0]
2025-11-19T13:59:48.514291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435910221578079:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.514343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.514368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.516058Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:48.516120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435912664155344:2048] [node 1] ICS01 socket: 679 reason# EndOfStream
2025-11-19T13:59:48.516125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435912664155244:2048] [node 1] ICP30 unregister session Session# [6:7574435912664155344:2048] VirtualId# [6:7574435912664155342:0]
2025-11-19T13:59:48.516129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435912664155344:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.516156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.516178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.515995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.516022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.531863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:48.531960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435913067684054:2049] [node 2] ICS01 socket: 643 reason# EndOfStream
2025-11-19T13:59:48.531977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435913067683961:2049] [node 2] ICP30 unregister session Session# [3:7574435913067684054:2049] VirtualId# [3:7574435913067684051:0]
2025-11-19T13:59:48.531986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435913067684054:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.532035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:48.532057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:48.539874Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:48.539971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435910221578083:2049] [node 2] ICS01 socket: 645 reason# EndOfStream
2025-11-19T13:59:48.539980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435910221577990:2049] [node 2] ICP30 unregister session Session# [4:7574435910221578083:2049] VirtualId# [4:7574435910221578071:0]
2025-11-19T13:59:48.539989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435910221578083:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.540048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:48.540074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:48.542313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:48.542405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435912664155339:2049] [node 2] ICS01 socket: 662 reason# EndOfStream
2025-11-19T13:59:48.542412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435912664155245:2049] [node 2] ICP30 unregister session Session# [6:7574435912664155339:2049] VirtualId# [6:7574435912664155335:0]
2025-11-19T13:59:48.542418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435912664155339:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.542462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:48.542484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:48.542562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:48.542624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435912219444187:2049] [node 2] ICS01 socket: 661 reason# EndOfStream
2025-11-19T13:59:48.542628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435912219444097:2049] [node 2] ICP30 unregister session Session# [5:7574435912219444187:2049] VirtualId# [5:7574435912219444185:0]
2025-11-19T13:59:48.542632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435912219444187:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.542658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:48.542677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:48.566739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:48.566806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435912664155327:2050] [node 3] ICS01 socket: 636 reason# EndOfStream
2025-11-19T13:59:48.566813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435912664155246:2050] [node 3] ICP30 unregister session Session# [6:7574435912664155327:2050] VirtualId# [6:7574435912664155325:0]
2025-11-19T13:59:48.566818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435912664155327:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.566849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:48.566871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:48.566104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:48.566202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435912219444195:2050] [node 3] ICS01 socket: 650 reason# EndOfStream
2025-11-19T13:59:48.566211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435912219444098:2050] [node 3] ICP30 unregister session Session# [5:7574435912219444195:2050] VirtualId# [5:7574435912219444182:0]
2025-11-19T13:59:48.566219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435912219444195:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.566274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:48.566299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:48.577304Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:48.577391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435912664155347:2074] [node 4] ICS01 socket: 672 reason# EndOfStream
2025-11-19T13:59:48.577401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435912664155309:2074] [node 4] ICP30 unregister session Session# [6:7574435912664155347:2074] VirtualId# [6:7574435912664155341:0]
2025-11-19T13:59:48.577407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435912664155347:2074] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.577456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:48.577477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:48.655876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435911353174180:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:48.655888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435911353174181:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:48.675926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435912959786301:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:48.675939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435912959786302:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:48.700288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435911354708111:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:48.700304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435911354708112:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:48.719645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435911310967537:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:48.719661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435911310967538:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:48.719666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435911310967539:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:48.763525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435910206996298:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:48.763543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435910206996299:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:48.763549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435910206996300:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:48.793109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435908989480643:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:48.793123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435908989480644:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:48.793127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435908989480645:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:48.827616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435908989480691:2094] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.828263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435912959786358:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23333
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.828811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435910206996359:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10549
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.830981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435911310967594:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37173
2025-11-19T13:59:48.829091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435908989480701:2101] [node 4] ICP01 ready to work
2025-11-19T13:59:48.836209Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435910206996359:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.836249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435912959786361:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.837572Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435911310967594:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.837023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435911353174246:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:48.837105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435911353174247:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:48.837174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435911353174248:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:48.836360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435910206996361:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:48.836747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435910206996361:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.837523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435910206996363:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10549
2025-11-19T13:59:48.838642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435911310967596:2105] [node 6] ICP01 ready to work
2025-11-19T13:59:48.839070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435911310967596:2105] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.837326Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435908989480697:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.837739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435908989480697:2099] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.840490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435911353174257:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.840886Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435912959786358:2104] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.841054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435912959786369:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:48.841623Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435908989480697:2099] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.847288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435911310967598:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37173
2025-11-19T13:59:48.845285Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435908989480703:2102] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.845299Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435908989480703:2102] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.849016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435912959786373:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:48.852205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435912959786301:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.852338Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435911310967598:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.852884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435908989480691:2094] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.852900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:48.852907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435908989480691:2094] [node 5] ICP22 created new session: [6:7574435908989480715:2094]
2025-11-19T13:59:48.852912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435908989480715:2094] [node 5] ICS09 handshake done sender: [6:7574435908989480697:2099] self: [6:7574435908989480696:0] peer: [5:7574435910206996362:0] socket: 620 qp: -1
2025-11-19T13:59:48.852919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435908989480715:2094] [node 5] ICS10 traffic start
2025-11-19T13:59:48.852927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:48.853088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435908989480714:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25423
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.851889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435910206996365:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.854240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435911310967610:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37173
2025-11-19T13:59:48.854254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435911310967611:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37173
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.853443Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435911353174241:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.853456Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435911353174241:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.856498Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435911353174241:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:48.856614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435911353174180:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.856626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:48.856633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435911353174180:2048] [node 2] ICP22 created new session: [1:7574435911353174259:2048]
2025-11-19T13:59:48.856638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435911353174259:2048] [node 2] ICS09 handshake done sender: [1:7574435911353174241:2107] self: [1:7574435911353174240:0] peer: [2:7574435912959786380:0] socket: 618 qp: -1
2025-11-19T13:59:48.856645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435911353174259:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:48.856652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:48.857694Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435908989480714:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.855881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435910206996369:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10549
2025-11-19T13:59:48.858068Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435910206996363:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.858429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435910206996299:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.858484Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435910206996359:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.861104Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435911353174248:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23478663ns) {Configure:104(23477839ns) {ProcessConfigured:135(23260282ns) {StartInitialHandshake:157(23259455ns) {DropHandshakes:469(195ns) {DropIncomingHandshake:441(32ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:476(888ns) {SwitchToState:178(465ns)}}}}}
2025-11-19T13:59:48.858997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435912959786370:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23333
2025-11-19T13:59:48.860418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435912959786381:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23333
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.864728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435911310967537:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17328555ns) {Configure:104(17326447ns) {ProcessConfigured:135(16986965ns) {StartInitialHandshake:157(16986407ns) {DropHandshakes:469(909ns) {DropIncomingHandshake:441(127ns) DropOutgoingHandshake:456(179ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:476(1090ns) {SwitchToState:178(554ns)}}}}}
2025-11-19T13:59:48.865882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435911310967537:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.866031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435911310967616:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37173
2025-11-19T13:59:48.866598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435912959786382:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23333
2025-11-19T13:59:48.865333Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435908989480703:2102] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:48.865428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435908989480701:2101] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.865440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:48.865448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435908989480701:2101] [node 4] ICP22 created new session: [6:7574435908989480721:2101]
2025-11-19T13:59:48.865455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435908989480721:2101] [node 4] ICS09 handshake done sender: [6:7574435908989480703:2102] self: [6:7574435908989480702:0] peer: [4:7574435911310967597:0] socket: 622 qp: -1
2025-11-19T13:59:48.865462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435908989480721:2101] [node 4] ICS10 traffic start
2025-11-19T13:59:48.865470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:48.866822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435911310967603:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:48.868157Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435912959786358:2104] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:48.867851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435911354708180:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:48.868034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435911354708182:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:48.868394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435911354708179:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:48.870476Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435912959786363:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.870493Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435912959786363:2107] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.870974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435912959786301:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.870987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:48.870992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435912959786301:2048] [node 1] ICP22 created new session: [2:7574435912959786385:2048]
2025-11-19T13:59:48.870998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435912959786385:2048] [node 1] ICS09 handshake done sender: [2:7574435912959786358:2104] self: [2:7574435912959786380:0] peer: [1:7574435911353174240:0] socket: 619 qp: -1
2025-11-19T13:59:48.871005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435912959786385:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:48.871012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:48.871210Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435912959786381:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.872033Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435912959786363:2107] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:48.871854Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435911353174250:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.871870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435911353174250:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.871879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435911353174181:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(34653662ns) {Configure:104(34652932ns) {ProcessConfigured:135(33653114ns) {StartInitialHandshake:157(33652377ns) {DropHandshakes:469(263ns) {DropIncomingHandshake:441(36ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(147ns) PrepareNewSessionHandshake:476(1147ns) {SwitchToState:178(386ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.872810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435908989480644:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19662855ns) {Configure:104(19662357ns) {ProcessConfigured:135(19402808ns) {StartInitialHandshake:157(19402416ns) {DropHandshakes:469(232ns) {DropIncomingHandshake:441(34ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:476(1355ns) {SwitchToState:178(794ns)}}}}}
2025-11-19T13:59:48.872968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435908989480725:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25423
2025-11-19T13:59:48.875800Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435911353174252:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.875823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435911353174252:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.876174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435908989480643:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.876226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435908989480643:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18464307ns) {Configure:104(18463753ns) {ProcessConfigured:135(18175835ns) {IncomingHandshake:261(18164536ns) {DropIncomingHandshake:441(75ns) PrepareNewSessionHandshake:476(434ns) {SwitchToState:178(92ns)} GenerateSessionVirtualId:988(964ns) IssueIncomingHandshakeReply:193(18158940ns) {SwitchToState:178(932ns)}}}}}
2025-11-19T13:59:48.876325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435911310967610:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.876417Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435911310967594:2104] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.877076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435912959786361:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.877095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:48.877103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435912959786361:2106] [node 5] ICP22 created new session: [2:7574435912959786390:2106]
2025-11-19T13:59:48.877110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435912959786390:2106] [node 5] ICS09 handshake done sender: [2:7574435912959786363:2107] self: [2:7574435912959786362:0] peer: [5:7574435910206996370:0] socket: 624 qp: -1
2025-11-19T13:59:48.877118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435912959786390:2106] [node 5] ICS10 traffic start
2025-11-19T13:59:48.877125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:48.878494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435912959786392:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23333
2025-11-19T13:59:48.880088Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435911353174250:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.880257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435911353174246:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.880272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:48.880278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435911353174246:2108] [node 4] ICP22 created new session: [1:7574435911353174265:2108]
2025-11-19T13:59:48.880284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435911353174265:2108] [node 4] ICS09 handshake done sender: [1:7574435911353174250:2111] self: [1:7574435911353174249:0] peer: [4:7574435911310967615:0] socket: 627 qp: -1
2025-11-19T13:59:48.880289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435911353174265:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:48.880296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:48.880174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435911310967596:2105] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.880196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:48.880212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435911310967596:2105] [node 6] ICP22 created new session: [4:7574435911310967621:2105]
2025-11-19T13:59:48.880219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435911310967621:2105] [node 6] ICS09 handshake done sender: [4:7574435911310967594:2104] self: [4:7574435911310967597:0] peer: [6:7574435908989480702:0] socket: 623 qp: -1
2025-11-19T13:59:48.880226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435911310967621:2105] [node 6] ICS10 traffic start
2025-11-19T13:59:48.880249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:48.880475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435911310967538:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.880510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435911310967620:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37173
2025-11-19T13:59:48.880729Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435911310967598:2106] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.882248Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435912959786375:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.882259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435912959786375:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.883000Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435912959786375:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:48.881899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435911310967537:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.881920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:48.881926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435911310967537:2048] [node 1] ICP22 created new session: [4:7574435911310967624:2048]
2025-11-19T13:59:48.881932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435911310967624:2048] [node 1] ICS09 handshake done sender: [4:7574435911310967598:2106] self: [4:7574435911310967615:0] peer: [1:7574435911353174249:0] socket: 634 qp: -1
2025-11-19T13:59:48.881938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435911310967624:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:48.881945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:48.882801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435911310967626:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37173
2025-11-19T13:59:48.883157Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435911310967610:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:48.883204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435911310967538:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.883213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:48.883217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435911310967538:2049] [node 2] ICP22 created new session: [4:7574435911310967627:2049]
2025-11-19T13:59:48.883222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435911310967627:2049] [node 2] ICS09 handshake done sender: [4:7574435911310967610:2112] self: [4:7574435911310967623:0] peer: [2:7574435912959786374:0] socket: 643 qp: -1
2025-11-19T13:59:48.883228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435911310967627:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:48.883235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:48.886652Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435908989480725:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.886689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435908989480644:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.888068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435911353174268:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-19T13:59:48.888134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435910206996371:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10549
2025-11-19T13:59:48.888177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435910206996372:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10549
2025-11-19T13:59:48.888749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435910206996361:2109] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.888778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:48.888788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435910206996361:2109] [node 6] ICP22 created new session: [5:7574435910206996376:2109]
2025-11-19T13:59:48.888796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435910206996376:2109] [node 6] ICS09 handshake done sender: [5:7574435910206996359:2108] self: [5:7574435910206996362:0] peer: [6:7574435908989480696:0] socket: 621 qp: -1
2025-11-19T13:59:48.888805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435910206996376:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:48.888842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:48.889036Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435912959786376:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.889053Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435912959786376:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.893849Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435911353174257:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.893507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435911354708184:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28771
2025-11-19T13:59:48.893187Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435911310967611:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.893775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435908989480730:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25423
2025-11-19T13:59:48.898497Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435911353174252:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:48.898727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435911353174247:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.898738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:48.898745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435911353174247:2110] [node 6] ICP22 created new session: [1:7574435911353174271:2110]
2025-11-19T13:59:48.898751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435911353174271:2110] [node 6] ICS09 handshake done sender: [1:7574435911353174252:2112] self: [1:7574435911353174251:0] peer: [6:7574435908989480718:0] socket: 628 qp: -1
2025-11-19T13:59:48.898756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435911353174271:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:48.898762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:48.896993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435912959786373:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.897019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:48.897025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435912959786373:2111] [node 4] ICP22 created new session: [2:7574435912959786394:2111]
2025-11-19T13:59:48.897033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435912959786394:2111] [node 4] ICS09 handshake done sender: [2:7574435912959786375:2112] self: [2:7574435912959786374:0] peer: [4:7574435911310967623:0] socket: 638 qp: -1
2025-11-19T13:59:48.897039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435912959786394:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:48.897057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:48.904050Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435911354708184:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.904134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435911354708112:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.904498Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435908989480730:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.904536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435908989480645:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.905395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435911353174246:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:48.907958Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435912959786392:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.908012Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435912959786370:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.908047Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435912959786366:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.908051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435912959786366:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.910079Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435912959786366:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:48.910200Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435912959786376:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:48.910229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435912959786302:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.910245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:48.910254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435912959786302:2049] [node 3] ICP22 created new session: [2:7574435912959786400:2049]
2025-11-19T13:59:48.910260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435912959786400:2049] [node 3] ICS09 handshake done sender: [2:7574435912959786366:2108] self: [2:7574435912959786365:0] peer: [3:7574435911354708190:0] socket: 626 qp: -1
2025-11-19T13:59:48.910265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435912959786400:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:48.910272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:48.910584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435912959786369:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.910596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:48.910603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435912959786369:2109] [node 6] ICP22 created new session: [2:7574435912959786402:2109]
2025-11-19T13:59:48.910608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435912959786402:2109] [node 6] ICS09 handshake done sender: [2:7574435912959786376:2113] self: [2:7574435912959786371:0] peer: [6:7574435908989480729:0] socket: 652 qp: -1
2025-11-19T13:59:48.910614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435912959786402:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:48.910620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:48.911785Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435908989480644:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9563) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435908989480725:2114] held: no
2025-11-19T13:59:48.915164Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435911354708180:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(47060681ns) {Configure:104(47059948ns) {ProcessConfigured:135(46750153ns) {StartInitialHandshake:157(46749419ns) {DropHandshakes:469(676ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(47ns)} GenerateSessionVirtualId:988(89ns) PrepareNewSessionHandshake:476(987ns) {SwitchToState:178(291ns)}}}}}
2025-11-19T13:59:48.915896Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435911354708186:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.915909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435911354708186:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.913541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435910206996298:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(33562335ns) {Configure:104(33560969ns) {ProcessConfigured:135(33198224ns) {StartInitialHandshake:157(33197379ns) {DropHandshakes:469(441ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(269ns) PrepareNewSessionHandshake:476(1298ns) {SwitchToState:178(194ns)}}}}}
2025-11-19T13:59:48.913607Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435910206996371:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.913697Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435910206996363:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.922408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435908989480732:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25423
2025-11-19T13:59:48.923223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435908989480733:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25423
2025-11-19T13:59:48.922510Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435910206996361:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(50638518ns) {HandleHandshakeStatus:329(50637617ns) {DropIncomingHandshake:441(841ns) DropOutgoingHandshake:456(123ns) SwitchToState:178(224ns) ProcessPendingSessionEvents:495(24160287ns) {ForwardSessionEventToSession:612(24156038ns) ForwardSessionEventToSession:612(2205ns)}}}
2025-11-19T13:59:48.922563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435910206996299:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.922575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:48.922582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435910206996299:2049] [node 2] ICP22 created new session: [5:7574435910206996381:2049]
2025-11-19T13:59:48.922587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435910206996381:2049] [node 2] ICS09 handshake done sender: [5:7574435910206996363:2110] self: [5:7574435910206996370:0] peer: [2:7574435912959786362:0] socket: 625 qp: -1
2025-11-19T13:59:48.922594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435910206996381:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:48.922602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:48.923001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435910206996298:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.923025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435910206996379:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10549
2025-11-19T13:59:48.923671Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435911353174261:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.923683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435911353174261:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.924259Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435908989480725:2114] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:48.924392Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435910206996379:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.927198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435911353174247:2110] [node 6] ICP35 event processing took too much time PendingConnection:278(28438877ns) {HandleHandshakeStatus:329(28438220ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(208ns) SwitchToState:178(267ns) ProcessPendingSessionEvents:495(28373883ns) {ForwardSessionEventToSession:612(28361268ns) ForwardSessionEventToSession:612(2362ns) ForwardSessionEventToSession:612(1005ns) ForwardSessionEventToSession:612(1476ns) ForwardSessionEventToSession:612(705ns) ForwardSessionEventToSession:612(600ns) ForwardSessionEventToSession:612(600ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(747ns) ForwardSessionEventToSession:612(1063ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.928407Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435911354708112:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9563) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435911354708184:2112] held: no
2025-11-19T13:59:48.928520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435911354708192:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28771
2025-11-19T13:59:48.928581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435911354708193:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28771
2025-11-19T13:59:48.928614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435911354708194:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28771
2025-11-19T13:59:48.928646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435911354708195:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28771
2025-11-19T13:59:48.929862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435910206996365:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.932307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435908989480644:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.932337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:48.932350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435908989480644:2049] [node 2] ICP22 created new session: [6:7574435908989480734:2049]
2025-11-19T13:59:48.932357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435908989480734:2049] [node 2] ICS09 handshake done sender: [6:7574435908989480725:2114] self: [6:7574435908989480729:0] peer: [2:7574435912959786371:0] socket: 656 qp: -1
2025-11-19T13:59:48.932366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435908989480734:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:48.932376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:48.932574Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435908989480714:2111] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:48.933782Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435911353174263:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.933798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435911353174263:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.932712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435911354708196:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28771
2025-11-19T13:59:48.932751Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435911354708194:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.932767Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435911354708184:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:48.933530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435911354708111:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.934643Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435911354708192:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.934669Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435911354708193:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.934724Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435911354708186:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.934889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435911354708112:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.934908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:48.934916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435911354708112:2049] [node 2] ICP22 created new session: [3:7574435911354708200:2049]
2025-11-19T13:59:48.934922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435911354708200:2049] [node 2] ICS09 handshake done sender: [3:7574435911354708184:2112] self: [3:7574435911354708190:0] peer: [2:7574435912959786365:0] socket: 637 qp: -1
2025-11-19T13:59:48.934929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435911354708200:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:48.934950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:48.935128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435911354708182:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.935137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:48.935142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435911354708182:2110] [node 6] ICP22 created new session: [3:7574435911354708202:2110]
2025-11-19T13:59:48.935147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435911354708202:2110] [node 6] ICS09 handshake done sender: [3:7574435911354708186:2113] self: [3:7574435911354708185:0] peer: [6:7574435908989480731:0] socket: 667 qp: -1
2025-11-19T13:59:48.935152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435911354708202:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:48.935159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:48.935347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435911354708182:2110] [node 6] ICP18 handshake (actor [3:7574435911354708193:2117]) is too old
2025-11-19T13:59:48.935563Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435911353174263:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:48.935659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435911353174181:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.935687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:48.935693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435911353174181:2049] [node 3] ICP22 created new session: [1:7574435911353174275:2049]
2025-11-19T13:59:48.935699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435911353174275:2049] [node 3] ICS09 handshake done sender: [1:7574435911353174263:2115] self: [1:7574435911353174256:0] peer: [3:7574435911354708198:0] socket: 663 qp: -1
2025-11-19T13:59:48.935705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435911353174275:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:48.935710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:48.936889Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435911353174261:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.937544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435908989480643:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.937561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:48.937567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435908989480643:2048] [node 1] ICP22 created new session: [6:7574435908989480737:2048]
2025-11-19T13:59:48.937574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435908989480737:2048] [node 1] ICS09 handshake done sender: [6:7574435908989480714:2111] self: [6:7574435908989480718:0] peer: [1:7574435911353174251:0] socket: 635 qp: -1
2025-11-19T13:59:48.937578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435908989480737:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:48.937584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.939937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435911310967631:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37173
2025-11-19T13:59:48.936978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435911353174277:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1824
2025-11-19T13:59:48.937033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435911353174248:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.937042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:48.937047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435911353174248:2109] [node 5] ICP22 created new session: [1:7574435911353174278:2109]
2025-11-19T13:59:48.937053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435911353174278:2109] [node 5] ICS09 handshake done sender: [1:7574435911353174261:2114] self: [1:7574435911353174254:0] peer: [5:7574435910206996383:0] socket: 647 qp: -1
2025-11-19T13:59:48.937058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435911353174278:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:48.937070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:48.945015Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435911353174277:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.945151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435911353174248:2109] [node 5] ICP18 handshake (actor [1:7574435911353174277:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.946363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435911354708199:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28771
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.944115Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435911310967619:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.944137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435911310967619:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.946685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435908989480645:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25456) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435908989480730:2116] held: no
2025-11-19T13:59:48.946774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435908989480736:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25423
2025-11-19T13:59:48.947028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435910206996386:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10549
2025-11-19T13:59:48.948769Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435910206996371:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:48.948643Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435908989480730:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:48.950073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435910206996298:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8199) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435910206996371:2114] held: no
2025-11-19T13:59:48.950100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435910206996298:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.950113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:48.950120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435910206996298:2048] [node 1] ICP22 created new session: [5:7574435910206996387:2048]
2025-11-19T13:59:48.950125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435910206996387:2048] [node 1] ICS09 handshake done sender: [5:7574435910206996371:2114] self: [5:7574435910206996383:0] peer: [1:7574435911353174254:0] socket: 654 qp: -1
2025-11-19T13:59:48.950133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435910206996387:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:48.950141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:48.950412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435908989480645:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.950425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:48.950430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435908989480645:2050] [node 3] ICP22 created new session: [6:7574435908989480739:2050]
2025-11-19T13:59:48.950434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435908989480739:2050] [node 3] ICS09 handshake done sender: [6:7574435908989480730:2116] self: [6:7574435908989480731:0] peer: [3:7574435911354708185:0] socket: 668 qp: -1
2025-11-19T13:59:48.950440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435908989480739:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:48.950449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.951212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435911354708111:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8199) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435911354708194:2118] held: no
2025-11-19T13:59:48.951379Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435911354708194:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:48.952624Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435911310967619:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:48.952929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435911310967603:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.952953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:48.952958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435911310967603:2109] [node 5] ICP22 created new session: [4:7574435911310967633:2109]
2025-11-19T13:59:48.952964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435911310967633:2109] [node 5] ICS09 handshake done sender: [4:7574435911310967619:2118] self: [4:7574435911310967618:0] peer: [5:7574435910206996385:0] socket: 666 qp: -1
2025-11-19T13:59:48.952968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435911310967633:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:48.952978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:48.951533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435910206996389:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10549
2025-11-19T13:59:48.954036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435911354708111:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.954053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:48.954059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435911354708111:2048] [node 1] ICP22 created new session: [3:7574435911354708207:2048]
2025-11-19T13:59:48.954065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435911354708207:2048] [node 1] ICS09 handshake done sender: [3:7574435911354708194:2118] self: [3:7574435911354708198:0] peer: [1:7574435911353174256:0] socket: 672 qp: -1
2025-11-19T13:59:48.954070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435911354708207:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:48.954077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:48.954471Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435911310967631:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.954502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435911310967539:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.955308Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435910206996379:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.955407Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435911354708188:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.955421Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435911354708188:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.955503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435910206996390:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10549
2025-11-19T13:59:48.955625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435910206996365:2111] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.955635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:48.955640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435910206996365:2111] [node 4] ICP22 created new session: [5:7574435910206996391:2111]
2025-11-19T13:59:48.955645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435910206996391:2111] [node 4] ICS09 handshake done sender: [5:7574435910206996379:2118] self: [5:7574435910206996385:0] peer: [4:7574435911310967618:0] socket: 633 qp: -1
2025-11-19T13:59:48.955651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435910206996391:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:48.955658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.956588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435911310967636:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37173
2025-11-19T13:59:48.956218Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435911354708196:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.957593Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435910206996365:2111] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:48.957775Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435910206996390:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.957822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435910206996300:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:48.958832Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435911354708188:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:48.959047Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435911310967631:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:48.959189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435911310967539:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.959207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:48.959214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435911310967539:2050] [node 3] ICP22 created new session: [4:7574435911310967637:2050]
2025-11-19T13:59:48.959220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435911310967637:2050] [node 3] ICS09 handshake done sender: [4:7574435911310967631:2121] self: [4:7574435911310967635:0] peer: [3:7574435911354708187:0] socket: 680 qp: -1
2025-11-19T13:59:48.959226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435911310967637:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:48.959233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:48.960375Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435911354708191:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:48.960385Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435911354708191:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:48.960937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435911354708179:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.960951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:48.960958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435911354708179:2108] [node 4] ICP22 created new session: [3:7574435911354708211:2108]
2025-11-19T13:59:48.960964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435911354708211:2108] [node 4] ICS09 handshake done sender: [3:7574435911354708188:2114] self: [3:7574435911354708187:0] peer: [4:7574435911310967635:0] socket: 671 qp: -1
2025-11-19T13:59:48.960970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435911354708211:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:48.960978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:48.961757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435910206996394:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10549
2025-11-19T13:59:48.961937Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435911354708191:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:48.962133Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435910206996390:2121] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:48.962262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435910206996300:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:48.962280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:48.962285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435910206996300:2050] [node 3] ICP22 created new session: [5:7574435910206996395:2050]
2025-11-19T13:59:48.962291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435910206996395:2050] [node 3] ICS09 handshake done sender: [5:7574435910206996390:2121] self: [5:7574435910206996393:0] peer: [3:7574435911354708183:0] socket: 661 qp: -1
2025-11-19T13:59:48.962298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435910206996395:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:48.962305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:48.963649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435911354708180:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:48.963668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:48.963673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435911354708180:2109] [node 5] ICP22 created new session: [3:7574435911354708214:2109]
2025-11-19T13:59:48.963679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435911354708214:2109] [node 5] ICS09 handshake done sender: [3:7574435911354708191:2115] self: [3:7574435911354708183:0] peer: [5:7574435910206996393:0] socket: 631 qp: -1
2025-11-19T13:59:48.963683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435911354708214:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:48.963691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:48.984981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:48.985071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574435912959786385:2048] [node 1] ICS07 socket disconnect 619 reason# EndOfStream
2025-11-19T13:59:48.985078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435912959786385:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.985113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.985129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435912959786385:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:48.987711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:48.987807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435911354708207:2048] [node 1] ICS01 socket: 672 reason# EndOfStream
2025-11-19T13:59:48.987816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435911354708111:2048] [node 1] ICP30 unregister session Session# [3:7574435911354708207:2048] VirtualId# [3:7574435911354708198:0]
2025-11-19T13:59:48.987823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435911354708207:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.987859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.987881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.992941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:48.993034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435911310967624:2048] [node 1] ICS07 socket disconnect 634 reason# EndOfStream
2025-11-19T13:59:48.993040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435911310967624:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.993091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.993113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435911310967624:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:48.993472Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:48.993537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435908989480737:2048] [node 1] ICS01 socket: 635 reason# EndOfStream
2025-11-19T13:59:48.993543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435908989480643:2048] [node 1] ICP30 unregister session Session# [6:7574435908989480737:2048] VirtualId# [6:7574435908989480718:0]
2025-11-19T13:59:48.993548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435908989480737:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.993574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.993597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.993816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:48.993887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435910206996387:2048] [node 1] ICS01 socket: 654 reason# EndOfStream
2025-11-19T13:59:48.993893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435910206996298:2048] [node 1] ICP30 unregister session Session# [5:7574435910206996387:2048] VirtualId# [5:7574435910206996383:0]
2025-11-19T13:59:48.993898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435910206996387:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:48.993942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:48.993962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.997485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435911310967537:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8199) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:48.997508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435911310967624:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:48.997513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435911310967537:2048] [node 1] ICP30 unregister session Session# [4:7574435911310967624:2048] VirtualId# [4:7574435911310967615:0]
2025-11-19T13:59:48.997528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:48.997535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435911310967537:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8199) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:48.997538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8199) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:49.002676Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:49.002786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435911354708200:2049] [node 2] ICS01 socket: 637 reason# EndOfStream
2025-11-19T13:59:49.002796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435911354708112:2049] [node 2] ICP30 unregister session Session# [3:7574435911354708200:2049] VirtualId# [3:7574435911354708190:0]
2025-11-19T13:59:49.002803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435911354708200:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.002844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:49.002865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:49.003895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:49.004002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435910206996381:2049] [node 2] ICS01 socket: 625 reason# EndOfStream
2025-11-19T13:59:49.004009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435910206996299:2049] [node 2] ICP30 unregister session Session# [5:7574435910206996381:2049] VirtualId# [5:7574435910206996370:0]
2025-11-19T13:59:49.004015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435910206996381:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.004057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:49.004081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:49.003959Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:49.004039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435908989480734:2049] [node 2] ICS01 socket: 656 reason# EndOfStream
2025-11-19T13:59:49.004046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435908989480644:2049] [node 2] ICP30 unregister session Session# [6:7574435908989480734:2049] VirtualId# [6:7574435908989480729:0]
2025-11-19T13:59:49.004052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435908989480734:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.004099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:49.004120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:49.004118Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:49.004186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435911310967627:2049] [node 2] ICS01 socket: 643 reason# EndOfStream
2025-11-19T13:59:49.004191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435911310967538:2049] [node 2] ICP30 unregister session Session# [4:7574435911310967627:2049] VirtualId# [4:7574435911310967623:0]
2025-11-19T13:59:49.004195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435911310967627:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.004222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:49.004240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:49.007713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435911310967537:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:49.027984Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:49.028106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435910206996395:2050] [node 3] ICS01 socket: 661 reason# EndOfStream
2025-11-19T13:59:49.028117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435910206996300:2050] [node 3] ICP30 unregister session Session# [5:7574435910206996395:2050] VirtualId# [5:7574435910206996393:0]
2025-11-19T13:59:49.028126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435910206996395:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.028198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:49.028225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:49.028019Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:49.028113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435908989480739:2050] [node 3] ICS01 socket: 668 reason# EndOfStream
2025-11-19T13:59:49.028120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435908989480645:2050] [node 3] ICP30 unregister session Session# [6:7574435908989480739:2050] VirtualId# [6:7574435908989480731:0]
2025-11-19T13:59:49.028127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435908989480739:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.028182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:49.028204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:49.049301Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:49.049423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435908989480721:2101] [node 4] ICS07 socket disconnect 622 reason# EndOfStream
2025-11-19T13:59:49.049432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435908989480721:2101] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.049492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:49.049515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435908989480721:2101] [node 4] ICS15 start handshake
2025-11-19T13:59:49.052396Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435908989480701:2101] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13713) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:49.052416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435908989480721:2101] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:49.052421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435908989480701:2101] [node 4] ICP30 unregister session Session# [6:7574435908989480721:2101] VirtualId# [6:7574435908989480702:0]
2025-11-19T13:59:49.052437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:49.052443Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435908989480701:2101] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13713) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:49.052446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13713) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:49.060005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435908989480701:2101] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:49.103847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435915539176505:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:49.103860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435915539176506:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:49.123957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435916631205582:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:49.123973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435916631205583:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:49.136025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435913585938752:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:49.136040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435913585938753:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:49.151853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435913334483827:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:49.151870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435913334483828:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:49.151875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435913334483829:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:49.171847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435916110206323:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:49.171861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435916110206324:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:49.171866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435916110206325:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:49.174606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435914577948622:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:49.174613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435914577948623:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:49.174617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435914577948624:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:49.207642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435914577948678:2102] [node 5] ICP01 ready to work
2025-11-19T13:59:49.208456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435916631205643:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:49.211180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435915539176572:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.216128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435915539176576:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:49.217196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435915539176569:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:49.218563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435916631205646:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4339
2025-11-19T13:59:49.223320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435916631205645:2106] [node 5] ICP01 ready to work
2025-11-19T13:59:49.225694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435913334483889:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.227915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435916110206384:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57605
2025-11-19T13:59:49.227629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435914577948690:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:49.228042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435914577948689:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61276
2025-11-19T13:59:49.229512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435916631205656:2112] [node 4] ICP01 ready to work
2025-11-19T13:59:49.232527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435913585938808:2102] [node 5] ICP01 ready to work
2025-11-19T13:59:49.233967Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435916110206384:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.234140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435916110206389:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:49.234487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435916110206389:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.239923Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435916631205645:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16518171ns) {Configure:104(16517453ns) {ProcessConfigured:135(16202506ns) {StartInitialHandshake:157(16201875ns) {DropHandshakes:469(665ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(141ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:476(1874ns) {SwitchToState:178(549ns)}}}}}
2025-11-19T13:59:49.240010Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435916631205646:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.240712Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435916631205582:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16814896ns) {Configure:104(16813104ns) {ProcessConfigured:135(16539591ns) {StartInitialHandshake:157(16539027ns) {DropHandshakes:469(358ns) {DropIncomingHandshake:441(43ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(302ns) PrepareNewSessionHandshake:476(1221ns) {SwitchToState:178(1027ns)}}}}}
2025-11-19T13:59:49.240740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435916631205582:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.240444Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435914577948689:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.240811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435914577948623:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.241106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435913334483896:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53558
2025-11-19T13:59:49.241162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435913334483897:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53558
2025-11-19T13:59:49.241318Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435915539176566:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.241328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435915539176566:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.242869Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435913334483896:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.243275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435913334483827:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.243471Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435915539176566:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:49.244326Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435914577948680:2103] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.244341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435914577948680:2103] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.245410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435913585938810:2103] [node 6] ICP01 ready to work
2025-11-19T13:59:49.245584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435913585938820:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:49.248404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435915539176569:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(31138195ns) {Configure:104(31137529ns) {ProcessConfigured:135(30871453ns) {StartInitialHandshake:157(30870938ns) {DropHandshakes:469(420ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:476(831ns) {SwitchToState:178(128ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.248570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435916110206392:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:49.249017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435916110206393:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57605
2025-11-19T13:59:49.248559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435916631205656:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16404906ns) {Configure:104(16402870ns) {ProcessConfigured:135(16054457ns) {StartInitialHandshake:157(16054347ns) {DropHandshakes:469(646ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(137ns)} GenerateSessionVirtualId:988(296ns) PrepareNewSessionHandshake:476(903ns) {SwitchToState:178(575ns)}}}}}
2025-11-19T13:59:49.248717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435914577948695:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61276
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.249286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435913585938753:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16704987ns) {Configure:104(16704139ns) {ProcessConfigured:135(16327072ns) {StartInitialHandshake:157(16326789ns) {DropHandshakes:469(618ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:476(1012ns) {SwitchToState:178(31ns)}}}}}
2025-11-19T13:59:49.249867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913585938818:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12061
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.250308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913585938830:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12061
2025-11-19T13:59:49.250354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913585938831:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12061
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.250470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435916631205664:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4339
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.249406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435913334483899:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53558
2025-11-19T13:59:49.250241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435913334483900:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53558
2025-11-19T13:59:49.252425Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435915539176578:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.252441Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435915539176578:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.252661Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435916631205647:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.252673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435916631205647:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.252758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435916631205663:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4339
2025-11-19T13:59:49.252812Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913585938818:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.251313Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435916110206393:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.253743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435916110206394:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57605
2025-11-19T13:59:49.254691Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913585938830:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.256878Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435916631205663:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.256898Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435916631205646:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:49.257362Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435914577948680:2103] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:49.257510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435914577948678:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.257532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:49.257541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435914577948678:2102] [node 5] ICP22 created new session: [6:7574435914577948697:2102]
2025-11-19T13:59:49.257548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435914577948697:2102] [node 5] ICS09 handshake done sender: [6:7574435914577948680:2103] self: [6:7574435914577948679:0] peer: [5:7574435916110206390:0] socket: 626 qp: -1
2025-11-19T13:59:49.257557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435914577948697:2102] [node 5] ICS10 traffic start
2025-11-19T13:59:49.257568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.258132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435916110206404:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57605
2025-11-19T13:59:49.258303Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435916110206384:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:49.261956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435916110206389:2110] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.261980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:49.261989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435916110206389:2110] [node 6] ICP22 created new session: [5:7574435916110206405:2110]
2025-11-19T13:59:49.261995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435916110206405:2110] [node 6] ICS09 handshake done sender: [5:7574435916110206384:2108] self: [5:7574435916110206390:0] peer: [6:7574435914577948679:0] socket: 627 qp: -1
2025-11-19T13:59:49.262003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435916110206405:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:49.262014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:49.263918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435915539176586:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28018
2025-11-19T13:59:49.264050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435915539176505:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.264068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:49.264077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435915539176505:2048] [node 2] ICP22 created new session: [1:7574435915539176587:2048]
2025-11-19T13:59:49.264087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435915539176587:2048] [node 2] ICS09 handshake done sender: [1:7574435915539176566:2107] self: [1:7574435915539176565:0] peer: [2:7574435916631205661:0] socket: 624 qp: -1
2025-11-19T13:59:49.264094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435915539176587:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:49.264104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:49.264255Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913585938831:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.264296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435913585938753:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.265842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435916631205665:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4339
2025-11-19T13:59:49.266019Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435916631205653:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.266024Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435916631205653:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.266651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435916631205582:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.266664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:49.266670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435916631205582:2048] [node 1] ICP22 created new session: [2:7574435916631205666:2048]
2025-11-19T13:59:49.266676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435916631205666:2048] [node 1] ICS09 handshake done sender: [2:7574435916631205646:2108] self: [2:7574435916631205661:0] peer: [1:7574435915539176565:0] socket: 625 qp: -1
2025-11-19T13:59:49.266682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435916631205666:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:49.266691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.268870Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435916110206394:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.268942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435916110206324:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.269263Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435913585938752:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(45124198ns) {Configure:104(45123369ns) {ProcessConfigured:135(44701136ns) {StartInitialHandshake:157(44700301ns) {DropHandshakes:469(484ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:476(1178ns) {SwitchToState:178(52ns)}}}}}
2025-11-19T13:59:49.269350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435913585938752:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.269444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913585938836:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12061
2025-11-19T13:59:49.270063Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435913585938831:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:49.269875Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435916631205653:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:49.270153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435913585938753:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.270168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:49.270174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435913585938753:2049] [node 2] ICP22 created new session: [3:7574435913585938838:2049]
2025-11-19T13:59:49.270180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435913585938838:2049] [node 2] ICS09 handshake done sender: [3:7574435913585938831:2115] self: [3:7574435913585938834:0] peer: [2:7574435916631205649:0] socket: 657 qp: -1
2025-11-19T13:59:49.270186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435913585938838:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:49.270202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.270619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435916110206408:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57605
2025-11-19T13:59:49.268713Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435915539176578:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:49.272825Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435916631205658:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.272836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435916631205658:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.275047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435916110206409:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57605
2025-11-19T13:59:49.275930Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435913334483897:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.276119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435913334483902:2115] [node 6] ICP01 ready to work
2025-11-19T13:59:49.278697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435916110206392:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.278734Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435916110206392:2111] [node 4] ICP35 event processing took too much time PendingConnection:278(26357588ns) {IncomingHandshake:261(26357038ns) {DropIncomingHandshake:441(60ns) PrepareNewSessionHandshake:476(572ns) {SwitchToState:178(289ns)} GenerateSessionVirtualId:988(245ns) IssueIncomingHandshakeReply:193(26349016ns) {SwitchToState:178(148ns)}}}
2025-11-19T13:59:49.279840Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435913334483893:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.279863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435913334483893:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.286870Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435915539176575:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.286887Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435915539176575:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.287356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435915539176576:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.287371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:49.287377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435915539176576:2110] [node 4] ICP22 created new session: [1:7574435915539176590:2110]
2025-11-19T13:59:49.287383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435915539176590:2110] [node 4] ICS09 handshake done sender: [1:7574435915539176578:2112] self: [1:7574435915539176577:0] peer: [4:7574435913334483898:0] socket: 629 qp: -1
2025-11-19T13:59:49.287391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435915539176590:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:49.287402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.287810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435916631205583:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.287832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:49.287841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435916631205583:2049] [node 3] ICP22 created new session: [2:7574435916631205673:2049]
2025-11-19T13:59:49.287848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435916631205673:2049] [node 3] ICS09 handshake done sender: [2:7574435916631205653:2111] self: [2:7574435916631205649:0] peer: [3:7574435913585938834:0] socket: 636 qp: -1
2025-11-19T13:59:49.287856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435916631205673:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:49.287886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:49.288088Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435916631205647:2109] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:49.287983Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435913334483900:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.288029Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435913334483899:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.288062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435913334483829:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.288269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435915539176594:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28018
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.289370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435916110206410:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57605
2025-11-19T13:59:49.292558Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913585938826:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.292573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435913585938826:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.291917Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435916631205658:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:49.292093Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435915539176575:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:49.292759Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435915539176586:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.292845Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435915539176594:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.293288Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435916110206394:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:49.293413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435916110206324:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.293430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:49.293437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435916110206324:2049] [node 2] ICP22 created new session: [5:7574435916110206411:2049]
2025-11-19T13:59:49.293444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435916110206411:2049] [node 2] ICS09 handshake done sender: [5:7574435916110206394:2113] self: [5:7574435916110206407:0] peer: [2:7574435916631205650:0] socket: 647 qp: -1
2025-11-19T13:59:49.293450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435916110206411:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:49.293458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:49.293832Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435914577948695:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.293911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435914577948622:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.296501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913585938841:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12061
2025-11-19T13:59:49.296289Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435915539176505:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(51989079ns) {HandleHandshakeStatus:329(51988264ns) {DropIncomingHandshake:441(676ns) DropOutgoingHandshake:456(327ns) SwitchToState:178(147ns) ProcessPendingSessionEvents:495(17606705ns) {ForwardSessionEventToSession:612(17595779ns) ForwardSessionEventToSession:612(1412ns) ForwardSessionEventToSession:612(937ns) ForwardSessionEventToSession:612(823ns) ForwardSessionEventToSession:612(923ns) ForwardSessionEventToSession:612(1087ns) ForwardSessionEventToSession:612(790ns) ForwardSessionEventToSession:612(675ns)}}}
2025-11-19T13:59:49.296548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913585938842:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12061
2025-11-19T13:59:49.297502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435916631205645:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.297524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:49.297530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435916631205645:2106] [node 5] ICP22 created new session: [2:7574435916631205675:2106]
2025-11-19T13:59:49.297536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435916631205675:2106] [node 5] ICS09 handshake done sender: [2:7574435916631205658:2113] self: [2:7574435916631205650:0] peer: [5:7574435916110206407:0] socket: 642 qp: -1
2025-11-19T13:59:49.297541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435916631205675:2106] [node 5] ICS10 traffic start
2025-11-19T13:59:49.297548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:49.297695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435916631205643:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.297700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:49.297706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435916631205643:2107] [node 6] ICP22 created new session: [2:7574435916631205677:2107]
2025-11-19T13:59:49.297710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435916631205677:2107] [node 6] ICS09 handshake done sender: [2:7574435916631205647:2109] self: [2:7574435916631205644:0] peer: [6:7574435914577948694:0] socket: 630 qp: -1
2025-11-19T13:59:49.297713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435916631205677:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:49.297719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:49.297919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435915539176506:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.297939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:49.297944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435915539176506:2049] [node 3] ICP22 created new session: [1:7574435915539176597:2049]
2025-11-19T13:59:49.297950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435915539176597:2049] [node 3] ICS09 handshake done sender: [1:7574435915539176575:2111] self: [1:7574435915539176570:0] peer: [3:7574435913585938837:0] socket: 628 qp: -1
2025-11-19T13:59:49.297955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435915539176597:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:49.297962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:49.297887Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435913585938818:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:49.298704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435913585938752:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.298719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:49.298725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435913585938752:2048] [node 1] ICP22 created new session: [3:7574435913585938843:2048]
2025-11-19T13:59:49.298730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435913585938843:2048] [node 1] ICS09 handshake done sender: [3:7574435913585938818:2110] self: [3:7574435913585938837:0] peer: [1:7574435915539176570:0] socket: 633 qp: -1
2025-11-19T13:59:49.298736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435913585938843:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:49.298743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.297955Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435913334483893:2110] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:49.298076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435913334483889:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.298087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:49.298092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435913334483889:2108] [node 5] ICP22 created new session: [4:7574435913334483912:2108]
2025-11-19T13:59:49.298097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435913334483912:2108] [node 5] ICS09 handshake done sender: [4:7574435913334483893:2110] self: [4:7574435913334483892:0] peer: [5:7574435916110206403:0] socket: 635 qp: -1
2025-11-19T13:59:49.298101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435913334483912:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:49.298107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:49.300164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435913334483915:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53558
2025-11-19T13:59:49.300202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435913334483916:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53558
2025-11-19T13:59:49.303023Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913585938841:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.303890Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435915539176582:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.303907Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435915539176582:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.308246Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435913334483896:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.309371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435916110206413:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57605
2025-11-19T13:59:49.309458Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435916110206324:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16027457ns) {HandleHandshakeStatus:329(16026995ns) {DropIncomingHandshake:441(238ns) DropOutgoingHandshake:456(1763ns) SwitchToState:178(150ns) ProcessPendingSessionEvents:495(15950378ns) {ForwardSessionEventToSession:612(15947478ns) ForwardSessionEventToSession:612(1101ns)}}}
2025-11-19T13:59:49.309121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435916631205679:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4339
2025-11-19T13:59:49.310143Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435913585938826:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:49.310315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435913585938820:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.310330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:49.310336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435913585938820:2109] [node 4] ICP22 created new session: [3:7574435913585938847:2109]
2025-11-19T13:59:49.310342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435913585938847:2109] [node 4] ICS09 handshake done sender: [3:7574435913585938826:2113] self: [3:7574435913585938825:0] peer: [4:7574435913334483907:0] socket: 653 qp: -1
2025-11-19T13:59:49.310348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435913585938847:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:49.310355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:49.312011Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435916110206408:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.312044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435916110206325:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.315829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435914577948706:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61276
2025-11-19T13:59:49.315915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435914577948707:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61276
2025-11-19T13:59:49.315954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435914577948708:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61276
2025-11-19T13:59:49.316625Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435913334483915:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.318103Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435914577948689:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:49.318295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435914577948623:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.318308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:49.318315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435914577948623:2049] [node 2] ICP22 created new session: [6:7574435914577948709:2049]
2025-11-19T13:59:49.318333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435914577948709:2049] [node 2] ICS09 handshake done sender: [6:7574435914577948689:2111] self: [6:7574435914577948694:0] peer: [2:7574435916631205644:0] socket: 632 qp: -1
2025-11-19T13:59:49.318340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435914577948709:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:49.318351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:49.317881Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435916110206410:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.317898Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435916110206393:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:49.318819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435913334483827:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.318835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:49.318841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435913334483827:2048] [node 1] ICP22 created new session: [4:7574435913334483917:2048]
2025-11-19T13:59:49.318845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435913334483917:2048] [node 1] ICS09 handshake done sender: [4:7574435913334483896:2111] self: [4:7574435913334483898:0] peer: [1:7574435915539176577:0] socket: 631 qp: -1
2025-11-19T13:59:49.318852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435913334483917:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:49.318871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:49.320384Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435914577948706:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.320426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435914577948624:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.320987Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435916631205643:2107] [node 6] ICP35 event processing took too much time PendingConnection:278(23262072ns) {HandleHandshakeStatus:329(23261426ns) {DropIncomingHandshake:441(42ns) DropOutgoingHandshake:456(86ns) SwitchToState:178(74ns) ProcessPendingSessionEvents:495(23216928ns) {ForwardSessionEventToSession:612(23208233ns) ForwardSessionEventToSession:612(2114ns) ForwardSessionEventToSession:612(584ns) ForwardSessionEventToSession:612(1132ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(489ns) ForwardSessionEventToSession:612(1028ns) ForwardSessionEventToSession:612(502ns)}}}
2025-11-19T13:59:49.323790Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913585938833:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.323812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435913585938833:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.325136Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435916631205662:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.325146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435916631205662:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.325167Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435915539176582:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:49.325742Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435914577948708:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.325782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435914577948690:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.324664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435913334483829:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7471) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435913334483900:2114] held: no
2025-11-19T13:59:49.324837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435913334483828:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.325053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435913334483920:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53558
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.327947Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435913334483904:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.327957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435913334483904:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.328494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435913334483921:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53558
2025-11-19T13:59:49.328689Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435913334483900:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:49.329846Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435913334483904:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:49.330031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435913334483902:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.330048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:49.330053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435913334483902:2115] [node 6] ICP22 created new session: [4:7574435913334483923:2115]
2025-11-19T13:59:49.330057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435913334483923:2115] [node 6] ICS09 handshake done sender: [4:7574435913334483904:2116] self: [4:7574435913334483903:0] peer: [6:7574435914577948712:0] socket: 674 qp: -1
2025-11-19T13:59:49.330061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435913334483923:2115] [node 6] ICS10 traffic start
2025-11-19T13:59:49.330067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:49.326001Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435916631205662:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:49.326079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435916631205656:2112] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.326090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:49.326097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435916631205656:2112] [node 4] ICP22 created new session: [2:7574435916631205681:2112]
2025-11-19T13:59:49.326103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435916631205681:2112] [node 4] ICS09 handshake done sender: [2:7574435916631205662:2115] self: [2:7574435916631205657:0] peer: [4:7574435913334483919:0] socket: 665 qp: -1
2025-11-19T13:59:49.326109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435916631205681:2112] [node 4] ICS10 traffic start
2025-11-19T13:59:49.326117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:49.330149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435916631205679:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.330171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435916631205656:2112] [node 4] ICP18 handshake (actor [2:7574435916631205679:2119]) is too old
2025-11-19T13:59:49.329575Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435915539176585:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.329585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435915539176585:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.330466Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435915539176585:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:49.330544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435915539176572:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.330560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:49.330565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435915539176572:2109] [node 6] ICP22 created new session: [1:7574435915539176601:2109]
2025-11-19T13:59:49.330570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435915539176601:2109] [node 6] ICS09 handshake done sender: [1:7574435915539176582:2113] self: [1:7574435915539176573:0] peer: [6:7574435914577948705:0] socket: 645 qp: -1
2025-11-19T13:59:49.330575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435915539176601:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:49.330587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:49.330602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435913334483829:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.330615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:49.330620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435913334483829:2050] [node 3] ICP22 created new session: [4:7574435913334483925:2050]
2025-11-19T13:59:49.330624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435913334483925:2050] [node 3] ICS09 handshake done sender: [4:7574435913334483900:2114] self: [4:7574435913334483907:0] peer: [3:7574435913585938825:0] socket: 656 qp: -1
2025-11-19T13:59:49.330629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435913334483925:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:49.330635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:49.330924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435915539176569:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.330943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:49.330947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435915539176569:2108] [node 5] ICP22 created new session: [1:7574435915539176603:2108]
2025-11-19T13:59:49.330952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435915539176603:2108] [node 5] ICS09 handshake done sender: [1:7574435915539176585:2114] self: [1:7574435915539176581:0] peer: [5:7574435916110206418:0] socket: 676 qp: -1
2025-11-19T13:59:49.330956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435915539176603:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:49.330961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:49.328754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435916110206392:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14033) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435916110206393:2112] held: no
2025-11-19T13:59:49.328783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435916110206392:2111] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.328791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:49.328796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435916110206392:2111] [node 4] ICP22 created new session: [5:7574435916110206415:2111]
2025-11-19T13:59:49.328802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435916110206415:2111] [node 4] ICS09 handshake done sender: [5:7574435916110206393:2112] self: [5:7574435916110206403:0] peer: [4:7574435913334483892:0] socket: 637 qp: -1
2025-11-19T13:59:49.328808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435916110206415:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:49.328815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:49.329055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435916110206323:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.332780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435913334483828:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62224) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435913334483915:2118] held: no
2025-11-19T13:59:49.332831Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435913334483915:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:49.332909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435913334483828:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.332920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:49.332926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435913334483828:2049] [node 2] ICP22 created new session: [4:7574435913334483927:2049]
2025-11-19T13:59:49.332930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435913334483927:2049] [node 2] ICS09 handshake done sender: [4:7574435913334483915:2118] self: [4:7574435913334483919:0] peer: [2:7574435916631205657:0] socket: 651 qp: -1
2025-11-19T13:59:49.332935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435913334483927:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:49.332941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.332802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435916110206323:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29293) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435916110206410:2120] held: no
2025-11-19T13:59:49.334105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435916110206419:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57605
2025-11-19T13:59:49.336800Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913585938824:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.336821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435913585938824:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.341090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435914577948690:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14033) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435914577948708:2118] held: no
2025-11-19T13:59:49.341350Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435916110206410:2120] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:49.341429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435916110206323:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.341446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:49.341452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435916110206323:2048] [node 1] ICP22 created new session: [5:7574435916110206420:2048]
2025-11-19T13:59:49.341458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435916110206420:2048] [node 1] ICS09 handshake done sender: [5:7574435916110206410:2120] self: [5:7574435916110206418:0] peer: [1:7574435915539176581:0] socket: 677 qp: -1
2025-11-19T13:59:49.341464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435916110206420:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:49.341472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.346041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435913585938850:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12061
2025-11-19T13:59:49.351991Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435913585938833:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:49.352091Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435913585938850:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.352393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435913334483902:2115] [node 6] ICP35 event processing took too much time PendingConnection:278(22334589ns) {HandleHandshakeStatus:329(22334045ns) {DropIncomingHandshake:441(3572ns) DropOutgoingHandshake:456(99ns) SwitchToState:178(71ns) ProcessPendingSessionEvents:495(22277794ns) {ForwardSessionEventToSession:612(22267914ns) ForwardSessionEventToSession:612(2104ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(793ns) ForwardSessionEventToSession:612(818ns) ForwardSessionEventToSession:612(738ns)}}}
2025-11-19T13:59:49.352397Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435914577948622:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29293) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435914577948695:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.355992Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435913585938824:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.356118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435914577948714:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61276
2025-11-19T13:59:49.356167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435914577948715:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61276
2025-11-19T13:59:49.356222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435916110206422:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57605
2025-11-19T13:59:49.356511Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435916110206408:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:49.356546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435916110206325:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.356559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:49.356565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435916110206325:2050] [node 3] ICP22 created new session: [5:7574435916110206423:2050]
2025-11-19T13:59:49.356573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435916110206423:2050] [node 3] ICS09 handshake done sender: [5:7574435916110206408:2118] self: [5:7574435916110206414:0] peer: [3:7574435913585938821:0] socket: 669 qp: -1
2025-11-19T13:59:49.356579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435916110206423:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:49.356586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:49.356114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435913585938810:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.356136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:49.356144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435913585938810:2103] [node 6] ICP22 created new session: [3:7574435913585938853:2103]
2025-11-19T13:59:49.356149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435913585938853:2103] [node 6] ICS09 handshake done sender: [3:7574435913585938824:2112] self: [3:7574435913585938823:0] peer: [6:7574435914577948711:0] socket: 652 qp: -1
2025-11-19T13:59:49.356155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435913585938853:2103] [node 6] ICS10 traffic start
2025-11-19T13:59:49.356165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:49.359462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435914577948713:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61276
2025-11-19T13:59:49.360130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435914577948624:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7471) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435914577948706:2116] held: no
2025-11-19T13:59:49.360159Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435914577948708:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:49.359986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435913585938808:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.360011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:49.360018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435913585938808:2102] [node 5] ICP22 created new session: [3:7574435913585938855:2102]
2025-11-19T13:59:49.360025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435913585938855:2102] [node 5] ICS09 handshake done sender: [3:7574435913585938833:2116] self: [3:7574435913585938821:0] peer: [5:7574435916110206414:0] socket: 668 qp: -1
2025-11-19T13:59:49.360032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435913585938855:2102] [node 5] ICS10 traffic start
2025-11-19T13:59:49.360041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:49.360295Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435914577948706:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:49.360393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435914577948624:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.360405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:49.360411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435914577948624:2050] [node 3] ICP22 created new session: [6:7574435914577948716:2050]
2025-11-19T13:59:49.360416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435914577948716:2050] [node 3] ICS09 handshake done sender: [6:7574435914577948706:2116] self: [6:7574435914577948711:0] peer: [3:7574435913585938823:0] socket: 659 qp: -1
2025-11-19T13:59:49.360421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435914577948716:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:49.360427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:49.360534Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435914577948695:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:49.360680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435914577948690:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.360691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:49.360696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435914577948690:2110] [node 4] ICP22 created new session: [6:7574435914577948718:2110]
2025-11-19T13:59:49.360701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435914577948718:2110] [node 4] ICS09 handshake done sender: [6:7574435914577948708:2118] self: [6:7574435914577948712:0] peer: [4:7574435913334483903:0] socket: 683 qp: -1
2025-11-19T13:59:49.360707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435914577948718:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:49.360715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:49.361043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435914577948622:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.361056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:49.361062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435914577948622:2048] [node 1] ICP22 created new session: [6:7574435914577948720:2048]
2025-11-19T13:59:49.361067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435914577948720:2048] [node 1] ICS09 handshake done sender: [6:7574435914577948695:2113] self: [6:7574435914577948705:0] peer: [1:7574435915539176573:0] socket: 646 qp: -1
2025-11-19T13:59:49.361073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435914577948720:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:49.361090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:49.391809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:49.391907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435913334483917:2048] [node 1] ICS01 socket: 631 reason# EndOfStream
2025-11-19T13:59:49.391916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435913334483827:2048] [node 1] ICP30 unregister session Session# [4:7574435913334483917:2048] VirtualId# [4:7574435913334483898:0]
2025-11-19T13:59:49.391924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435913334483917:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.391975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:49.392000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:49.391918Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:49.392019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435913585938843:2048] [node 1] ICS01 socket: 633 reason# EndOfStream
2025-11-19T13:59:49.392027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435913585938752:2048] [node 1] ICP30 unregister session Session# [3:7574435913585938843:2048] VirtualId# [3:7574435913585938837:0]
2025-11-19T13:59:49.392035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435913585938843:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.392090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:49.392119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:49.392212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:49.392279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435916110206420:2048] [node 1] ICS01 socket: 677 reason# EndOfStream
2025-11-19T13:59:49.392284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435916110206323:2048] [node 1] ICP30 unregister session Session# [5:7574435916110206420:2048] VirtualId# [5:7574435916110206418:0]
2025-11-19T13:59:49.392288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435916110206420:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.392320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:49.392343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:49.399763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:49.399863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435914577948720:2048] [node 1] ICS01 socket: 646 reason# EndOfStream
2025-11-19T13:59:49.399873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435914577948622:2048] [node 1] ICP30 unregister session Session# [6:7574435914577948720:2048] VirtualId# [6:7574435914577948705:0]
2025-11-19T13:59:49.399883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435914577948720:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.399947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:49.399975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:49.409792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:49.409876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435913585938838:2049] [node 2] ICS01 socket: 657 reason# EndOfStream
2025-11-19T13:59:49.409884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435913585938753:2049] [node 2] ICP30 unregister session Session# [3:7574435913585938838:2049] VirtualId# [3:7574435913585938834:0]
2025-11-19T13:59:49.409891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435913585938838:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.409917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:49.409937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:49.426693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:49.426796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435914577948709:2049] [node 2] ICS07 socket disconnect 632 reason# EndOfStream
2025-11-19T13:59:49.426802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435914577948709:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.426869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:49.426891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435914577948709:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:49.427224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:49.427287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435913334483927:2049] [node 2] ICS07 socket disconnect 651 reason# EndOfStream
2025-11-19T13:59:49.427291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435913334483927:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.427317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:49.427336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435913334483927:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:49.426888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:49.426969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435916110206411:2049] [node 2] ICS07 socket disconnect 647 reason# EndOfStream
2025-11-19T13:59:49.426975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435916110206411:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.427017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:49.427037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435916110206411:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:49.428060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435916110206324:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62224) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:49.428072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435916110206411:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:49.428078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435916110206324:2049] [node 2] ICP30 unregister session Session# [5:7574435916110206411:2049] VirtualId# [5:7574435916110206407:0]
2025-11-19T13:59:49.428088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:49.428091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435916110206324:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:49.428092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:49.427829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435913334483828:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62224) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:49.427843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435913334483927:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:49.427847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435913334483828:2049] [node 2] ICP30 unregister session Session# [4:7574435913334483927:2049] VirtualId# [4:7574435913334483919:0]
2025-11-19T13:59:49.427862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:49.427868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435913334483828:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:49.427871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:49.428443Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435914577948623:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62224) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:49.428455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435914577948709:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:49.428458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435914577948623:2049] [node 2] ICP30 unregister session Session# [6:7574435914577948709:2049] VirtualId# [6:7574435914577948694:0]
2025-11-19T13:59:49.428467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:49.428470Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435914577948623:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:49.428472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:49.439886Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:49.439988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435914577948716:2050] [node 3] ICS01 socket: 659 reason# EndOfStream
2025-11-19T13:59:49.439998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435914577948624:2050] [node 3] ICP30 unregister session Session# [6:7574435914577948716:2050] VirtualId# [6:7574435914577948711:0]
2025-11-19T13:59:49.440006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435914577948716:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.440058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:49.440092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:49.440128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:49.440187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435916110206423:2050] [node 3] ICS07 socket disconnect 669 reason# EndOfStream
2025-11-19T13:59:49.440191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435916110206423:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.440221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:49.440243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435916110206423:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:49.440298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435916110206324:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:49.440677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435914577948623:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:49.443964Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435916110206325:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7471) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:49.443982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435916110206423:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:49.443987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435916110206325:2050] [node 3] ICP30 unregister session Session# [5:7574435916110206423:2050] VirtualId# [5:7574435916110206414:0]
2025-11-19T13:59:49.444007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:49.444014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435916110206325:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7471) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:49.444017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7471) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:49.453095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435916110206325:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:59:49.492341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:49.492407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435914577948718:2110] [node 4] ICS01 socket: 683 reason# EndOfStream
2025-11-19T13:59:49.492412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435914577948690:2110] [node 4] ICP30 unregister session Session# [6:7574435914577948718:2110] VirtualId# [6:7574435914577948712:0]
2025-11-19T13:59:49.492416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435914577948718:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:49.492447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:49.492467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:49.630924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435914673337851:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:49.630935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435914673337852:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:49.663948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435917330476725:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:49.663960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435917330476726:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:49.706829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435916846606478:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:49.706841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435916846606479:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:49.731964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435916257234057:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:49.731979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435916257234059:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:49.731984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435916257234058:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:49.839882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435915268826606:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:49.839901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435915268826607:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:49.839901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435915268826605:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:49.865068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435916921387872:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:49.865082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435916921387873:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:49.865087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435916921387874:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:49.904583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435916921387898:2067] [node 5] ICP01 ready to work
2025-11-19T13:59:49.904736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435916921387900:2070] [node 4] ICP01 ready to work
2025-11-19T13:59:49.904096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435916257234100:2088] [node 5] ICP01 ready to work
2025-11-19T13:59:49.904762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435916257234106:2091] [node 6] ICP01 ready to work
2025-11-19T13:59:49.905052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435914673337912:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:49.904746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435915268826637:2069] [node 6] ICP01 ready to work
2025-11-19T13:59:49.904838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435915268826638:2076] [node 4] ICP01 ready to work
2025-11-19T13:59:49.912711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435917330476786:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.916879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435914673337921:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:49.917262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435914673337919:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24648
2025-11-19T13:59:49.917275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435914673337920:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24648
2025-11-19T13:59:49.920319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435916921387950:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-19T13:59:49.920388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435916921387951:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-19T13:59:49.923947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435916257234114:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51582
2025-11-19T13:59:49.923286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435914673337917:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:49.924301Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435914673337920:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.925419Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435916921387950:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.932237Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435914673337852:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(26740855ns) {Configure:104(18014966ns) {ProcessConfigured:135(17669556ns) {StartInitialHandshake:157(17669203ns) {DropHandshakes:469(595ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(115ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:476(573ns) {SwitchToState:178(261ns)}}}}}
2025-11-19T13:59:49.933641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435914673337931:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24648
2025-11-19T13:59:49.935981Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435916921387951:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.938496Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435916257234110:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.938518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435916257234110:2094] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.938083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435917330476794:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.942953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435917330476799:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18162
2025-11-19T13:59:49.942980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435917330476800:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18162
2025-11-19T13:59:49.943140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435916257234059:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(37795294ns) {Configure:104(37793684ns) {ProcessConfigured:135(37508932ns) {StartInitialHandshake:157(37508282ns) {DropHandshakes:469(500ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(1624ns) {SwitchToState:178(121ns)}}}}}
2025-11-19T13:59:49.943264Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435916257234058:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24051963ns) {Configure:104(24050035ns) {ProcessConfigured:135(23702697ns) {StartInitialHandshake:157(23702393ns) {DropHandshakes:469(773ns) {DropIncomingHandshake:441(146ns) DropOutgoingHandshake:456(90ns)} GenerateSessionVirtualId:988(309ns) PrepareNewSessionHandshake:476(619ns) {SwitchToState:178(164ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.944462Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435916846606479:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20745865ns) {Configure:104(20744816ns) {ProcessConfigured:135(20265649ns) {StartInitialHandshake:157(20264912ns) {DropHandshakes:469(636ns) {DropIncomingHandshake:441(130ns) DropOutgoingHandshake:456(96ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:476(1178ns) {SwitchToState:178(610ns)}}}}}
2025-11-19T13:59:49.944873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435916846606541:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:843
2025-11-19T13:59:49.945222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435915268826637:2069] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(40333633ns) {Configure:104(40333121ns) {ProcessConfigured:135(39924156ns) {StartInitialHandshake:157(39923713ns) {DropHandshakes:469(299ns) {DropIncomingHandshake:441(36ns) DropOutgoingHandshake:456(39ns)} GenerateSessionVirtualId:988(132ns) PrepareNewSessionHandshake:476(871ns) {SwitchToState:178(91ns)}}}}}
2025-11-19T13:59:49.945392Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435915268826638:2076] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(38821263ns) {Configure:104(38819501ns) {ProcessConfigured:135(38544457ns) {StartInitialHandshake:157(38543968ns) {DropHandshakes:469(486ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(52ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:476(1070ns) {SwitchToState:178(696ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.945659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435917330476725:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(21721340ns) {Configure:104(21720326ns) {ProcessConfigured:135(21268325ns) {StartInitialHandshake:157(21268166ns) {DropHandshakes:469(590ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(322ns) PrepareNewSessionHandshake:476(607ns) {SwitchToState:178(115ns)}}}}}
2025-11-19T13:59:49.943316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435914673337917:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19932721ns) {Configure:104(19931459ns) {ProcessConfigured:135(19454096ns) {StartInitialHandshake:157(19453578ns) {DropHandshakes:469(1084ns) {DropIncomingHandshake:441(221ns) DropOutgoingHandshake:456(147ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:476(1410ns) {SwitchToState:178(703ns)}}}}}
2025-11-19T13:59:49.945562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435914673337933:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24648
2025-11-19T13:59:49.947443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435916921387900:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.947486Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435916921387900:2070] [node 4] ICP35 event processing took too much time PendingConnection:278(19660909ns) {IncomingHandshake:261(19660407ns) {DropIncomingHandshake:441(183ns) PrepareNewSessionHandshake:476(561ns) {SwitchToState:178(230ns)} GenerateSessionVirtualId:988(335ns) IssueIncomingHandshakeReply:193(19650240ns) {SwitchToState:178(863ns)}}}
2025-11-19T13:59:49.947036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435916846606545:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:49.947816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435916846606543:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:49.947808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435915268826681:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34847
2025-11-19T13:59:49.947353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435917330476796:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:49.948195Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435917330476800:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.948160Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435915268826681:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.949414Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435917330476799:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.950325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435916846606542:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:49.950477Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435916846606541:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.956143Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435916257234110:2094] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:49.956365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435916921387872:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.956406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435916921387872:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20362509ns) {IncomingHandshake:261(20361851ns) {DropIncomingHandshake:441(142ns) PrepareNewSessionHandshake:476(425ns) {SwitchToState:178(209ns)} GenerateSessionVirtualId:988(230ns) IssueIncomingHandshakeReply:193(20349424ns) {SwitchToState:178(783ns)}}}
2025-11-19T13:59:49.956532Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435914673337923:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.956550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435914673337923:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.956808Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435914673337919:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.963964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435917330476796:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16536183ns) {Configure:104(16535440ns) {ProcessConfigured:135(16224475ns) {StartInitialHandshake:157(16224008ns) {DropHandshakes:469(559ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(111ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(1062ns) {SwitchToState:178(714ns)}}}}}
2025-11-19T13:59:49.959828Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435916257234104:2092] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.959848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435916257234104:2092] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-19T13:59:49.960138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435916257234106:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.960149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:49.960158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435916257234106:2091] [node 6] ICP22 created new session: [4:7574435916257234137:2091]
2025-11-19T13:59:49.960164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435916257234137:2091] [node 6] ICS09 handshake done sender: [4:7574435916257234110:2094] self: [4:7574435916257234107:0] peer: [6:7574435916921387952:0] socket: 640 qp: -1
2025-11-19T13:59:49.960172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435916257234137:2091] [node 6] ICS10 traffic start
2025-11-19T13:59:49.960182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.963225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435916846606545:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16065992ns) {Configure:104(16065252ns) {ProcessConfigured:135(15715503ns) {StartInitialHandshake:157(15714877ns) {DropHandshakes:469(696ns) {DropIncomingHandshake:441(127ns) DropOutgoingHandshake:456(113ns)} GenerateSessionVirtualId:988(311ns) PrepareNewSessionHandshake:476(1067ns) {SwitchToState:178(843ns)}}}}}
2025-11-19T13:59:49.963309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435916846606554:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:843
2025-11-19T13:59:49.964070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435916846606543:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16200322ns) {Configure:104(16200035ns) {ProcessConfigured:135(15970992ns) {StartInitialHandshake:157(15970738ns) {DropHandshakes:469(341ns) {DropIncomingHandshake:441(32ns) DropOutgoingHandshake:456(90ns)} GenerateSessionVirtualId:988(516ns) PrepareNewSessionHandshake:476(579ns) {SwitchToState:178(302ns)}}}}}
2025-11-19T13:59:49.963946Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435914673337931:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.964326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435914673337937:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24648
::ffff:127.1.0.0
2025-11-19T13:59:49.964648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435917330476808:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18162
2025-11-19T13:59:49.967934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435916846606542:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(17377528ns) {Configure:104(17375729ns) {ProcessConfigured:135(17029919ns) {StartInitialHandshake:157(17028939ns) {DropHandshakes:469(1282ns) {DropIncomingHandshake:441(167ns) DropOutgoingHandshake:456(158ns)} GenerateSessionVirtualId:988(532ns) PrepareNewSessionHandshake:476(2432ns) {SwitchToState:178(1373ns)}}}}}
2025-11-19T13:59:49.969125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435915268826638:2076] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.969170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435915268826638:2076] [node 4] ICP35 event processing took too much time PendingConnection:278(20948606ns) {IncomingHandshake:261(20948012ns) {DropIncomingHandshake:441(61ns) PrepareNewSessionHandshake:476(441ns) {SwitchToState:178(207ns)} GenerateSessionVirtualId:988(234ns) IssueIncomingHandshakeReply:193(20942686ns) {SwitchToState:178(215ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.970642Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435914673337923:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:49.970505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435915268826685:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34847
2025-11-19T13:59:49.970542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435915268826686:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34847
2025-11-19T13:59:49.971843Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435915268826685:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.971872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435915268826605:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.969598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435916846606560:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:843
2025-11-19T13:59:49.969656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435916846606561:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:843
2025-11-19T13:59:49.969688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435916846606562:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:843
2025-11-19T13:59:49.971594Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435916846606561:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.971597Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435916846606560:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.971615Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435916846606562:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.971641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435916846606478:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.972168Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435916257234104:2092] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:49.975873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435914673337851:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(51178396ns) {Configure:104(51164444ns) {ProcessConfigured:135(48935631ns) {StartInitialHandshake:157(48935070ns) {DropHandshakes:469(257ns) {DropIncomingHandshake:441(26ns) DropOutgoingHandshake:456(18ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:476(810ns) {SwitchToState:178(639ns)}}}}}
2025-11-19T13:59:49.975899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435914673337921:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.975918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:49.975926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435914673337921:2109] [node 6] ICP22 created new session: [1:7574435914673337939:2109]
2025-11-19T13:59:49.975933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435914673337939:2109] [node 6] ICS09 handshake done sender: [1:7574435914673337923:2112] self: [1:7574435914673337922:0] peer: [6:7574435916921387953:0] socket: 645 qp: -1
2025-11-19T13:59:49.975940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435914673337939:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:49.975959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:49.976043Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435914673337929:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.976051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435914673337929:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.976854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435915268826688:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34847
2025-11-19T13:59:49.976116Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435914673337925:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.976122Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435914673337925:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.977451Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435914673337925:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:49.977585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435916921387954:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-19T13:59:49.977627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435916921387955:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-19T13:59:49.977777Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435916921387951:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.980700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435915268826689:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34847
2025-11-19T13:59:49.980927Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435915268826681:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:49.981273Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435915268826685:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:49.981853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435916846606566:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:843
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.982875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435915268826638:2076] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.982900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:49.982906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435915268826638:2076] [node 4] ICP22 created new session: [5:7574435915268826690:2076]
2025-11-19T13:59:49.982911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435915268826690:2076] [node 4] ICS09 handshake done sender: [5:7574435915268826681:2113] self: [5:7574435915268826684:0] peer: [4:7574435916257234103:0] socket: 641 qp: -1
2025-11-19T13:59:49.982916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435915268826690:2076] [node 4] ICS10 traffic start
2025-11-19T13:59:49.982926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:49.983134Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435914673337929:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:49.979894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435916921387872:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.979919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:49.979926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435916921387872:2048] [node 1] ICP22 created new session: [6:7574435916921387956:2048]
2025-11-19T13:59:49.979932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435916921387956:2048] [node 1] ICS09 handshake done sender: [6:7574435916921387951:2116] self: [6:7574435916921387953:0] peer: [1:7574435914673337922:0] socket: 648 qp: -1
2025-11-19T13:59:49.979940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435916921387956:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:49.979962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:49.981157Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435916921387950:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:49.983161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435916921387900:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.983174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:49.983184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435916921387900:2070] [node 4] ICP22 created new session: [6:7574435916921387959:2070]
2025-11-19T13:59:49.983189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435916921387959:2070] [node 4] ICS09 handshake done sender: [6:7574435916921387950:2115] self: [6:7574435916921387952:0] peer: [4:7574435916257234107:0] socket: 647 qp: -1
2025-11-19T13:59:49.983195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435916921387959:2070] [node 4] ICS10 traffic start
2025-11-19T13:59:49.983214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:49.981416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435916257234100:2088] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.981438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:49.981449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435916257234100:2088] [node 5] ICP22 created new session: [4:7574435916257234142:2088]
2025-11-19T13:59:49.981455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435916257234142:2088] [node 5] ICS09 handshake done sender: [4:7574435916257234104:2092] self: [4:7574435916257234103:0] peer: [5:7574435915268826684:0] socket: 630 qp: -1
2025-11-19T13:59:49.981463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435916257234142:2088] [node 5] ICS10 traffic start
2025-11-19T13:59:49.981481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:49.982011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435916257234144:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51582
2025-11-19T13:59:49.984934Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435916257234114:2098] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.984968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435916257234058:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.986872Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435916846606561:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:49.988992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435917330476809:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18162
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.989492Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435917330476790:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.989504Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435917330476790:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.989184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435915268826692:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34847
2025-11-19T13:59:49.989870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435914673337912:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.989890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:49.989895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435914673337912:2107] [node 5] ICP22 created new session: [1:7574435914673337944:2107]
2025-11-19T13:59:49.989900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435914673337944:2107] [node 5] ICS09 handshake done sender: [1:7574435914673337925:2113] self: [1:7574435914673337915:0] peer: [5:7574435915268826687:0] socket: 650 qp: -1
2025-11-19T13:59:49.989906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435914673337944:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:49.989914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.990728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435916921387961:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-19T13:59:49.991167Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435916257234144:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.991208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435916257234057:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.991599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435914673337852:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.991613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:49.991619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435914673337852:2049] [node 3] ICP22 created new session: [1:7574435914673337946:2049]
2025-11-19T13:59:49.991626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435914673337946:2049] [node 3] ICS09 handshake done sender: [1:7574435914673337929:2114] self: [1:7574435914673337918:0] peer: [3:7574435916846606563:0] socket: 652 qp: -1
2025-11-19T13:59:49.991631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435914673337946:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:49.991648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:49.991846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435916846606478:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:49.991869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:49.991877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435916846606478:2048] [node 1] ICP22 created new session: [3:7574435916846606567:2048]
2025-11-19T13:59:49.991886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435916846606567:2048] [node 1] ICS09 handshake done sender: [3:7574435916846606561:2118] self: [3:7574435916846606563:0] peer: [1:7574435914673337918:0] socket: 667 qp: -1
2025-11-19T13:59:49.991893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435916846606567:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:49.991911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:49.991905Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435915268826682:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.991912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435915268826682:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.991793Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435916921387961:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.991827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435916921387898:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.993641Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435916846606554:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.993689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435916846606479:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.995708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435916846606571:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:843
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.995989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435916921387963:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-19T13:59:49.993923Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435917330476808:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.993969Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435917330476788:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.993973Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435917330476788:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:49.996009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435916921387964:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-19T13:59:49.995347Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435917330476788:2108] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:49.995473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435917330476726:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.995487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:49.995494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435917330476726:2049] [node 3] ICP22 created new session: [2:7574435917330476812:2049]
2025-11-19T13:59:49.995501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435917330476812:2049] [node 3] ICS09 handshake done sender: [2:7574435917330476788:2108] self: [2:7574435917330476787:0] peer: [3:7574435916846606570:0] socket: 637 qp: -1
2025-11-19T13:59:49.995508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435917330476812:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:49.995515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:49.996246Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435914673337932:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.996258Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435914673337932:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:49.997790Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435914673337932:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:49.998522Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435914673337937:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.998534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435914673337917:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:49.998546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:49.998552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435914673337917:2108] [node 4] ICP22 created new session: [1:7574435914673337950:2108]
2025-11-19T13:59:49.998557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435914673337950:2108] [node 4] ICS09 handshake done sender: [1:7574435914673337932:2116] self: [1:7574435914673337927:0] peer: [4:7574435916257234146:0] socket: 674 qp: -1
2025-11-19T13:59:49.998561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435914673337950:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:49.998568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:50.000151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435916846606479:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62022) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435916846606554:2113] held: no
2025-11-19T13:59:50.000204Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435916846606554:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:50.000833Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435915268826686:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:50.000860Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435915268826692:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:50.000891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435915268826607:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:50.000749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435916846606479:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:50.000764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:50.000771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435916846606479:2049] [node 2] ICP22 created new session: [3:7574435921141573868:2049]
2025-11-19T13:59:50.000777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435921141573868:2049] [node 2] ICS09 handshake done sender: [3:7574435916846606554:2113] self: [3:7574435916846606570:0] peer: [2:7574435917330476787:0] socket: 646 qp: -1
2025-11-19T13:59:50.000790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435921141573868:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:50.000809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:50.001022Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435916846606555:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:50.001031Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435916846606555:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:50.001674Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435916921387964:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:50.001722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435916921387873:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:50.001821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435916257234149:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51582
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:50.002004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435915268826696:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34847
2025-11-19T13:59:50.002019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435915268826697:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34847
2025-11-19T13:59:50.002108Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435916846606555:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:50.002276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435916846606545:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:50.002293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:50.002298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435916846606545:2108] [node 5] ICP22 created new session: [3:7574435921141573872:2108]
2025-11-19T13:59:50.002303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435921141573872:2108] [node 5] ICS09 handshake done sender: [3:7574435916846606555:2114] self: [3:7574435916846606551:0] peer: [5:7574435915268826694:0] socket: 666 qp: -1
2025-11-19T13:59:50.002307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435921141573872:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:50.002312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:50.003782Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435916257234149:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:50.003832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435916257234059:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:49.998563Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435917330476809:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:49.998944Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435917330476790:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:50.000279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435917330476786:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:50.000300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:50.000308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435917330476786:2107] [node 4] ICP22 created new session: [2:7574435917330476817:2107]
2025-11-19T13:59:50.000316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435917330476817:2107] [node 4] ICS09 handshake done sender: [2:7574435917330476790:2109] self: [2:7574435917330476789:0] peer: [4:7574435916257234145:0] socket: 638 qp: -1
2025-11-19T13:59:50.000323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435917330476817:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:50.000331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:50.001996Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435917330476804:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:50.002000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435917330476804:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:50.003992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435915268826605:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:50.004011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:50.004017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435915268826605:2048] [node 1] ICP22 created new session: [5:7574435919563793994:2048]
2025-11-19T13:59:50.004024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435919563793994:2048] [node 1] ICS09 handshake done sender: [5:7574435915268826685:2116] self: [5:7574435915268826687:0] peer: [1:7574435914673337915:0] socket: 654 qp: -1
2025-11-19T13:59:50.004031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435919563793994:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:50.004038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:50.004095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435916257234147:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51582
2025-11-19T13:59:50.004156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435916257234148:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51582
2025-11-19T13:59:50.004322Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435916257234114:2098] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:50.009039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435914673337912:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(19133537ns) {HandleHandshakeStatus:329(19132888ns) {DropIncomingHandshake:441(3650ns) DropOutgoingHandshake:456(279ns) SwitchToState:178(73ns) ProcessPendingSessionEvents:495(19062747ns) {ForwardSessionEventToSession:612(19050289ns) ForwardSessionEventToSession:612(1426ns) ForwardSessionEventToSession:612(692ns) ForwardSessionEventToSession:612(733ns) ForwardSessionEventToSession:612(807ns) ForwardSessionEventToSession:612(1096ns) ForwardSessionEventToSession:612(1090ns) ForwardSessionEventToSession:612(528ns) ForwardSessionEventToSession:612(671ns) ForwardSessionEventToSession:612(1046ns)}}}
2025-11-19T13:59:50.011310Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435915268826686:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:50.010582Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435916846606478:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(18693793ns) {HandleHandshakeStatus:329(18692924ns) {DropIncomingHandshake:441(289ns) DropOutgoingHandshake:456(1818ns) SwitchToState:178(78ns) ProcessPendingSessionEvents:495(18580289ns) {ForwardSessionEventToSession:612(18566956ns) ForwardSessionEventToSession:612(2026ns) ForwardSessionEventToSession:612(1062ns) ForwardSessionEventToSession:612(830ns) ForwardSessionEventToSession:612(586ns) ForwardSessionEventToSession:612(1365ns) ForwardSessionEventToSession:612(700ns) ForwardSessionEventToSession:612(519ns) ForwardSessionEventToSession:612(655ns) ForwardSessionEventToSession:612(868ns)}}}
2025-11-19T13:59:50.007964Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435916257234144:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:50.007989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435916257234058:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:50.008007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:50.008014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435916257234058:2049] [node 2] ICP22 created new session: [4:7574435920552201447:2049]
2025-11-19T13:59:50.008021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435920552201447:2049] [node 2] ICS09 handshake done sender: [4:7574435916257234114:2098] self: [4:7574435916257234145:0] peer: [2:7574435917330476789:0] socket: 639 qp: -1
2025-11-19T13:59:50.008029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435920552201447:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:50.008037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:50.011833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435916257234057:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:50.011849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:50.011857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435916257234057:2048] [node 1] ICP22 created new session: [4:7574435920552201449:2048]
2025-11-19T13:59:50.011864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435920552201449:2048] [node 1] ICS09 handshake done sender: [4:7574435916257234144:2116] self: [4:7574435916257234146:0] peer: [1:7574435914673337927:0] socket: 677 qp: -1
2025-11-19T13:59:50.011871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435920552201449:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:50.011879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:50.014098Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435915268826638:2076] [node 4] ICP35 event processing took too much time PendingConnection:278(31196996ns) {HandleHandshakeStatus:329(31196054ns) {DropIncomingHandshake:441(211ns) DropOutgoingHandshake:456(6099ns) SwitchToState:178(171ns) ProcessPendingSessionEvents:495(24307310ns) {ForwardSessionEventToSession:612(24294598ns) ForwardSessionEventToSession:612(2226ns) ForwardSessionEventToSession:612(968ns) ForwardSessionEventToSession:612(540ns) ForwardSessionEventToSession:612(657ns) ForwardSessionEventToSession:612(988ns) ForwardSessionEventToSession:612(668ns) ForwardSessionEventToSession:612(906ns) ForwardSessionEventToSession:612(508ns) ForwardSessionEventToSession:612(804ns)}}}
2025-11-19T13:59:50.014123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435915268826607:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19203) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435915268826686:2117] held: no
2025-11-19T13:59:50.014800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435915268826607:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:50.014814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:50.014821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435915268826607:2050] [node 3] ICP22 created new session: [5:7574435919563793997:2050]
2025-11-19T13:59:50.014836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435919563793997:2050] [node 3] ICS09 handshake done sender: [5:7574435915268826686:2117] self: [5:7574435915268826694:0] peer: [3:7574435916846606551:0] socket: 670 qp: -1
2025-11-19T13:59:50.014842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435919563793997:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:50.014848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:50.012283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435921625444116:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18162
2025-11-19T13:59:50.013585Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435921625444116:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:50.014011Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435917330476804:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:50.015836Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435916846606559:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:50.015851Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435916846606559:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:50.015978Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435916921387963:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:50.016089Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435915268826682:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:50.016166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435916921387874:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:50.017903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435915268826637:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:50.017921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:50.017927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435915268826637:2069] [node 6] ICP22 created new session: [5:7574435919563793999:2069]
2025-11-19T13:59:50.017932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435919563793999:2069] [node 6] ICS09 handshake done sender: [5:7574435915268826682:2114] self: [5:7574435915268826639:0] peer: [6:7574435916921387962:0] socket: 634 qp: -1
2025-11-19T13:59:50.017937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435919563793999:2069] [node 6] ICS10 traffic start
2025-11-19T13:59:50.017944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:50.016831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435914673337852:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(25203249ns) {HandleHandshakeStatus:329(25202487ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(290ns) SwitchToState:178(67ns) ProcessPendingSessionEvents:495(12714754ns) {ForwardSessionEventToSession:612(12701902ns) ForwardSessionEventToSession:612(2125ns) ForwardSessionEventToSession:612(835ns) ForwardSessionEventToSession:612(549ns) ForwardSessionEventToSession:612(740ns) ForwardSessionEventToSession:612(869ns) ForwardSessionEventToSession:612(554ns) ForwardSessionEventToSession:612(453ns) ForwardSessionEventToSession:612(550ns) ForwardSessionEventToSession:612(1069ns)}}}
2025-11-19T13:59:50.016940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435914673337852:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:50.017099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435914673337917:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(18538751ns) {HandleHandshakeStatus:329(18538157ns) {DropIncomingHandshake:441(2905ns) DropOutgoingHandshake:456(154ns) SwitchToState:178(172ns) ProcessPendingSessionEvents:495(12923666ns) {ForwardSessionEventToSession:612(12914675ns) ForwardSessionEventToSession:612(1196ns) ForwardSessionEventToSession:612(882ns) ForwardSessionEventToSession:612(528ns) ForwardSessionEventToSession:612(583ns) ForwardSessionEventToSession:612(1169ns) ForwardSessionEventToSession:612(765ns) ForwardSessionEventToSession:612(728ns) ForwardSessionEventToSession:612(543ns) ForwardSessionEventToSession:612(700ns)}}}
2025-11-19T13:59:50.019982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435915268826605:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(36758462ns) {HandleHandshakeStatus:329(36758017ns) {DropIncomingHandshake:441(480ns) DropOutgoingHandshake:456(1661ns) SwitchToState:178(65ns) ProcessPendingSessionEvents:495(5481542ns) {ForwardSessionEventToSession:612(5469561ns) ForwardSessionEventToSession:612(1205ns) ForwardSessionEventToSession:612(1067ns) ForwardSessionEventToSession:612(961ns) ForwardSessionEventToSession:612(633ns) ForwardSessionEventToSession:612(995ns) ForwardSessionEventToSession:612(712ns) ForwardSessionEventToSession:612(731ns) ForwardSessionEventToSession:612(632ns) ForwardSessionEventToSession:612(1066ns)}}}
2025-11-19T13:59:50.020007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435915268826605:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:50.020895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435917330476794:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:50.020927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:50.020934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435917330476794:2110] [node 6] ICP22 created new session: [2:7574435921625444119:2110]
2025-11-19T13:59:50.020940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435921625444119:2110] [node 6] ICS09 handshake done sender: [2:7574435917330476804:2115] self: [2:7574435917330476798:0] peer: [6:7574435916921387965:0] socket: 687 qp: -1
2025-11-19T13:59:50.020947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435921625444119:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:50.020955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:50.019718Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435914673337934:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:50.019732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435914673337934:2118] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:50.021604Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435915268826697:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:50.021653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435915268826606:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:50.022944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435919563794002:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34847
2025-11-19T13:59:50.023915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435916921387898:2067] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8072) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435916921387961:2119] held: no
2025-11-19T13:59:50.023944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435916921387873:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62022) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435916921387964:2121] held: no
2025-11-19T13:59:50.021858Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435917330476806:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:50.021870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435917330476806:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:50.023282Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435917330476806:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:50.023973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435917330476796:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:50.023990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:50.023999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435917330476796:2111] [node 5] ICP22 created new session: [2:7574435921625444122:2111]
2025-11-19T13:59:50.024004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435921625444122:2111] [node 5] ICS09 handshake done sender: [2:7574435917330476806:2116] self: [2:7574435917330476803:0] peer: [5:7574435919563794001:0] socket: 657 qp: -1
2025-11-19T13:59:50.024011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435921625444122:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:50.024018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:50.024899Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435916846606557:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:50.024912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435916846606557:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:50.025246Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435916846606559:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:50.025313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435916846606542:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:50.025330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:50.025336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435916846606542:2107] [node 4] ICP22 created new session: [3:7574435921141573875:2107]
2025-11-19T13:59:50.025343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435921141573875:2107] [node 4] ICS09 handshake done sender: [3:7574435916846606559:2116] self: [3:7574435916846606553:0] peer: [4:7574435916257234150:0] socket: 656 qp: -1
2025-11-19T13:59:50.025348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435921141573875:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:50.025354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:50.024968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435917330476725:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:50.026632Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435915268826697:2122] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:50.026755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435915268826606:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:50.026774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:50.026780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435915268826606:2049] [node 2] ICP22 created new session: [5:7574435919563794003:2049]
2025-11-19T13:59:50.026785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435919563794003:2049] [node 2] ICS09 handshake done sender: [5:7574435915268826697:2122] self: [5:7574435919563794001:0] peer: [2:7574435917330476803:0] socket: 696 qp: -1
2025-11-19T13:59:50.026790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435919563794003:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:50.026800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:50.029091Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435914673337934:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:50.029305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435914673337851:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:50.029333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:50.029343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435914673337851:2048] [node 2] ICP22 created new session: [1:7574435918968305250:2048]
2025-11-19T13:59:50.029353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435918968305250:2048] [node 2] ICS09 handshake done sender: [1:7574435914673337934:2118] self: [1:7574435914673337916:0] peer: [2:7574435921625444117:0] socket: 690 qp: -1
2025-11-19T13:59:50.029359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435918968305250:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:50.029367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:50.038428Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435916846606557:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:50.038547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435916846606543:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:50.039869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435920552201451:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51582
2025-11-19T13:59:50.039999Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435916257234149:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:50.040066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435916257234059:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:50.040084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:50.040181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435916921387874:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19203) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435916921387963:2120] held: no
2025-11-19T13:59:50.040092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435916257234059:2050] [node 3] ICP22 created new session: [4:7574435920552201452:2050]
2025-11-19T13:59:50.040104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435920552201452:2050] [node 3] ICS09 handshake done sender: [4:7574435916257234149:2119] self: [4:7574435916257234150:0] peer: [3:7574435916846606553:0] socket: 694 qp: -1
2025-11-19T13:59:50.040110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435920552201452:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:50.040120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:50.043890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:50.043907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435916846606543:2109] [node 6] ICP22 created new session: [3:7574435921141573878:2109]
2025-11-19T13:59:50.043916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435921141573878:2109] [node 6] ICS09 handshake done sender: [3:7574435916846606557:2115] self: [3:7574435916846606552:0] peer: [6:7574435921216355262:0] socket: 684 qp: -1
2025-11-19T13:59:50.043922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435921141573878:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:50.043937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:50.044510Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435915268826606:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(17726723ns) {HandleHandshakeStatus:329(17726370ns) {DropIncomingHandshake:441(166ns) DropOutgoingHandshake:456(6378ns) SwitchToState:178(78ns) ProcessPendingSessionEvents:495(8551299ns) {ForwardSessionEventToSession:612(8538894ns) ForwardSessionEventToSession:612(1468ns) ForwardSessionEventToSession:612(885ns) ForwardSessionEventToSession:612(494ns) ForwardSessionEventToSession:612(502ns) ForwardSessionEventToSession:612(762ns) ForwardSessionEventToSession:612(833ns) ForwardSessionEventToSession:612(897ns) ForwardSessionEventToSession:612(650ns) ForwardSessionEventToSession:612(932ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:50.044415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435921216355263:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-19T13:59:50.044686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435921216355264:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-19T13:59:50.049500Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435917330476725:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18528) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435921625444116:2119] held: no
2025-11-19T13:59:50.052263Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435916921387964:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:50.052301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435921216355265:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47368
2025-11-19T13:59:50.055259Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435916921387961:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:50.055907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435921625444124:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18162
2025-11-19T13:59:50.055501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435916921387873:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:50.055529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:50.055540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435916921387873:2049] [node 2] ICP22 created new session: [6:7574435921216355266:2049]
2025-11-19T13:59:50.055548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435921216355266:2049] [node 2] ICS09 handshake done sender: [6:7574435916921387964:2121] self: [6:7574435916921387965:0] peer: [2:7574435917330476798:0] socket: 689 qp: -1
2025-11-19T13:59:50.055560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435921216355266:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:50.055572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:50.056153Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435916921387963:2120] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:50.056259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435916257234059:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(16163965ns) {HandleHandshakeStatus:329(16163588ns) {DropIncomingHandshake:441(126ns) DropOutgoingHandshake:456(5919ns) SwitchToState:178(309ns) ProcessPendingSessionEvents:495(8341065ns) {ForwardSessionEventToSession:612(8327654ns) ForwardSessionEventToSession:612(2412ns) ForwardSessionEventToSession:612(1173ns) ForwardSessionEventToSession:612(489ns) ForwardSessionEventToSession:612(558ns) ForwardSessionEventToSession:612(1222ns) ForwardSessionEventToSession:612(616ns) ForwardSessionEventToSession:612(804ns) ForwardSessionEventToSession:612(756ns) ForwardSessionEventToSession:612(827ns)}}}
2025-11-19T13:59:50.057173Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435921625444116:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:50.057265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435916921387898:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:50.057281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:50.057288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435916921387898:2067] [node 5] ICP22 created new session: [6:7574435921216355268:2067]
2025-11-19T13:59:50.057294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435921216355268:2067] [node 5] ICS09 handshake done sender: [6:7574435916921387961:2119] self: [6:7574435916921387962:0] peer: [5:7574435915268826639:0] socket: 683 qp: -1
2025-11-19T13:59:50.057301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435921216355268:2067] [node 5] ICS10 traffic start
2025-11-19T13:59:50.057309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:50.057555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435916921387874:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:50.057568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:50.057574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435916921387874:2050] [node 3] ICP22 created new session: [6:7574435921216355270:2050]
2025-11-19T13:59:50.057579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435921216355270:2050] [node 3] ICS09 handshake done sender: [6:7574435916921387963:2120] self: [6:7574435921216355262:0] peer: [3:7574435916846606552:0] socket: 688 qp: -1
2025-11-19T13:59:50.057584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435921216355270:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:50.057590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:50.057823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435917330476725:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:50.057837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:50.057841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435917330476725:2048] [node 1] ICP22 created new session: [2:7574435921625444125:2048]
2025-11-19T13:59:50.057846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435921625444125:2048] [node 1] ICS09 handshake done sender: [2:7574435921625444116:2119] self: [2:7574435921625444117:0] peer: [1:7574435914673337916:0] socket: 660 qp: -1
2025-11-19T13:59:50.057852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435921625444125:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:50.057861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:50.067857Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:50.067953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435921625444125:2048] [node 1] ICS01 socket: 660 reason# EndOfStream
2025-11-19T13:59:50.067973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435917330476725:2048] [node 1] ICP30 unregister session Session# [2:7574435921625444125:2048] VirtualId# [2:7574435921625444117:0]
2025-11-19T13:59:50.067981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435921625444125:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:50.068023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:50.068045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:50.075880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:50.075984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435916846606567:2048] [node 1] ICS01 socket: 667 reason# EndOfStream
2025-11-19T13:59:50.075995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435916846606478:2048] [node 1] ICP30 unregister session Session# [3:7574435916846606567:2048] VirtualId# [3:7574435916846606563:0]
2025-11-19T13:59:50.076005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435916846606567:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:50.076063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:50.076093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:50.083967Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:50.084068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435920552201449:2048] [node 1] ICS07 socket disconnect 677 reason# EndOfStream
2025-11-19T13:59:50.084078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435920552201449:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:50.084141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:50.084167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435920552201449:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:50.085313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:50.085374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435916921387956:2048] [node 1] ICS01 socket: 648 reason# EndOfStream
2025-11-19T13:59:50.085379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435916921387872:2048] [node 1] ICP30 unregister session Session# [6:7574435916921387956:2048] VirtualId# [6:7574435916921387953:0]
2025-11-19T13:59:50.085383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435916921387956:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:50.085416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:50.085439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:50.084757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435916257234057:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18528) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:50.084769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435920552201449:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:50.084775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435916257234057:2048] [node 1] ICP30 unregister session Session# [4:7574435920552201449:2048] VirtualId# [4:7574435916257234146:0]
2025-11-19T13:59:50.084791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:50.084796Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435916257234057:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18528) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:50.084799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18528) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:50.085020Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:50.085087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435919563793994:2048] [node 1] ICS07 socket disconnect 654 reason# EndOfStream
2025-11-19T13:59:50.085094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435919563793994:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:50.085134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:50.085157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435919563793994:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:50.086286Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435915268826605:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18528) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:50.086292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435919563793994:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:50.086297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435915268826605:2048] [node 1] ICP30 unregister session Session# [5:7574435919563793994:2048] VirtualId# [5:7574435915268826687:0]
2025-11-19T13:59:50.086309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:50.086313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435915268826605:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18528) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:50.086315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18528) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:50.094885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435915268826605:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:50.096226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435916257234057:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:50.099839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:50.099952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435921141573868:2049] [node 2] ICS01 socket: 646 reason# EndOfStream
2025-11-19T13:59:50.099962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435916846606479:2049] [node 2] ICP30 unregister session Session# [3:7574435921141573868:2049] VirtualId# [3:7574435916846606570:0]
2025-11-19T13:59:50.099970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435921141573868:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:50.100039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:50.100077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:50.111916Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:50.112051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435921216355266:2049] [node 2] ICS01 socket: 689 reason# EndOfStream
2025-11-19T13:59:50.112063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435916921387873:2049] [node 2] ICP30 unregister session Session# [6:7574435921216355266:2049] VirtualId# [6:7574435916921387965:0]
2025-11-19T13:59:50.112072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435921216355266:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:50.112148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:50.112179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:50.112339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:50.112434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435920552201447:2049] [node 2] ICS01 socket: 639 reason# EndOfStream
2025-11-19T13:59:50.112442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435916257234058:2049] [node 2] ICP30 unregister session Session# [4:7574435920552201447:2049] VirtualId# [4:7574435916257234145:0]
2025-11-19T13:59:50.112450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435920552201447:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:50.112499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:50.112523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:50.112557Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:50.112622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435919563794003:2049] [node 2] ICS07 socket disconnect 696 reason# EndOfStream
2025-11-19T13:59:50.112627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435919563794003:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:50.112652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:50.112672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435919563794003:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:50.120769Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435915268826606:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62022) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:50.120790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435919563794003:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:50.120795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435915268826606:2049] [node 2] ICP30 unregister session Session# [5:7574435919563794003:2049] VirtualId# [5:7574435919563794001:0]
2025-11-19T13:59:50.120812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:50.120818Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435915268826606:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62022) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:50.120822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62022) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:50.126775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:50.126868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435919563793997:2050] [node 3] ICS01 socket: 670 reason# EndOfStream
2025-11-19T13:59:50.126877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435915268826607:2050] [node 3] ICP30 unregister session Session# [5:7574435919563793997:2050] VirtualId# [5:7574435915268826694:0]
2025-11-19T13:59:50.126887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435919563793997:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:50.126942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:50.126967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:50.130062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:50.130156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435921216355270:2050] [node 3] ICS01 socket: 688 reason# EndOfStream
2025-11-19T13:59:50.130164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435916921387874:2050] [node 3] ICP30 unregister session Session# [6:7574435921216355270:2050] VirtualId# [6:7574435921216355262:0]
2025-11-19T13:59:50.130171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435921216355270:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:50.130228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:50.130252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:50.131848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435915268826606:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:50.140628Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:50.140718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435916921387959:2070] [node 4] ICS07 socket disconnect 647 reason# EndOfStream
2025-11-19T13:59:50.140724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435916921387959:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:50.140767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:50.140787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435916921387959:2070] [node 4] ICS15 start handshake
2025-11-19T13:59:50.145754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435916921387900:2070] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32457) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:50.145775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435916921387959:2070] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:50.145782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435916921387900:2070] [node 4] ICP30 unregister session Session# [6:7574435916921387959:2070] VirtualId# [6:7574435916921387952:0]
2025-11-19T13:59:50.145800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:50.145806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435916921387900:2070] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32457) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:50.145808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32457) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-11-19T13:59:51.568312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435925674853647:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:51.568330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435925674853648:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:51.574463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435923636355936:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:51.574479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435923636355937:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:51.595919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435922576464369:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:51.595940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435922576464370:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:51.624129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435922442723077:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:51.624147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435922442723078:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:51.624152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435922442723079:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:51.643970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435922150831424:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:51.643991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435922150831425:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:51.644002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435922150831426:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:51.655329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435922372782167:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:51.655344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435922372782168:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:51.655348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435922372782169:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.684717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435925674853707:2103] [node 4] ICP01 ready to work
2025-11-19T13:59:51.684827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435925674853708:2104] [node 5] ICP01 ready to work
2025-11-19T13:59:51.684164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435922576464409:2086] [node 4] ICP01 ready to work
2025-11-19T13:59:51.692641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435923636355994:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14437
2025-11-19T13:59:51.692166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435922372782202:2063] [node 5] ICP01 ready to work
2025-11-19T13:59:51.692301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435922372782203:2068] [node 4] ICP01 ready to work
2025-11-19T13:59:51.692889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435923636355999:2106] [node 4] ICP01 ready to work
2025-11-19T13:59:51.692972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435923636356000:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:51.696633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435922150831472:2079] [node 6] ICP01 ready to work
2025-11-19T13:59:51.696742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435922150831473:2083] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.703405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435923636356010:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:51.703743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435923636356011:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14437
2025-11-19T13:59:51.708552Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435922576464409:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24314513ns) {Configure:104(24313558ns) {ProcessConfigured:135(23879823ns) {StartInitialHandshake:157(23879200ns) {DropHandshakes:469(580ns) {DropIncomingHandshake:441(87ns) DropOutgoingHandshake:456(90ns)} GenerateSessionVirtualId:988(184ns) PrepareNewSessionHandshake:476(2401ns) {SwitchToState:178(1160ns)}}}}}
2025-11-19T13:59:51.708750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435922576464434:2088] [node 6] ICP01 ready to work
2025-11-19T13:59:51.708866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435922576464435:2094] [node 5] ICP01 ready to work
2025-11-19T13:59:51.709350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435925674853719:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.713620Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435922442723077:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(28398295ns) {Configure:104(28397327ns) {ProcessConfigured:135(28057336ns) {StartInitialHandshake:157(28056814ns) {DropHandshakes:469(314ns) {DropIncomingHandshake:441(33ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:476(1508ns) {SwitchToState:178(283ns)}}}}}
2025-11-19T13:59:51.714443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435922442723146:2080] [node 5] ICP01 ready to work
2025-11-19T13:59:51.714574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435922442723147:2104] [node 6] ICP01 ready to work
2025-11-19T13:59:51.715206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435922442723148:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.725579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435922576464437:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23910
2025-11-19T13:59:51.725670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435922576464438:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23910
2025-11-19T13:59:51.725691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435922576464439:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23910
2025-11-19T13:59:51.728204Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435922150831424:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(30987968ns) {Configure:104(30987323ns) {ProcessConfigured:135(30768965ns) {StartInitialHandshake:157(30768643ns) {DropHandshakes:469(156ns) {DropIncomingHandshake:441(30ns) DropOutgoingHandshake:456(29ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:476(1608ns) {SwitchToState:178(865ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.743158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435925674853722:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:353
2025-11-19T13:59:51.743244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435925674853723:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:353
2025-11-19T13:59:51.743262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435925674853724:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:353
2025-11-19T13:59:51.744471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435922150831499:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14623
2025-11-19T13:59:51.744563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435922150831500:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14623
2025-11-19T13:59:51.747269Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435922150831499:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.760153Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435923636356011:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.760337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435923636356017:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14437
2025-11-19T13:59:51.760400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435923636356018:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14437
2025-11-19T13:59:51.765581Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435923636355994:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.765607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435923636355936:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:51.765280Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435925674853722:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.765324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435925674853729:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:353
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.768216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435922442723156:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
2025-11-19T13:59:51.768238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435922442723157:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
2025-11-19T13:59:51.772077Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435923636356017:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.774634Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435925674853703:2102] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.774653Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435925674853703:2102] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.775973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435923636356022:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14437
2025-11-19T13:59:51.780157Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435923636356018:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.777232Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435922576464437:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.780562Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435922576464438:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.780632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435922576464451:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23910
2025-11-19T13:59:51.780666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435922576464452:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23910
2025-11-19T13:59:51.780702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435922576464370:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:51.781875Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435923636356005:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.781888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435923636356005:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.782990Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435922442723148:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.783022Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435922442723157:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.783119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435922442723078:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:51.783128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435922442723077:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:51.783307Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435923636356005:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:51.783431Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435923636356007:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.783439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435923636356007:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.783855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435923636355937:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.783869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:51.783877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435923636355937:2049] [node 3] ICP22 created new session: [2:7574435923636356025:2049]
2025-11-19T13:59:51.783883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435923636356025:2049] [node 3] ICS09 handshake done sender: [2:7574435923636356005:2112] self: [2:7574435923636356004:0] peer: [3:7574435922576464454:0] socket: 43 qp: -1
2025-11-19T13:59:51.783891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435923636356025:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:51.783902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:51.784570Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435925674853703:2102] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:51.784814Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435925674853723:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.784845Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435925674853713:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.784847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435925674853713:2107] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.785280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435925674853647:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.785294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:51.785299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435925674853647:2048] [node 2] ICP22 created new session: [1:7574435925674853732:2048]
2025-11-19T13:59:51.785305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435925674853732:2048] [node 2] ICS09 handshake done sender: [1:7574435925674853703:2102] self: [1:7574435925674853702:0] peer: [2:7574435923636356020:0] socket: 36 qp: -1
2025-11-19T13:59:51.785310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435925674853732:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:51.785318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:51.785776Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435923636356007:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:51.785994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435923636355999:2106] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.786010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:51.786016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435923636355999:2106] [node 4] ICP22 created new session: [2:7574435923636356029:2106]
2025-11-19T13:59:51.786021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435923636356029:2106] [node 4] ICS09 handshake done sender: [2:7574435923636356007:2113] self: [2:7574435923636356006:0] peer: [4:7574435922442723158:0] socket: 44 qp: -1
2025-11-19T13:59:51.786025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435923636356029:2106] [node 4] ICS10 traffic start
2025-11-19T13:59:51.786031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.786757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435922442723160:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
2025-11-19T13:59:51.787471Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435925674853713:2107] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.788118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435922442723161:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
2025-11-19T13:59:51.788809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435922442723078:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25912) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435922442723148:2113] held: no
2025-11-19T13:59:51.788905Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435922442723148:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:51.789030Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435922442723157:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:51.789428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435925674853707:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.789450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:51.789458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435925674853707:2103] [node 4] ICP22 created new session: [1:7574435925674853737:2103]
2025-11-19T13:59:51.789464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435925674853737:2103] [node 4] ICS09 handshake done sender: [1:7574435925674853713:2107] self: [1:7574435925674853712:0] peer: [4:7574435922442723159:0] socket: 60 qp: -1
2025-11-19T13:59:51.789469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435925674853737:2103] [node 4] ICS10 traffic start
2025-11-19T13:59:51.789477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:51.789168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435922442723078:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.789183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:51.789189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435922442723078:2049] [node 2] ICP22 created new session: [4:7574435922442723162:2049]
2025-11-19T13:59:51.789194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435922442723162:2049] [node 2] ICS09 handshake done sender: [4:7574435922442723148:2113] self: [4:7574435922442723158:0] peer: [2:7574435923636356006:0] socket: 47 qp: -1
2025-11-19T13:59:51.789200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435922442723162:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:51.789207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:51.789431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435922442723077:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.789439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:51.789443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435922442723077:2048] [node 1] ICP22 created new session: [4:7574435922442723164:2048]
2025-11-19T13:59:51.789447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435922442723164:2048] [node 1] ICS09 handshake done sender: [4:7574435922442723157:2117] self: [4:7574435922442723159:0] peer: [1:7574435925674853712:0] socket: 68 qp: -1
2025-11-19T13:59:51.789451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435922442723164:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:51.789455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.790671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435922150831505:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14623
2025-11-19T13:59:51.790692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435922150831506:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14623
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.791070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435922150831507:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14623
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.791367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435922372782245:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63827
2025-11-19T13:59:51.792782Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435925674853729:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.794041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435922372782246:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63827
2025-11-19T13:59:51.794161Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435922150831507:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.794192Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435922150831500:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.794217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435922150831473:2083] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:51.794227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435922150831426:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:51.793864Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435922576464439:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.793978Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435922576464451:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.794030Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435922576464452:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.794080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435922576464455:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23910
2025-11-19T13:59:51.794392Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435923636355994:2105] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:51.794486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435922576464369:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:51.794534Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435922576464443:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.794537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435922576464443:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.794947Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435922576464438:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:51.795562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435922576464370:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.795578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:51.795584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435922576464370:2049] [node 2] ICP22 created new session: [3:7574435922576464460:2049]
2025-11-19T13:59:51.795590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435922576464460:2049] [node 2] ICS09 handshake done sender: [3:7574435922576464438:2112] self: [3:7574435922576464454:0] peer: [2:7574435923636356004:0] socket: 52 qp: -1
2025-11-19T13:59:51.795595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435922576464460:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:51.795601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.795951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435922150831510:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14623
2025-11-19T13:59:51.796145Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435925674853724:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.796186Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435925674853711:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.796189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435925674853711:2106] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.797864Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435922150831506:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.797887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435922150831424:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.798415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435922576464462:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23910
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.798607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435922372782247:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63827
2025-11-19T13:59:51.798708Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435922576464443:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:51.798871Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435922150831500:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:51.798936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435922150831426:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.798952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:51.798958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435922150831426:2050] [node 3] ICP22 created new session: [5:7574435922150831512:2050]
2025-11-19T13:59:51.798963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435922150831512:2050] [node 3] ICS09 handshake done sender: [5:7574435922150831500:2116] self: [5:7574435922150831509:0] peer: [3:7574435922576464442:0] socket: 62 qp: -1
2025-11-19T13:59:51.798969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435922150831512:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:51.798977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.799954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435922372782248:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63827
2025-11-19T13:59:51.800243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435923636355936:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.800266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:51.800272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435923636355936:2048] [node 1] ICP22 created new session: [2:7574435923636356032:2048]
2025-11-19T13:59:51.800277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435923636356032:2048] [node 1] ICS09 handshake done sender: [2:7574435923636355994:2105] self: [2:7574435923636356020:0] peer: [1:7574435925674853702:0] socket: 37 qp: -1
2025-11-19T13:59:51.800283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435923636356032:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:51.800290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:51.800415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435923636355936:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.801308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435922372782249:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63827
2025-11-19T13:59:51.801572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435922576464435:2094] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.801591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:51.801596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435922576464435:2094] [node 5] ICP22 created new session: [3:7574435922576464464:2094]
2025-11-19T13:59:51.801601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435922576464464:2094] [node 5] ICS09 handshake done sender: [3:7574435922576464443:2115] self: [3:7574435922576464442:0] peer: [5:7574435922150831509:0] socket: 56 qp: -1
2025-11-19T13:59:51.801604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435922576464464:2094] [node 5] ICS10 traffic start
2025-11-19T13:59:51.801609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:51.801933Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435922150831505:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.802019Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435922372782248:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.802044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435922372782202:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:51.803496Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435925674853711:2106] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:51.803679Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435925674853715:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.803686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435925674853715:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.804253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435925674853648:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.804264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:51.804269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435925674853648:2049] [node 3] ICP22 created new session: [1:7574435925674853743:2049]
2025-11-19T13:59:51.804274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435925674853743:2049] [node 3] ICS09 handshake done sender: [1:7574435925674853711:2106] self: [1:7574435925674853709:0] peer: [3:7574435922576464457:0] socket: 59 qp: -1
2025-11-19T13:59:51.804279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435925674853743:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:51.804286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:51.804778Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435922372782246:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.804804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435922372782167:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:51.808627Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435922150831496:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.808639Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435922150831496:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.809218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435922150831425:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.809537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435922372782252:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63827
2025-11-19T13:59:51.809658Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435923636356009:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.809670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435923636356009:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.810347Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435922372782249:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.810370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435922372782168:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:51.810584Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435922372782247:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.810599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435922372782169:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:51.812123Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435922576464445:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.812138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435922576464445:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.812739Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435922576464451:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:51.815964Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435922442723156:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.816082Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435922442723150:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.816087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435922442723150:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.816673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435922442723079:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:51.816875Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435922372782245:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.816907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435922372782203:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:51.817059Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435922442723150:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:51.817163Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435922442723152:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.817169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435922442723152:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.817598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435922442723146:2080] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.817609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:51.817616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435922442723146:2080] [node 5] ICP22 created new session: [4:7574435922442723170:2080]
2025-11-19T13:59:51.817623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435922442723170:2080] [node 5] ICS09 handshake done sender: [4:7574435922442723150:2114] self: [4:7574435922442723149:0] peer: [5:7574435922150831508:0] socket: 48 qp: -1
2025-11-19T13:59:51.817631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435922442723170:2080] [node 5] ICS10 traffic start
2025-11-19T13:59:51.817640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.818138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435922372782256:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63827
2025-11-19T13:59:51.818232Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435922442723152:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:51.818419Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435922372782245:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:51.818480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435922372782203:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.818494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:51.818499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435922372782203:2068] [node 4] ICP22 created new session: [6:7574435922372782257:2068]
2025-11-19T13:59:51.818504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435922372782257:2068] [node 4] ICS09 handshake done sender: [6:7574435922372782245:2115] self: [6:7574435922372782255:0] peer: [4:7574435922442723151:0] socket: 57 qp: -1
2025-11-19T13:59:51.818510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435922372782257:2068] [node 4] ICS10 traffic start
2025-11-19T13:59:51.818516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:51.819953Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435922150831496:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:51.820282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435922150831472:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.820306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:51.820314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435922150831472:2079] [node 6] ICP22 created new session: [5:7574435922150831520:2079]
2025-11-19T13:59:51.820321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435922150831520:2079] [node 6] ICS09 handshake done sender: [5:7574435922150831496:2113] self: [5:7574435922150831495:0] peer: [6:7574435922372782250:0] socket: 64 qp: -1
2025-11-19T13:59:51.820326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435922150831520:2079] [node 6] ICS10 traffic start
2025-11-19T13:59:51.820333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.820272Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435925674853726:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.820286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435925674853726:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.820898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435925674853746:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:353
2025-11-19T13:59:51.821054Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435923636356009:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:51.821184Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435923636356013:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.821190Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435923636356013:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.821576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435923636356000:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.821590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:51.821604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435923636356000:2107] [node 5] ICP22 created new session: [2:7574435923636356035:2107]
2025-11-19T13:59:51.821609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435923636356035:2107] [node 5] ICS09 handshake done sender: [2:7574435923636356009:2114] self: [2:7574435923636356008:0] peer: [5:7574435922150831516:0] socket: 76 qp: -1
2025-11-19T13:59:51.821615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435923636356035:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:51.821623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:51.823553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435922442723147:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.823571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:51.823577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435922442723147:2104] [node 6] ICP22 created new session: [4:7574435922442723173:2104]
2025-11-19T13:59:51.823583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435922442723173:2104] [node 6] ICS09 handshake done sender: [4:7574435922442723152:2115] self: [4:7574435922442723151:0] peer: [6:7574435922372782255:0] socket: 81 qp: -1
2025-11-19T13:59:51.823589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435922442723173:2104] [node 6] ICS10 traffic start
2025-11-19T13:59:51.823596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:51.823937Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435922372782248:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:51.824407Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435925674853715:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:51.824549Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435925674853726:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:51.824650Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435925674853746:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.824661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435925674853708:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.824674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:51.824684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435925674853708:2104] [node 5] ICP22 created new session: [1:7574435925674853748:2104]
2025-11-19T13:59:51.824689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435925674853748:2104] [node 5] ICS09 handshake done sender: [1:7574435925674853715:2108] self: [1:7574435925674853714:0] peer: [5:7574435922150831511:0] socket: 77 qp: -1
2025-11-19T13:59:51.824694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435925674853748:2104] [node 5] ICS10 traffic start
2025-11-19T13:59:51.824699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:51.824868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435925674853719:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.824873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:51.824877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435925674853719:2105] [node 6] ICP22 created new session: [1:7574435925674853750:2105]
2025-11-19T13:59:51.824880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435925674853750:2105] [node 6] ICS09 handshake done sender: [1:7574435925674853726:2117] self: [1:7574435925674853725:0] peer: [6:7574435922372782251:0] socket: 82 qp: -1
2025-11-19T13:59:51.824883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435925674853750:2105] [node 6] ICS10 traffic start
2025-11-19T13:59:51.824888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:51.825290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435925674853719:2105] [node 6] ICP18 handshake (actor [1:7574435925674853746:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.829509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435922150831517:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14623
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.829696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435922150831518:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14623
2025-11-19T13:59:51.829796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435922372782202:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.829817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:51.829823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435922372782202:2063] [node 5] ICP22 created new session: [6:7574435922372782261:2063]
2025-11-19T13:59:51.829829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435922372782261:2063] [node 5] ICS09 handshake done sender: [6:7574435922372782248:2118] self: [6:7574435922372782250:0] peer: [5:7574435922150831495:0] socket: 91 qp: -1
2025-11-19T13:59:51.829835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435922372782261:2063] [node 5] ICS10 traffic start
2025-11-19T13:59:51.829843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:51.830220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435922372782167:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24833) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435922372782246:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.830984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435922150831519:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14623
2025-11-19T13:59:51.831138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435922150831424:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24833) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435922150831506:2118] held: no
2025-11-19T13:59:51.831563Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435922150831506:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.831915Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435922576464433:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:51.831927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435922576464433:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:51.832914Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435922150831505:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:51.833048Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435922150831507:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:51.833347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435922150831424:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.833362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:51.833368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435922150831424:2048] [node 1] ICP22 created new session: [5:7574435922150831523:2048]
2025-11-19T13:59:51.833375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435922150831523:2048] [node 1] ICS09 handshake done sender: [5:7574435922150831506:2118] self: [5:7574435922150831511:0] peer: [1:7574435925674853714:0] socket: 51 qp: -1
2025-11-19T13:59:51.833381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435922150831523:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:51.833389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:51.833552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435922150831425:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.833561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:51.833564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435922150831425:2049] [node 2] ICP22 created new session: [5:7574435922150831525:2049]
2025-11-19T13:59:51.833568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435922150831525:2049] [node 2] ICS09 handshake done sender: [5:7574435922150831505:2117] self: [5:7574435922150831516:0] peer: [2:7574435923636356008:0] socket: 49 qp: -1
2025-11-19T13:59:51.833572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435922150831525:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:51.833578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:51.834442Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435923636356013:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.834595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435922372782260:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63827
2025-11-19T13:59:51.834894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435923636356010:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.834914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:51.834919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435923636356010:2110] [node 6] ICP22 created new session: [2:7574435923636356038:2110]
2025-11-19T13:59:51.834924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435923636356038:2110] [node 6] ICS09 handshake done sender: [2:7574435923636356013:2116] self: [2:7574435923636356012:0] peer: [6:7574435922372782253:0] socket: 42 qp: -1
2025-11-19T13:59:51.834929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435923636356038:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:51.834937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:51.831695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435922442723175:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
2025-11-19T13:59:51.835441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435922372782202:2063] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:51.835781Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435922372782246:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:51.835923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435922576464369:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.835942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:51.835947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435922576464369:2048] [node 1] ICP22 created new session: [3:7574435922576464467:2048]
2025-11-19T13:59:51.835953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435922576464467:2048] [node 1] ICS09 handshake done sender: [3:7574435922576464451:2118] self: [3:7574435922576464457:0] peer: [1:7574435925674853709:0] socket: 72 qp: -1
2025-11-19T13:59:51.835958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435922576464467:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:51.835966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:51.836102Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435922576464369:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:51.836751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435922372782167:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.836763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:51.836770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435922372782167:2048] [node 1] ICP22 created new session: [6:7574435922372782265:2048]
2025-11-19T13:59:51.836776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435922372782265:2048] [node 1] ICS09 handshake done sender: [6:7574435922372782246:2116] self: [6:7574435922372782251:0] peer: [1:7574435925674853725:0] socket: 84 qp: -1
2025-11-19T13:59:51.836781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435922372782265:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:51.836788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:51.836969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435922372782263:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63827
2025-11-19T13:59:51.836981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435922372782264:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63827
2025-11-19T13:59:51.837262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435922150831473:2083] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.837277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:51.837283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435922150831473:2083] [node 4] ICP22 created new session: [5:7574435922150831527:2083]
2025-11-19T13:59:51.837288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435922150831527:2083] [node 4] ICS09 handshake done sender: [5:7574435922150831507:2119] self: [5:7574435922150831508:0] peer: [4:7574435922442723149:0] socket: 80 qp: -1
2025-11-19T13:59:51.837295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435922150831527:2083] [node 4] ICS10 traffic start
2025-11-19T13:59:51.837302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:51.836653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435922442723176:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
2025-11-19T13:59:51.841021Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435922576464445:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:51.843953Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435922576464433:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:51.844459Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435922442723156:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:51.844536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435922442723079:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.844554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:51.844560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435922442723079:2050] [node 3] ICP22 created new session: [4:7574435922442723177:2050]
2025-11-19T13:59:51.844565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435922442723177:2050] [node 3] ICS09 handshake done sender: [4:7574435922442723156:2116] self: [4:7574435922442723169:0] peer: [3:7574435922576464412:0] socket: 67 qp: -1
2025-11-19T13:59:51.844572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435922442723177:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:51.844582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:51.845727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435922150831425:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:51.846309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435922576464434:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.846329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:51.846336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435922576464434:2088] [node 6] ICP22 created new session: [3:7574435922576464470:2088]
2025-11-19T13:59:51.846342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435922576464470:2088] [node 6] ICS09 handshake done sender: [3:7574435922576464445:2116] self: [3:7574435922576464444:0] peer: [6:7574435922372782254:0] socket: 89 qp: -1
2025-11-19T13:59:51.846348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435922576464470:2088] [node 6] ICS10 traffic start
2025-11-19T13:59:51.846354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:51.846542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435922576464409:2086] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:51.846551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:51.846554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435922576464409:2086] [node 4] ICP22 created new session: [3:7574435922576464472:2086]
2025-11-19T13:59:51.846557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435922576464472:2086] [node 4] ICS09 handshake done sender: [3:7574435922576464433:2109] self: [3:7574435922576464412:0] peer: [4:7574435922442723169:0] socket: 54 qp: -1
2025-11-19T13:59:51.846561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435922576464472:2086] [node 4] ICS10 traffic start
2025-11-19T13:59:51.846566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:51.847428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435922372782168:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25912) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435922372782249:2119] held: no
2025-11-19T13:59:51.847704Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435922372782249:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:51.847845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435922372782168:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.847859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:51.847864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435922372782168:2049] [node 2] ICP22 created new session: [6:7574435922372782267:2049]
2025-11-19T13:59:51.847869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435922372782267:2049] [node 2] ICS09 handshake done sender: [6:7574435922372782249:2119] self: [6:7574435922372782253:0] peer: [2:7574435923636356012:0] socket: 92 qp: -1
2025-11-19T13:59:51.847875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435922372782267:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:51.847881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:51.848052Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435922372782247:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:51.849302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435922372782169:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:51.849317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:51.849323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435922372782169:2050] [node 3] ICP22 created new session: [6:7574435922372782269:2050]
2025-11-19T13:59:51.849327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435922372782269:2050] [node 3] ICS09 handshake done sender: [6:7574435922372782247:2117] self: [6:7574435922372782254:0] peer: [3:7574435922576464444:0] socket: 90 qp: -1
2025-11-19T13:59:51.849332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435922372782269:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:51.849339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:51.849472Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435922372782169:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:51.870921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:51.871024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435923636356032:2048] [node 1] ICS01 socket: 37 reason# EndOfStream
2025-11-19T13:59:51.871034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435923636355936:2048] [node 1] ICP30 unregister session Session# [2:7574435923636356032:2048] VirtualId# [2:7574435923636356020:0]
2025-11-19T13:59:51.871042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435923636356032:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:51.871110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:51.871134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:51.871242Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:51.871319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435922576464467:2048] [node 1] ICS07 socket disconnect 72 reason# EndOfStream
2025-11-19T13:59:51.871324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435922576464467:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:51.871353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:51.871372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435922576464467:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:51.871891Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435922576464369:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24833) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:51.871904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435922576464467:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:51.871908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435922576464369:2048] [node 1] ICP30 unregister session Session# [3:7574435922576464467:2048] VirtualId# [3:7574435922576464457:0]
2025-11-19T13:59:51.871939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:51.871945Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435922576464369:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24833) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:51.871948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24833) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:51.880059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:51.880148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435922442723164:2048] [node 1] ICS01 socket: 68 reason# EndOfStream
2025-11-19T13:59:51.880157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435922442723077:2048] [node 1] ICP30 unregister session Session# [4:7574435922442723164:2048] VirtualId# [4:7574435922442723159:0]
2025-11-19T13:59:51.880165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435922442723164:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:51.880206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:51.880229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:51.880265Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:51.880324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435922150831523:2048] [node 1] ICS01 socket: 51 reason# EndOfStream
2025-11-19T13:59:51.880328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435922150831424:2048] [node 1] ICP30 unregister session Session# [5:7574435922150831523:2048] VirtualId# [5:7574435922150831511:0]
2025-11-19T13:59:51.880332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435922150831523:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:51.880363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:51.880384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:51.880417Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:51.880472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435922372782265:2048] [node 1] ICS01 socket: 84 reason# EndOfStream
2025-11-19T13:59:51.880476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435922372782167:2048] [node 1] ICP30 unregister session Session# [6:7574435922372782265:2048] VirtualId# [6:7574435922372782251:0]
2025-11-19T13:59:51.880481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435922372782265:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:51.880501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:51.880520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:51.887845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574435922576464369:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:51.899871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:51.900018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435922576464460:2049] [node 2] ICS01 socket: 52 reason# EndOfStream
2025-11-19T13:59:51.900028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435922576464370:2049] [node 2] ICP30 unregister session Session# [3:7574435922576464460:2049] VirtualId# [3:7574435922576464454:0]
2025-11-19T13:59:51.900035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435922576464460:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:51.900088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:51.900108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:51.902820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:51.902917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435922442723162:2049] [node 2] ICS01 socket: 47 reason# EndOfStream
2025-11-19T13:59:51.902925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435922442723078:2049] [node 2] ICP30 unregister session Session# [4:7574435922442723162:2049] VirtualId# [4:7574435922442723158:0]
2025-11-19T13:59:51.902931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435922442723162:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:51.902980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:51.903001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:51.903035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:51.903142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435922150831525:2049] [node 2] ICS01 socket: 49 reason# EndOfStream
2025-11-19T13:59:51.903146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435922150831425:2049] [node 2] ICP30 unregister session Session# [5:7574435922150831525:2049] VirtualId# [5:7574435922150831516:0]
2025-11-19T13:59:51.903152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435922150831525:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:51.903177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:51.903197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:51.903224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:51.903288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435922372782267:2049] [node 2] ICS01 socket: 92 reason# EndOfStream
2025-11-19T13:59:51.903293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435922372782168:2049] [node 2] ICP30 unregister session Session# [6:7574435922372782267:2049] VirtualId# [6:7574435922372782253:0]
2025-11-19T13:59:51.903296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435922372782267:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:51.903322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:51.903341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:51.917118Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:51.917220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435922372782269:2050] [node 3] ICS01 socket: 90 reason# EndOfStream
2025-11-19T13:59:51.917230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435922372782169:2050] [node 3] ICP30 unregister session Session# [6:7574435922372782269:2050] VirtualId# [6:7574435922372782254:0]
2025-11-19T13:59:51.917238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435922372782269:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:51.917299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:51.917323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:51.917364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:51.917476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435922150831512:2050] [node 3] ICS01 socket: 62 reason# EndOfStream
2025-11-19T13:59:51.917480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435922150831426:2050] [node 3] ICP30 unregister session Session# [5:7574435922150831512:2050] VirtualId# [5:7574435922150831509:0]
2025-11-19T13:59:51.917483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435922150831512:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:51.917511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:51.917531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:51.927943Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:51.928044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435922372782257:2068] [node 4] ICS01 socket: 57 reason# EndOfStream
2025-11-19T13:59:51.928054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435922372782203:2068] [node 4] ICP30 unregister session Session# [6:7574435922372782257:2068] VirtualId# [6:7574435922372782255:0]
2025-11-19T13:59:51.928061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435922372782257:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:51.928132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:51.928159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T13:59:52.144257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435928716866824:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:52.144272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435928716866825:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:52.178589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435926850816160:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:52.178614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435926850816161:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:52.189258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435927267125564:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:52.189253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435927267125565:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:52.215532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435926382456149:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:52.215545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435926382456150:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:52.215550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435926382456151:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:52.220138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435926185846840:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:52.220149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435926185846841:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:52.220154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435926185846842:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:52.256737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435927322551738:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:52.256786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435927322551740:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:52.256956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435927322551739:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:52.268319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435928716866881:2101] [node 6] ICP01 ready to work
2025-11-19T13:59:52.268432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435928716866882:2102] [node 4] ICP01 ready to work
2025-11-19T13:59:52.268554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435928716866883:2103] [node 5] ICP01 ready to work
2025-11-19T13:59:52.269015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435926850816204:2090] [node 5] ICP01 ready to work
2025-11-19T13:59:52.268846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435927322551759:2062] [node 4] ICP01 ready to work
2025-11-19T13:59:52.268983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435927322551760:2067] [node 5] ICP01 ready to work
2025-11-19T13:59:52.270750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435927267125607:2086] [node 4] ICP01 ready to work
2025-11-19T13:59:52.278785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435927267125613:2089] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.280029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435926185846859:2061] [node 6] ICP01 ready to work
2025-11-19T13:59:52.280117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435926185846861:2063] [node 4] ICP01 ready to work
2025-11-19T13:59:52.280990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435926185846858:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9780
2025-11-19T13:59:52.281978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435927267125619:2093] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.294659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435928716866882:2102] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24464432ns) {Configure:104(24460068ns) {ProcessConfigured:135(24159015ns) {StartInitialHandshake:157(24158756ns) {DropHandshakes:469(435ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(187ns) PrepareNewSessionHandshake:476(600ns) {SwitchToState:178(280ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.292228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435926382456207:2081] [node 5] ICP01 ready to work
2025-11-19T13:59:52.294426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435926382456208:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43814
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.298072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435927267125641:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46331
2025-11-19T13:59:52.296140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435928716866903:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55879
2025-11-19T13:59:52.296160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435928716866904:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55879
2025-11-19T13:59:52.296202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435928716866905:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55879
2025-11-19T13:59:52.296501Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435926850816204:2090] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(27343572ns) {Configure:104(26216633ns) {ProcessConfigured:135(25941888ns) {StartInitialHandshake:157(25941077ns) {DropHandshakes:469(465ns) {DropIncomingHandshake:441(77ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:476(829ns) {SwitchToState:178(45ns)}}}}}
2025-11-19T13:59:52.296726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435926850816220:2093] [node 6] ICP01 ready to work
2025-11-19T13:59:52.296842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435926850816221:2101] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.304123Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435927322551760:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(32280263ns) {Configure:104(32278526ns) {ProcessConfigured:135(31951901ns) {StartInitialHandshake:157(31951646ns) {DropHandshakes:469(579ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(76ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:476(699ns) {SwitchToState:178(376ns)}}}}}
2025-11-19T13:59:52.305256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927322551803:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45841
2025-11-19T13:59:52.305322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927322551804:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45841
2025-11-19T13:59:52.305332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927322551805:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45841
2025-11-19T13:59:52.303860Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435927267125607:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(33035081ns) {Configure:104(33034627ns) {ProcessConfigured:135(32732352ns) {StartInitialHandshake:157(32731587ns) {DropHandshakes:469(482ns) {DropIncomingHandshake:441(186ns) DropOutgoingHandshake:456(46ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:476(971ns) {SwitchToState:178(751ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.316711Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435928716866904:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.316365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435926185846916:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9780
2025-11-19T13:59:52.316388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435926185846917:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9780
2025-11-19T13:59:52.316399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435926185846918:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9780
2025-11-19T13:59:52.318951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435926850816237:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54644
2025-11-19T13:59:52.319008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435926850816238:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54644
2025-11-19T13:59:52.319041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435926850816239:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54644
2025-11-19T13:59:52.319077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435926850816240:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54644
2025-11-19T13:59:52.319777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435928716866907:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55879
2025-11-19T13:59:52.319806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435928716866908:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55879
2025-11-19T13:59:52.322285Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435926850816237:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.322325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435926850816239:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.322369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435926850816160:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.323174Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435926850816238:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.323297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435926850816243:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54644
2025-11-19T13:59:52.323426Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435927267125641:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.323466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435927267125565:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.321935Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435928716866905:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.322511Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435928716866888:2105] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.322516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435928716866888:2105] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.324087Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435928716866888:2105] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:52.323621Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435926850816223:2104] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.323628Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435926850816223:2104] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.324977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435928716866824:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.324997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:52.325007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435928716866824:2048] [node 2] ICP22 created new session: [1:7574435928716866911:2048]
2025-11-19T13:59:52.325012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435928716866911:2048] [node 2] ICS09 handshake done sender: [1:7574435928716866888:2105] self: [1:7574435928716866887:0] peer: [2:7574435926850816242:0] socket: 45 qp: -1
2025-11-19T13:59:52.325020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435928716866911:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:52.325028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:52.327999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435926382456197:2079] [node 6] ICP01 ready to work
2025-11-19T13:59:52.329609Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435926850816239:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:52.328967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435927322551759:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(59936351ns) {Configure:104(59935500ns) {ProcessConfigured:135(59435868ns) {StartInitialHandshake:157(59435507ns) {DropHandshakes:469(528ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(76ns)} GenerateSessionVirtualId:988(51ns) PrepareNewSessionHandshake:476(1905ns) {SwitchToState:178(847ns)}}}}}
2025-11-19T13:59:52.328326Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435926185846858:2064] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.329024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435927322551805:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.329098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435927322551803:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.329165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435927322551740:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.329174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435927322551738:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.328442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435926185846840:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.329835Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435926185846917:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.329861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435926185846841:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.333296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435926382456208:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.333599Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435926185846916:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.333628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435926185846861:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.333782Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435926382456212:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.333788Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435926382456212:2104] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.334578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435926185846924:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9780
2025-11-19T13:59:52.334676Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435926382456212:2104] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:52.334856Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435926185846916:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:52.334792Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435927267125615:2091] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.334807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435927267125615:2091] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.335542Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435926185846842:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(59159076ns) {Configure:104(59157261ns) {ProcessConfigured:135(58720871ns) {StartInitialHandshake:157(58720014ns) {DropHandshakes:469(991ns) {DropIncomingHandshake:441(128ns) DropOutgoingHandshake:456(79ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:476(812ns) {SwitchToState:178(442ns)}}}}}
2025-11-19T13:59:52.335559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435926185846861:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.335573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:52.335580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435926185846861:2063] [node 4] ICP22 created new session: [5:7574435926185846926:2063]
2025-11-19T13:59:52.335585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435926185846926:2063] [node 4] ICS09 handshake done sender: [5:7574435926185846916:2114] self: [5:7574435926185846923:0] peer: [4:7574435926382456211:0] socket: 71 qp: -1
2025-11-19T13:59:52.335592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435926185846926:2063] [node 4] ICS10 traffic start
2025-11-19T13:59:52.335611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:52.338597Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435927322551804:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.336681Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435928716866903:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.336713Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435928716866890:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.336716Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435928716866890:2106] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.337205Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435928716866886:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.337207Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435928716866886:2104] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.337604Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435928716866907:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.339198Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435928716866890:2106] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:52.339336Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435928716866886:2104] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:52.338834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435926382456207:2081] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.338853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:52.338859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435926382456207:2081] [node 5] ICP22 created new session: [4:7574435926382456230:2081]
2025-11-19T13:59:52.338866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435926382456230:2081] [node 5] ICS09 handshake done sender: [4:7574435926382456212:2104] self: [4:7574435926382456211:0] peer: [5:7574435926185846923:0] socket: 56 qp: -1
2025-11-19T13:59:52.338872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435926382456230:2081] [node 5] ICS10 traffic start
2025-11-19T13:59:52.338891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:52.339489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435926850816160:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.339509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:52.339515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435926850816160:2048] [node 1] ICP22 created new session: [2:7574435926850816248:2048]
2025-11-19T13:59:52.339521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435926850816248:2048] [node 1] ICS09 handshake done sender: [2:7574435926850816239:2115] self: [2:7574435926850816242:0] peer: [1:7574435928716866887:0] socket: 75 qp: -1
2025-11-19T13:59:52.339527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435926850816248:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:52.339556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:52.339814Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435926850816219:2103] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.339818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435926850816219:2103] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.339499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435928716866883:2103] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.339518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:52.339523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435928716866883:2103] [node 5] ICP22 created new session: [1:7574435928716866917:2103]
2025-11-19T13:59:52.339528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435928716866917:2103] [node 5] ICS09 handshake done sender: [1:7574435928716866890:2106] self: [1:7574435928716866889:0] peer: [5:7574435926185846921:0] socket: 46 qp: -1
2025-11-19T13:59:52.339531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435928716866917:2103] [node 5] ICS10 traffic start
2025-11-19T13:59:52.339537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:52.339737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435928716866824:2048] [node 2] ICP18 handshake (actor [1:7574435928716866907:2118]) is too old
2025-11-19T13:59:52.340063Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435927267125615:2091] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.345142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435927322551760:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.345176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927322551821:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45841
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.347200Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435926185846840:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18394) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435926185846858:2064] held: no
2025-11-19T13:59:52.347509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435926185846928:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9780
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.344985Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435926850816223:2104] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:52.347814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435926185846929:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9780
2025-11-19T13:59:52.345239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435927267125613:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.345259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:52.345265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435927267125613:2089] [node 6] ICP22 created new session: [3:7574435927267125646:2089]
2025-11-19T13:59:52.345272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435927267125646:2089] [node 6] ICS09 handshake done sender: [3:7574435927267125615:2091] self: [3:7574435927267125614:0] peer: [6:7574435927322551819:0] socket: 53 qp: -1
2025-11-19T13:59:52.345278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435927267125646:2089] [node 6] ICS10 traffic start
2025-11-19T13:59:52.345286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:52.345734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435927267125649:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46331
2025-11-19T13:59:52.345746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435927267125650:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46331
2025-11-19T13:59:52.346790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435927267125651:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46331
2025-11-19T13:59:52.350838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435928716866881:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.350861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:52.350869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435928716866881:2101] [node 6] ICP22 created new session: [1:7574435928716866919:2101]
2025-11-19T13:59:52.350876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435928716866919:2101] [node 6] ICS09 handshake done sender: [1:7574435928716866886:2104] self: [1:7574435928716866884:0] peer: [6:7574435927322551820:0] socket: 44 qp: -1
2025-11-19T13:59:52.350883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435928716866919:2101] [node 6] ICS10 traffic start
2025-11-19T13:59:52.350892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.351614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435926850816251:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54644
2025-11-19T13:59:52.351974Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435926850816219:2103] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:52.352645Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435927267125641:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:52.356181Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435926185846858:2064] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:52.356332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927322551822:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45841
2025-11-19T13:59:52.355976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435927267125565:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.356018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:52.356031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435927267125565:2049] [node 2] ICP22 created new session: [3:7574435927267125652:2049]
2025-11-19T13:59:52.356038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435927267125652:2049] [node 2] ICS09 handshake done sender: [3:7574435927267125641:2113] self: [3:7574435927267125644:0] peer: [2:7574435926850816222:0] socket: 63 qp: -1
2025-11-19T13:59:52.356048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435927267125652:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:52.356062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.356496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435926185846930:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9780
2025-11-19T13:59:52.356608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435926850816161:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.356633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:52.356640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435926850816161:2049] [node 3] ICP22 created new session: [2:7574435926850816253:2049]
2025-11-19T13:59:52.356646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435926850816253:2049] [node 3] ICS09 handshake done sender: [2:7574435926850816223:2104] self: [2:7574435926850816222:0] peer: [3:7574435927267125644:0] socket: 62 qp: -1
2025-11-19T13:59:52.356650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435926850816253:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:52.356657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:52.356520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435926382456233:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43814
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.359007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435926382456234:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43814
2025-11-19T13:59:52.356985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435926850816204:2090] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.356997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:52.357001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435926850816204:2090] [node 5] ICP22 created new session: [2:7574435926850816255:2090]
2025-11-19T13:59:52.357005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435926850816255:2090] [node 5] ICS09 handshake done sender: [2:7574435926850816219:2103] self: [2:7574435926850816206:0] peer: [5:7574435926185846922:0] socket: 61 qp: -1
2025-11-19T13:59:52.357008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435926850816255:2090] [node 5] ICS10 traffic start
2025-11-19T13:59:52.357013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:52.359163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435926185846840:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.359182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:52.359190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435926185846840:2048] [node 1] ICP22 created new session: [5:7574435926185846933:2048]
2025-11-19T13:59:52.359196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435926185846933:2048] [node 1] ICS09 handshake done sender: [5:7574435926185846858:2064] self: [5:7574435926185846921:0] peer: [1:7574435928716866889:0] socket: 48 qp: -1
2025-11-19T13:59:52.359202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435926185846933:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:52.359209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:52.360524Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435926185846917:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.362605Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435927267125649:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.363573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435927267125654:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46331
2025-11-19T13:59:52.363594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435927267125564:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.364356Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435927322551805:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.364959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435926382456235:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43814
2025-11-19T13:59:52.361950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435926185846841:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.361965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:52.361970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435926185846841:2049] [node 2] ICP22 created new session: [5:7574435926185846935:2049]
2025-11-19T13:59:52.361976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435926185846935:2049] [node 2] ICS09 handshake done sender: [5:7574435926185846917:2115] self: [5:7574435926185846922:0] peer: [2:7574435926850816206:0] socket: 72 qp: -1
2025-11-19T13:59:52.361981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435926185846935:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:52.361990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:52.365239Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435926850816251:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.365453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435926850816204:2090] [node 5] ICP18 handshake (actor [2:7574435926850816251:2120]) is too old
2025-11-19T13:59:52.365656Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435927322551803:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:52.365729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435926382456234:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.365753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435926382456150:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.365854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435927322551740:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.365867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:52.365872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435927322551740:2050] [node 3] ICP22 created new session: [6:7574435927322551825:2050]
2025-11-19T13:59:52.365877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435927322551825:2050] [node 3] ICS09 handshake done sender: [6:7574435927322551805:2110] self: [6:7574435927322551819:0] peer: [3:7574435927267125614:0] socket: 67 qp: -1
2025-11-19T13:59:52.365883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435927322551825:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:52.365901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:52.366188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435927322551738:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.366195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:52.366199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435927322551738:2048] [node 1] ICP22 created new session: [6:7574435927322551827:2048]
2025-11-19T13:59:52.366202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435927322551827:2048] [node 1] ICS09 handshake done sender: [6:7574435927322551803:2108] self: [6:7574435927322551820:0] peer: [1:7574435928716866884:0] socket: 65 qp: -1
2025-11-19T13:59:52.366206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435927322551827:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:52.366211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:52.366569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927322551824:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45841
2025-11-19T13:59:52.366846Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435926850816227:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.366855Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435926850816227:2106] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.367460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435926382456237:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43814
2025-11-19T13:59:52.367907Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435928716866892:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.367923Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435928716866892:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.368132Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435927267125651:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.367896Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435926850816227:2106] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:52.368717Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435926382456233:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.368736Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435926382456234:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:52.368799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435926382456150:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.368813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:52.368818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435926382456150:2049] [node 2] ICP22 created new session: [4:7574435926382456238:2049]
2025-11-19T13:59:52.368823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435926382456238:2049] [node 2] ICS09 handshake done sender: [4:7574435926382456234:2117] self: [4:7574435926382456236:0] peer: [2:7574435926850816226:0] socket: 99 qp: -1
2025-11-19T13:59:52.368828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435926382456238:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:52.368834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:52.369029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435926382456149:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.369848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435926850816221:2101] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.369866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:52.369872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435926850816221:2101] [node 4] ICP22 created new session: [2:7574435926850816259:2101]
2025-11-19T13:59:52.369876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435926850816259:2101] [node 4] ICS09 handshake done sender: [2:7574435926850816227:2106] self: [2:7574435926850816226:0] peer: [4:7574435926382456236:0] socket: 98 qp: -1
2025-11-19T13:59:52.369881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435926850816259:2101] [node 4] ICS10 traffic start
2025-11-19T13:59:52.369886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:52.370690Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435928716866892:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:52.370803Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435928716866902:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.370810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435928716866902:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.371262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435928716866825:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.371274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:52.371279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435928716866825:2049] [node 3] ICP22 created new session: [1:7574435928716866923:2049]
2025-11-19T13:59:52.371284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435928716866923:2049] [node 3] ICS09 handshake done sender: [1:7574435928716866892:2107] self: [1:7574435928716866891:0] peer: [3:7574435927267125655:0] socket: 85 qp: -1
2025-11-19T13:59:52.371289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435928716866923:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:52.371295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.373448Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435927322551824:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.373639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435927322551739:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.373659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927322551829:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45841
2025-11-19T13:59:52.373826Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435926850816225:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.373833Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435926850816225:2105] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.374447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435926382456242:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43814
2025-11-19T13:59:52.374543Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435926850816225:2105] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:52.374600Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435926185846930:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.374630Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435926185846865:2067] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.374633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435926185846865:2067] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.375497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435926185846842:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.375678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435926850816220:2093] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.375694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:52.375700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435926850816220:2093] [node 6] ICP22 created new session: [2:7574435926850816262:2093]
2025-11-19T13:59:52.375707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435926850816262:2093] [node 6] ICS09 handshake done sender: [2:7574435926850816225:2105] self: [2:7574435926850816224:0] peer: [6:7574435927322551831:0] socket: 64 qp: -1
2025-11-19T13:59:52.375713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435926850816262:2093] [node 6] ICS10 traffic start
2025-11-19T13:59:52.375721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.376733Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435927267125654:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.376770Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435927267125622:2094] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.376772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435927267125622:2094] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.378539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435927267125657:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46331
2025-11-19T13:59:52.379066Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435927267125622:2094] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:52.382191Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435926185846865:2067] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:52.382192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435927267125619:2093] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.382215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:52.382222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435927267125619:2093] [node 5] ICP22 created new session: [3:7574435927267125661:2093]
2025-11-19T13:59:52.382228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435927267125661:2093] [node 5] ICS09 handshake done sender: [3:7574435927267125622:2094] self: [3:7574435927267125621:0] peer: [5:7574435926185846938:0] socket: 90 qp: -1
2025-11-19T13:59:52.382234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435927267125661:2093] [node 5] ICS10 traffic start
2025-11-19T13:59:52.382244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:52.383053Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435928716866902:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:52.383262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435928716866882:2102] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.383277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:52.383282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435928716866882:2102] [node 4] ICP22 created new session: [1:7574435928716866926:2102]
2025-11-19T13:59:52.383287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435928716866926:2102] [node 4] ICS09 handshake done sender: [1:7574435928716866902:2114] self: [1:7574435928716866885:0] peer: [4:7574435926382456240:0] socket: 95 qp: -1
2025-11-19T13:59:52.383291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435928716866926:2102] [node 4] ICS10 traffic start
2025-11-19T13:59:52.383297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:52.386041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435927267125564:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18394) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435927267125649:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.386373Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435927322551829:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.386517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927322551832:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45841
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.386651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435926185846859:2061] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.386669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:52.386675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435926185846859:2061] [node 6] ICP22 created new session: [5:7574435926185846940:2061]
2025-11-19T13:59:52.386682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435926185846940:2061] [node 6] ICS09 handshake done sender: [5:7574435926185846865:2067] self: [5:7574435926185846864:0] peer: [6:7574435927322551823:0] socket: 50 qp: -1
2025-11-19T13:59:52.386688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435926185846940:2061] [node 6] ICS10 traffic start
2025-11-19T13:59:52.386696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:52.386875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435926185846842:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64436) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435926185846930:2122] held: no
2025-11-19T13:59:52.387027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435926185846939:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9780
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.387204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435926382456149:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18394) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435926382456233:2116] held: no
2025-11-19T13:59:52.387325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435926382456235:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.387425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435926382456243:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43814
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.387644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927322551833:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45841
2025-11-19T13:59:52.391958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435927322551759:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.394905Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435927322551824:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:52.395505Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435926382456229:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.395517Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435926382456229:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.395901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435926382456151:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.396230Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435927267125642:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.396241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435927267125642:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.398431Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435927322551804:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:52.398544Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435926850816240:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.398580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435926850816220:2093] [node 6] ICP18 handshake (actor [2:7574435926850816240:2116]) is too old
2025-11-19T13:59:52.398958Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435926382456229:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:52.399190Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435926185846930:2122] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:52.399386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435926382456197:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.399401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:52.399407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435926382456197:2079] [node 6] ICP22 created new session: [4:7574435926382456247:2079]
2025-11-19T13:59:52.399413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435926382456247:2079] [node 6] ICS09 handshake done sender: [4:7574435926382456229:2115] self: [4:7574435926382456226:0] peer: [6:7574435927322551834:0] socket: 104 qp: -1
2025-11-19T13:59:52.399418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435926382456247:2079] [node 6] ICS10 traffic start
2025-11-19T13:59:52.399428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:52.399635Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435926382456233:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:52.399855Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435927267125642:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:52.399958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435926185846842:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.399972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:52.399978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435926185846842:2050] [node 3] ICP22 created new session: [5:7574435926185846942:2050]
2025-11-19T13:59:52.399983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435926185846942:2050] [node 3] ICS09 handshake done sender: [5:7574435926185846930:2122] self: [5:7574435926185846938:0] peer: [3:7574435927267125621:0] socket: 93 qp: -1
2025-11-19T13:59:52.399988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435926185846942:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:52.399995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:52.400475Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435927267125649:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:52.400541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435927267125564:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.400555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:52.400560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435927267125564:2048] [node 1] ICP22 created new session: [3:7574435927267125664:2048]
2025-11-19T13:59:52.400565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435927267125664:2048] [node 1] ICS09 handshake done sender: [3:7574435927267125649:2116] self: [3:7574435927267125655:0] peer: [1:7574435928716866891:0] socket: 88 qp: -1
2025-11-19T13:59:52.400569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435927267125664:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:52.400576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.401042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435926382456246:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43814
2025-11-19T13:59:52.401129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435927267125607:2086] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.401144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:52.401150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435927267125607:2086] [node 4] ICP22 created new session: [3:7574435927267125666:2086]
2025-11-19T13:59:52.401155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435927267125666:2086] [node 4] ICS09 handshake done sender: [3:7574435927267125642:2114] self: [3:7574435927267125608:0] peer: [4:7574435926382456244:0] socket: 101 qp: -1
2025-11-19T13:59:52.401159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435927267125666:2086] [node 4] ICS10 traffic start
2025-11-19T13:59:52.401164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:52.401548Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435926382456235:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:52.401611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435926382456149:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.401622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:52.401626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435926382456149:2048] [node 1] ICP22 created new session: [4:7574435926382456249:2048]
2025-11-19T13:59:52.401630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435926382456249:2048] [node 1] ICS09 handshake done sender: [4:7574435926382456233:2116] self: [4:7574435926382456240:0] peer: [1:7574435928716866885:0] socket: 96 qp: -1
2025-11-19T13:59:52.401635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435926382456249:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:52.401641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:52.402044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435926382456151:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.402056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:52.402061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435926382456151:2050] [node 3] ICP22 created new session: [4:7574435926382456251:2050]
2025-11-19T13:59:52.402065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435926382456251:2050] [node 3] ICS09 handshake done sender: [4:7574435926382456235:2118] self: [4:7574435926382456244:0] peer: [3:7574435927267125608:0] socket: 103 qp: -1
2025-11-19T13:59:52.402069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435926382456251:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:52.402074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.404340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435927322551760:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.404358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:52.404364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435927322551760:2067] [node 5] ICP22 created new session: [6:7574435927322551836:2067]
2025-11-19T13:59:52.404370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435927322551836:2067] [node 5] ICS09 handshake done sender: [6:7574435927322551804:2109] self: [6:7574435927322551823:0] peer: [5:7574435926185846864:0] socket: 66 qp: -1
2025-11-19T13:59:52.404375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435927322551836:2067] [node 5] ICS10 traffic start
2025-11-19T13:59:52.404381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:52.404693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927322551835:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45841
2025-11-19T13:59:52.405478Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435926185846918:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.405718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435926185846859:2061] [node 6] ICP18 handshake (actor [5:7574435926185846918:2116]) is too old
2025-11-19T13:59:52.406039Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435926382456242:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.406064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435926382456197:2079] [node 6] ICP18 handshake (actor [4:7574435926382456242:2120]) is too old
2025-11-19T13:59:52.406381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435927322551739:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29909) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435927322551824:2120] held: no
2025-11-19T13:59:52.406463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435927322551739:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.406472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:52.406476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435927322551739:2049] [node 2] ICP22 created new session: [6:7574435927322551838:2049]
2025-11-19T13:59:52.406481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435927322551838:2049] [node 2] ICS09 handshake done sender: [6:7574435927322551824:2120] self: [6:7574435927322551831:0] peer: [2:7574435926850816224:0] socket: 102 qp: -1
2025-11-19T13:59:52.406487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435927322551838:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:52.406493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:52.408508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435927322551759:2062] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9899) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435927322551829:2121] held: no
2025-11-19T13:59:52.408802Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435927322551829:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:52.409028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435927322551759:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.409039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:52.409044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435927322551759:2062] [node 4] ICP22 created new session: [6:7574435927322551840:2062]
2025-11-19T13:59:52.409049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435927322551840:2062] [node 4] ICS09 handshake done sender: [6:7574435927322551829:2121] self: [6:7574435927322551834:0] peer: [4:7574435926382456226:0] socket: 42 qp: -1
2025-11-19T13:59:52.409054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435927322551840:2062] [node 4] ICS10 traffic start
2025-11-19T13:59:52.409061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:52.416562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:52.416653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435926850816248:2048] [node 1] ICS01 socket: 75 reason# EndOfStream
2025-11-19T13:59:52.416661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435926850816160:2048] [node 1] ICP30 unregister session Session# [2:7574435926850816248:2048] VirtualId# [2:7574435926850816242:0]
2025-11-19T13:59:52.416668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435926850816248:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:52.416702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:52.416720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:52.417155Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:52.417220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435927267125664:2048] [node 1] ICS01 socket: 88 reason# EndOfStream
2025-11-19T13:59:52.417224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435927267125564:2048] [node 1] ICP30 unregister session Session# [3:7574435927267125664:2048] VirtualId# [3:7574435927267125655:0]
2025-11-19T13:59:52.417229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435927267125664:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:52.417257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:52.417274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:52.420846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:52.420936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435926185846933:2048] [node 1] ICS01 socket: 48 reason# EndOfStream
2025-11-19T13:59:52.420943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435926185846840:2048] [node 1] ICP30 unregister session Session# [5:7574435926185846933:2048] VirtualId# [5:7574435926185846921:0]
2025-11-19T13:59:52.420949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435926185846933:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:52.420997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:52.421020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:52.420718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:52.420826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435926382456249:2048] [node 1] ICS01 socket: 96 reason# EndOfStream
2025-11-19T13:59:52.420838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435926382456149:2048] [node 1] ICP30 unregister session Session# [4:7574435926382456249:2048] VirtualId# [4:7574435926382456240:0]
2025-11-19T13:59:52.420847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435926382456249:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:52.420912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:52.420942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:52.421462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:52.421550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435927322551827:2048] [node 1] ICS01 socket: 65 reason# EndOfStream
2025-11-19T13:59:52.421557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435927322551738:2048] [node 1] ICP30 unregister session Session# [6:7574435927322551827:2048] VirtualId# [6:7574435927322551820:0]
2025-11-19T13:59:52.421564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435927322551827:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:52.421620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:52.421651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:52.425315Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:52.425401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435927267125652:2049] [node 2] ICS01 socket: 63 reason# EndOfStream
2025-11-19T13:59:52.425409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435927267125565:2049] [node 2] ICP30 unregister session Session# [3:7574435927267125652:2049] VirtualId# [3:7574435927267125644:0]
2025-11-19T13:59:52.425414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435927267125652:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:52.425454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:52.425475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:52.444541Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:52.444691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435927322551838:2049] [node 2] ICS01 socket: 102 reason# EndOfStream
2025-11-19T13:59:52.444705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435927322551739:2049] [node 2] ICP30 unregister session Session# [6:7574435927322551838:2049] VirtualId# [6:7574435927322551831:0]
2025-11-19T13:59:52.444714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435927322551838:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:52.444774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:52.444812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:52.444865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:52.444959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435926382456238:2049] [node 2] ICS01 socket: 99 reason# EndOfStream
2025-11-19T13:59:52.444967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435926382456150:2049] [node 2] ICP30 unregister session Session# [4:7574435926382456238:2049] VirtualId# [4:7574435926382456236:0]
2025-11-19T13:59:52.444973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435926382456238:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:52.445012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:52.445039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:52.446385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:52.446534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435926185846935:2049] [node 2] ICS01 socket: 72 reason# EndOfStream
2025-11-19T13:59:52.446542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435926185846841:2049] [node 2] ICP30 unregister session Session# [5:7574435926185846935:2049] VirtualId# [5:7574435926185846922:0]
2025-11-19T13:59:52.446548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435926185846935:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:52.446594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:52.446617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:52.477750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:52.477856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435926185846942:2050] [node 3] ICS01 socket: 93 reason# EndOfStream
2025-11-19T13:59:52.477864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435926185846842:2050] [node 3] ICP30 unregister session Session# [5:7574435926185846942:2050] VirtualId# [5:7574435926185846938:0]
2025-11-19T13:59:52.477871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435926185846942:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:52.477926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:52.477950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:52.478127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:52.478188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435927322551825:2050] [node 3] ICS01 socket: 67 reason# EndOfStream
2025-11-19T13:59:52.478192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435927322551740:2050] [node 3] ICP30 unregister session Session# [6:7574435927322551825:2050] VirtualId# [6:7574435927322551819:0]
2025-11-19T13:59:52.478196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435927322551825:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:52.478224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:52.478245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:52.486759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:52.486848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435927322551840:2062] [node 4] ICS01 socket: 42 reason# EndOfStream
2025-11-19T13:59:52.486856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435927322551759:2062] [node 4] ICP30 unregister session Session# [6:7574435927322551840:2062] VirtualId# [6:7574435927322551834:0]
2025-11-19T13:59:52.486863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435927322551840:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:52.486905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:52.486926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T13:59:52.609793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435930361924093:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:52.609816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435930361924094:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:52.639916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435927238969414:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:52.639932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435927238969415:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:52.695867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435930110577011:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:52.695890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435930110577012:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:52.720272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435927648109694:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:52.720287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435927648109695:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:52.720291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435927648109696:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:52.767876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435928154391538:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:52.767892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435928154391539:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:52.767899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435928154391540:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:52.840079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435927125525233:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:52.840184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435927125525234:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:52.840220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435927125525235:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.847646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435928154391573:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37468
2025-11-19T13:59:52.847373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435930110577062:2094] [node 5] ICP01 ready to work
2025-11-19T13:59:52.847492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435930110577063:2096] [node 6] ICP01 ready to work
2025-11-19T13:59:52.846698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435930361924154:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:52.848073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435928154391580:2084] [node 4] ICP01 ready to work
2025-11-19T13:59:52.848181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435928154391583:2090] [node 6] ICP01 ready to work
2025-11-19T13:59:52.848844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435930361924159:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.852078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435927648109755:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36433
2025-11-19T13:59:52.853062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435930110577078:2105] [node 4] ICP01 ready to work
2025-11-19T13:59:52.851922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435927125525257:2061] [node 5] ICP01 ready to work
2025-11-19T13:59:52.852067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435927125525258:2068] [node 4] ICP01 ready to work
2025-11-19T13:59:52.852007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435928154391592:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37468
2025-11-19T13:59:52.852027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435927238969475:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:52.854972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435930361924167:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.865823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927125525297:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22290
2025-11-19T13:59:52.865884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927125525298:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22290
2025-11-19T13:59:52.865894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927125525299:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22290
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.866288Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435928154391592:2095] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.867156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435930110577089:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55087
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.869032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435927648109763:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:52.869200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435927648109764:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:52.872922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435930361924168:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59915
2025-11-19T13:59:52.872944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435930361924169:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59915
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.874271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435927238969475:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(22181645ns) {Configure:104(22181003ns) {ProcessConfigured:135(21771301ns) {StartInitialHandshake:157(21770882ns) {DropHandshakes:469(446ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(150ns) PrepareNewSessionHandshake:476(730ns) {SwitchToState:178(560ns)}}}}}
2025-11-19T13:59:52.874455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435927238969481:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:52.875538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435928154391540:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19543954ns) {Configure:104(19543298ns) {ProcessConfigured:135(19246114ns) {StartInitialHandshake:157(19245847ns) {DropHandshakes:469(693ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(159ns)} GenerateSessionVirtualId:988(296ns) PrepareNewSessionHandshake:476(587ns) {SwitchToState:178(34ns)}}}}}
2025-11-19T13:59:52.875685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435928154391540:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.875784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435928154391614:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37468
2025-11-19T13:59:52.876462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435928154391612:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37468
2025-11-19T13:59:52.880912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435927648109696:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21041320ns) {Configure:104(21040522ns) {ProcessConfigured:135(20618000ns) {StartInitialHandshake:157(20617011ns) {DropHandshakes:469(923ns) {DropIncomingHandshake:441(129ns) DropOutgoingHandshake:456(143ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:476(1110ns) {SwitchToState:178(609ns)}}}}}
2025-11-19T13:59:52.880927Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435927648109755:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.881853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435927238969478:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:52.880356Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435927125525297:2101] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.881492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435927125525235:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.881795Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435930110577067:2100] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.881817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435930110577067:2100] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.884205Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435930361924169:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.884353Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435928154391573:2082] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.889413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435930110577091:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55087
2025-11-19T13:59:52.889666Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435930361924168:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.891863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435927125525258:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(38439349ns) {Configure:104(38437983ns) {ProcessConfigured:135(38235122ns) {StartInitialHandshake:157(38234551ns) {DropHandshakes:469(198ns) {DropIncomingHandshake:441(23ns) DropOutgoingHandshake:456(14ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:476(1199ns) {SwitchToState:178(244ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.893258Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435930110577070:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.893268Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435930110577070:2102] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.894515Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435930361924156:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.894528Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435930361924156:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.894864Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435930361924161:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.894870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435930361924161:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.894995Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435930110577070:2102] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:52.893166Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435927125525299:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.893196Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435927125525298:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.893313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435927125525233:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.893335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435927125525257:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.895079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927125525315:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22290
2025-11-19T13:59:52.899902Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435928154391588:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.899927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435928154391588:2093] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.900922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435927648109772:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36433
2025-11-19T13:59:52.904164Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435927125525315:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.904209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435927125525258:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.905032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435927648109695:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(44988791ns) {Configure:104(44987797ns) {ProcessConfigured:135(44683053ns) {StartInitialHandshake:157(44681996ns) {DropHandshakes:469(646ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(116ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:476(1128ns) {SwitchToState:178(568ns)}}}}}
2025-11-19T13:59:52.904716Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435930110577089:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.905205Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435927238969478:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23257244ns) {Configure:104(23256793ns) {ProcessConfigured:135(22879795ns) {StartInitialHandshake:157(22879481ns) {DropHandshakes:469(605ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(124ns)} GenerateSessionVirtualId:988(143ns) PrepareNewSessionHandshake:476(867ns) {SwitchToState:178(23ns)}}}}}
2025-11-19T13:59:52.905907Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435930110577067:2100] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:52.907013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435928154391538:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.907056Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435928154391538:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22649683ns) {IncomingHandshake:261(22649107ns) {DropIncomingHandshake:441(171ns) PrepareNewSessionHandshake:476(451ns) {SwitchToState:178(235ns)} GenerateSessionVirtualId:988(238ns) IssueIncomingHandshakeReply:193(22645048ns) {SwitchToState:178(619ns)}}}
2025-11-19T13:59:52.907921Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435928154391614:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.907894Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435927648109766:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.907918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435927648109766:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.908380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927125525319:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22290
2025-11-19T13:59:52.908627Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435927125525297:2101] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:52.908738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435927125525235:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.908760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:52.908766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435927125525235:2050] [node 3] ICP22 created new session: [6:7574435927125525320:2050]
2025-11-19T13:59:52.908772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435927125525320:2050] [node 3] ICS09 handshake done sender: [6:7574435927125525297:2101] self: [6:7574435927125525313:0] peer: [3:7574435930110577069:0] socket: 60 qp: -1
2025-11-19T13:59:52.908779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435927125525320:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:52.908801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:52.911894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435928154391580:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.916182Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435930361924156:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:52.915879Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435927648109768:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.915896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435927648109768:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.917389Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435927648109768:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:52.917487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435927648109764:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.917510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:52.917519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435927648109764:2111] [node 5] ICP22 created new session: [4:7574435927648109776:2111]
2025-11-19T13:59:52.917527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435927648109776:2111] [node 5] ICS09 handshake done sender: [4:7574435927648109768:2113] self: [4:7574435927648109767:0] peer: [5:7574435928154391622:0] socket: 72 qp: -1
2025-11-19T13:59:52.917533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435927648109776:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:52.917544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:52.917802Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435928154391612:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.922507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927125525322:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22290
2025-11-19T13:59:52.920416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435930361924154:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.920439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:52.920448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435930361924154:2107] [node 5] ICP22 created new session: [1:7574435930361924177:2107]
2025-11-19T13:59:52.920455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435930361924177:2107] [node 5] ICS09 handshake done sender: [1:7574435930361924156:2108] self: [1:7574435930361924155:0] peer: [5:7574435928154391621:0] socket: 48 qp: -1
2025-11-19T13:59:52.920462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435930361924177:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:52.920471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:52.922214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435930110577062:2094] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.922233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:52.922240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435930110577062:2094] [node 5] ICP22 created new session: [3:7574435930110577095:2094]
2025-11-19T13:59:52.922247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435930110577095:2094] [node 5] ICS09 handshake done sender: [3:7574435930110577067:2100] self: [3:7574435930110577065:0] peer: [5:7574435928154391615:0] socket: 50 qp: -1
2025-11-19T13:59:52.922253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435930110577095:2094] [node 5] ICS10 traffic start
2025-11-19T13:59:52.922261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.925160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927125525323:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22290
2025-11-19T13:59:52.930071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435930110577063:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.930109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:52.930122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435930110577063:2096] [node 6] ICP22 created new session: [3:7574435930110577098:2096]
2025-11-19T13:59:52.930130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435930110577098:2096] [node 6] ICS09 handshake done sender: [3:7574435930110577070:2102] self: [3:7574435930110577069:0] peer: [6:7574435927125525313:0] socket: 51 qp: -1
2025-11-19T13:59:52.930140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435930110577098:2096] [node 6] ICS10 traffic start
2025-11-19T13:59:52.930152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:52.930914Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435927125525299:2103] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:52.930777Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435930361924161:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:52.931897Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435928154391588:2093] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:52.935679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435928154391583:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.935719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:52.935734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435928154391583:2090] [node 6] ICP22 created new session: [5:7574435928154391625:2090]
2025-11-19T13:59:52.935768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435928154391625:2090] [node 6] ICS09 handshake done sender: [5:7574435928154391588:2093] self: [5:7574435928154391587:0] peer: [6:7574435927125525317:0] socket: 53 qp: -1
2025-11-19T13:59:52.935780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435928154391625:2090] [node 6] ICS10 traffic start
2025-11-19T13:59:52.935792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.942092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435927238969493:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10359
2025-11-19T13:59:52.943317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435930361924159:2109] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.943341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:52.943351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435930361924159:2109] [node 6] ICP22 created new session: [1:7574435930361924182:2109]
2025-11-19T13:59:52.943361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435930361924182:2109] [node 6] ICS09 handshake done sender: [1:7574435930361924161:2110] self: [1:7574435930361924160:0] peer: [6:7574435927125525316:0] socket: 54 qp: -1
2025-11-19T13:59:52.943368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435930361924182:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:52.943392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:52.944948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435930361924181:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59915
2025-11-19T13:59:52.946184Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435927238969493:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.946325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435927238969414:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.947732Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435930110577091:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.947817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435930110577012:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.948900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435927238969495:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10359
2025-11-19T13:59:52.948955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435927238969496:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10359
2025-11-19T13:59:52.948985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435927238969497:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10359
2025-11-19T13:59:52.949217Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435930361924181:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.953200Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435927648109766:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:52.953948Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435927238969497:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.956566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435927125525233:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.956593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:52.956602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435927125525233:2048] [node 1] ICP22 created new session: [6:7574435927125525324:2048]
2025-11-19T13:59:52.956610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435927125525324:2048] [node 1] ICS09 handshake done sender: [6:7574435927125525299:2103] self: [6:7574435927125525316:0] peer: [1:7574435930361924160:0] socket: 67 qp: -1
2025-11-19T13:59:52.956617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435927125525324:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:52.956641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.958371Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435927238969496:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.959886Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435927238969495:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.960201Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435930361924164:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.960222Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435930361924164:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.960269Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435927238969485:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.960279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435927238969485:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.958269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435930110577102:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55087
2025-11-19T13:59:52.960029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435930110577101:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55087
2025-11-19T13:59:52.960214Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435930110577101:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.960239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435930110577011:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.960516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435928154391580:2084] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20878) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435928154391614:2113] held: no
2025-11-19T13:59:52.960635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435928154391627:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37468
2025-11-19T13:59:52.961122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435928154391583:2090] [node 6] ICP35 event processing took too much time PendingConnection:278(25347931ns) {HandleHandshakeStatus:329(25345072ns) {DropIncomingHandshake:441(7312ns) DropOutgoingHandshake:456(181ns) SwitchToState:178(145ns) ProcessPendingSessionEvents:495(11739342ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11721881ns max# 11666058ns}}}
2025-11-19T13:59:52.961148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435928154391628:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37468
2025-11-19T13:59:52.960709Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435927125525257:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23698) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435927125525298:2102] held: no
2025-11-19T13:59:52.960884Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435927125525298:2102] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.961576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927125525326:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22290
2025-11-19T13:59:52.961376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435928154391629:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37468
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.963110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435927238969501:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10359
2025-11-19T13:59:52.961318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435927648109763:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.961336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:52.961342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435927648109763:2110] [node 6] ICP22 created new session: [4:7574435927648109782:2110]
2025-11-19T13:59:52.961353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435927648109782:2110] [node 6] ICS09 handshake done sender: [4:7574435927648109766:2112] self: [4:7574435927648109765:0] peer: [6:7574435927125525318:0] socket: 71 qp: -1
2025-11-19T13:59:52.961360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435927648109782:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:52.961379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:52.961609Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435927648109772:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.962860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435927648109780:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36433
2025-11-19T13:59:52.962900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435927648109781:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36433
2025-11-19T13:59:52.963007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435927648109695:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.964346Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435927125525315:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:52.963366Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435930110577063:2096] [node 6] ICP35 event processing took too much time PendingConnection:278(68203071ns) {HandleHandshakeStatus:329(68202046ns) {DropIncomingHandshake:441(991ns) DropOutgoingHandshake:456(599ns) SwitchToState:178(235ns) ProcessPendingSessionEvents:495(12861468ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 12842680ns max# 12792417ns}}}
2025-11-19T13:59:52.963398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435930110577063:2096] [node 6] ICP18 handshake (actor [3:7574435930110577089:2114]) is too old
2025-11-19T13:59:52.963841Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435928154391592:2095] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:52.964153Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435928154391614:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.964489Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435927648109780:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.965963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435927648109696:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.965992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435927648109786:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36433
2025-11-19T13:59:52.965816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435927125525258:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.965837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:52.965845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435927125525258:2068] [node 4] ICP22 created new session: [6:7574435927125525328:2068]
2025-11-19T13:59:52.965852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435927125525328:2068] [node 4] ICS09 handshake done sender: [6:7574435927125525315:2118] self: [6:7574435927125525318:0] peer: [4:7574435927648109765:0] socket: 79 qp: -1
2025-11-19T13:59:52.965859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435927125525328:2068] [node 4] ICS10 traffic start
2025-11-19T13:59:52.965868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:52.966192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435927125525257:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.966198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:52.966202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435927125525257:2061] [node 5] ICP22 created new session: [6:7574435927125525330:2061]
2025-11-19T13:59:52.966206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435927125525330:2061] [node 5] ICS09 handshake done sender: [6:7574435927125525298:2102] self: [6:7574435927125525317:0] peer: [5:7574435928154391587:0] socket: 66 qp: -1
2025-11-19T13:59:52.966210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435927125525330:2061] [node 5] ICS10 traffic start
2025-11-19T13:59:52.966215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:52.967879Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435927238969480:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.967890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435927238969480:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.968496Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435927238969501:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.966349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435930361924186:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59915
2025-11-19T13:59:52.967651Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435930361924171:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.967660Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435930361924171:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.967804Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435930361924164:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:52.968794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435928154391580:2084] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.968814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:52.968820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435928154391580:2084] [node 4] ICP22 created new session: [5:7574435928154391632:2084]
2025-11-19T13:59:52.968827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435928154391632:2084] [node 4] ICS09 handshake done sender: [5:7574435928154391614:2113] self: [5:7574435928154391622:0] peer: [4:7574435927648109767:0] socket: 77 qp: -1
2025-11-19T13:59:52.968832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435928154391632:2084] [node 4] ICS10 traffic start
2025-11-19T13:59:52.968839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:52.969020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435928154391540:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.969027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:52.969031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435928154391540:2050] [node 3] ICP22 created new session: [5:7574435928154391634:2050]
2025-11-19T13:59:52.969036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435928154391634:2050] [node 3] ICS09 handshake done sender: [5:7574435928154391592:2095] self: [5:7574435928154391615:0] peer: [3:7574435930110577065:0] socket: 55 qp: -1
2025-11-19T13:59:52.969040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435928154391634:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:52.969044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:52.969441Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435928154391573:2082] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:52.971522Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435930110577102:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.971569Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435930110577080:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.971575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435930110577080:2106] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.971859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435930361924093:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.971919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:52.971928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435930361924093:2048] [node 2] ICP22 created new session: [1:7574435930361924188:2048]
2025-11-19T13:59:52.971935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435930361924188:2048] [node 2] ICS09 handshake done sender: [1:7574435930361924164:2111] self: [1:7574435930361924163:0] peer: [2:7574435927238969494:0] socket: 57 qp: -1
2025-11-19T13:59:52.971943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435930361924188:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:52.971952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:52.973340Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435930110577080:2106] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:52.973501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435930110577078:2105] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.973517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:52.973523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435930110577078:2105] [node 4] ICP22 created new session: [3:7574435930110577106:2105]
2025-11-19T13:59:52.973529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435930110577106:2105] [node 4] ICS09 handshake done sender: [3:7574435930110577080:2106] self: [3:7574435930110577079:0] peer: [4:7574435927648109787:0] socket: 85 qp: -1
2025-11-19T13:59:52.973536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435930110577106:2105] [node 4] ICS10 traffic start
2025-11-19T13:59:52.973543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:52.976841Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435927648109696:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12247) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435927648109780:2117] held: no
2025-11-19T13:59:52.976235Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435927238969480:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.978137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435927238969504:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10359
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.978604Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435927648109781:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.978656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435927648109788:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36433
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.978916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435927648109789:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36433
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.979082Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435927238969485:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:52.979226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435927648109694:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.979429Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435930361924171:2115] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:52.979072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435930110577108:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55087
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.980018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435930110577109:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55087
2025-11-19T13:59:52.979761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435930361924094:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.979784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:52.979791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435930361924094:2049] [node 3] ICP22 created new session: [1:7574435930361924191:2049]
2025-11-19T13:59:52.979796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435930361924191:2049] [node 3] ICS09 handshake done sender: [1:7574435930361924171:2115] self: [1:7574435930361924170:0] peer: [3:7574435930110577103:0] socket: 92 qp: -1
2025-11-19T13:59:52.979803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435930361924191:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:52.979808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:52.980314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435927648109763:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:52.980331Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435927648109772:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.980405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435930110577110:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55087
2025-11-19T13:59:52.980506Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435930110577091:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:52.980205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435927238969415:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.980224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:52.980229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435927238969415:2049] [node 3] ICP22 created new session: [2:7574435927238969506:2049]
2025-11-19T13:59:52.980235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435927238969506:2049] [node 3] ICS09 handshake done sender: [2:7574435927238969485:2112] self: [2:7574435927238969484:0] peer: [3:7574435930110577100:0] socket: 76 qp: -1
2025-11-19T13:59:52.980244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435927238969506:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:52.980267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:52.980474Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435927238969493:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:52.980660Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435930361924186:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.983245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435930361924154:2107] [node 5] ICP18 handshake (actor [1:7574435930361924186:2118]) is too old
2025-11-19T13:59:52.980803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435930110577012:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.980818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:52.980823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435930110577012:2049] [node 2] ICP22 created new session: [3:7574435930110577111:2049]
2025-11-19T13:59:52.980829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435930110577111:2049] [node 2] ICS09 handshake done sender: [3:7574435930110577091:2116] self: [3:7574435930110577100:0] peer: [2:7574435927238969484:0] socket: 78 qp: -1
2025-11-19T13:59:52.980833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435930110577111:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:52.980840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:52.981134Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435930110577101:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:52.984203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435930110577011:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.984218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:52.984225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435930110577011:2048] [node 1] ICP22 created new session: [3:7574435930110577113:2048]
2025-11-19T13:59:52.984232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435930110577113:2048] [node 1] ICS09 handshake done sender: [3:7574435930110577101:2117] self: [3:7574435930110577103:0] peer: [1:7574435930361924170:0] socket: 97 qp: -1
2025-11-19T13:59:52.984238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435930110577113:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:52.984244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:52.981173Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435927648109780:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:52.981648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435927648109695:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.981658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:52.981664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435927648109695:2049] [node 2] ICP22 created new session: [4:7574435927648109791:2049]
2025-11-19T13:59:52.981669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435927648109791:2049] [node 2] ICS09 handshake done sender: [4:7574435927648109772:2115] self: [4:7574435927648109785:0] peer: [2:7574435927238969476:0] socket: 81 qp: -1
2025-11-19T13:59:52.981674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435927648109791:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:52.981681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:52.983600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435927238969414:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.983615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:52.983620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435927238969414:2048] [node 1] ICP22 created new session: [2:7574435927238969508:2048]
2025-11-19T13:59:52.983625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435927238969508:2048] [node 1] ICS09 handshake done sender: [2:7574435927238969493:2115] self: [2:7574435927238969494:0] peer: [1:7574435930361924163:0] socket: 61 qp: -1
2025-11-19T13:59:52.983630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435927238969508:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:52.983649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:52.984274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435927238969475:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.984289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:52.984295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435927238969475:2107] [node 4] ICP22 created new session: [2:7574435927238969510:2107]
2025-11-19T13:59:52.984300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435927238969510:2107] [node 4] ICS09 handshake done sender: [2:7574435927238969480:2110] self: [2:7574435927238969476:0] peer: [4:7574435927648109785:0] socket: 74 qp: -1
2025-11-19T13:59:52.984305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435927238969510:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:52.984310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:52.987024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435927648109696:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.987043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:52.987054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435927648109696:2050] [node 3] ICP22 created new session: [4:7574435927648109793:2050]
2025-11-19T13:59:52.987061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435927648109793:2050] [node 3] ICS09 handshake done sender: [4:7574435927648109780:2117] self: [4:7574435927648109787:0] peer: [3:7574435930110577079:0] socket: 86 qp: -1
2025-11-19T13:59:52.987068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435927648109793:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:52.987077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:52.987789Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435930110577011:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:52.988687Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435930361924173:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.988713Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435930361924173:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.987567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435928154391538:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3050) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435928154391573:2082] held: no
2025-11-19T13:59:52.987921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435928154391538:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.987932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:52.987939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435928154391538:2048] [node 1] ICP22 created new session: [5:7574435928154391636:2048]
2025-11-19T13:59:52.987943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435928154391636:2048] [node 1] ICS09 handshake done sender: [5:7574435928154391573:2082] self: [5:7574435928154391621:0] peer: [1:7574435930361924155:0] socket: 49 qp: -1
2025-11-19T13:59:52.987949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435928154391636:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:52.987956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.993025Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435930361924173:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:52.993124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435930361924167:2112] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:52.993144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:52.993149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435930361924167:2112] [node 4] ICP22 created new session: [1:7574435930361924194:2112]
2025-11-19T13:59:52.993155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435930361924194:2112] [node 4] ICS09 handshake done sender: [1:7574435930361924173:2116] self: [1:7574435930361924172:0] peer: [4:7574435927648109790:0] socket: 98 qp: -1
2025-11-19T13:59:52.993161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435930361924194:2112] [node 4] ICS10 traffic start
2025-11-19T13:59:52.993169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:52.993654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435927648109795:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36433
2025-11-19T13:59:52.993818Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435927648109781:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:52.993912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435927648109694:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:52.993923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:52.993928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435927648109694:2048] [node 1] ICP22 created new session: [4:7574435927648109796:2048]
2025-11-19T13:59:52.993933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435927648109796:2048] [node 1] ICS09 handshake done sender: [4:7574435927648109781:2118] self: [4:7574435927648109790:0] peer: [1:7574435930361924172:0] socket: 52 qp: -1
2025-11-19T13:59:52.993937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435927648109796:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:52.993943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:52.992710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927125525332:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22290
2025-11-19T13:59:52.992842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435928154391638:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37468
2025-11-19T13:59:52.998579Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435927125525332:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.998627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435927125525234:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.998761Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435928154391638:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.998796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435928154391539:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:52.999024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435927238969487:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.999034Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435927238969487:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:52.999401Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435927238969492:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:52.999403Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435927238969492:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.000256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435927125525334:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22290
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.000516Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435927238969487:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:53.001978Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435927238969492:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:53.003336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435927238969481:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.003354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:53.003361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435927238969481:2109] [node 6] ICP22 created new session: [2:7574435927238969516:2109]
2025-11-19T13:59:53.000753Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435927125525332:2123] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:53.003366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435927238969516:2109] [node 6] ICS09 handshake done sender: [2:7574435927238969487:2113] self: [2:7574435927238969486:0] peer: [6:7574435927125525333:0] socket: 63 qp: -1
2025-11-19T13:59:53.003371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435927238969516:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:53.000330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435928154391640:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37468
2025-11-19T13:59:53.002145Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435928154391638:2122] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:53.002241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435928154391539:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.002256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:53.002262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435928154391539:2049] [node 2] ICP22 created new session: [5:7574435928154391641:2049]
2025-11-19T13:59:53.002267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435928154391641:2049] [node 2] ICS09 handshake done sender: [5:7574435928154391638:2122] self: [5:7574435928154391639:0] peer: [2:7574435927238969491:0] socket: 94 qp: -1
2025-11-19T13:59:53.002271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435928154391641:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:53.003377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:53.003689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435927238969478:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.002278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:53.003697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:53.003700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435927238969478:2108] [node 5] ICP22 created new session: [2:7574435927238969518:2108]
2025-11-19T13:59:53.003703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435927238969518:2108] [node 5] ICS09 handshake done sender: [2:7574435927238969492:2114] self: [2:7574435927238969491:0] peer: [5:7574435928154391639:0] socket: 69 qp: -1
2025-11-19T13:59:53.003707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435927238969518:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:53.003711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:53.000830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435927125525234:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.000844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:53.000850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435927125525234:2049] [node 2] ICP22 created new session: [6:7574435931420492631:2049]
2025-11-19T13:59:53.000856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435931420492631:2049] [node 2] ICS09 handshake done sender: [6:7574435927125525332:2123] self: [6:7574435927125525333:0] peer: [2:7574435927238969486:0] socket: 80 qp: -1
2025-11-19T13:59:53.000862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435931420492631:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:53.000870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:53.016456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:53.016542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435927238969508:2048] [node 1] ICS01 socket: 61 reason# EndOfStream
2025-11-19T13:59:53.016550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435927238969414:2048] [node 1] ICP30 unregister session Session# [2:7574435927238969508:2048] VirtualId# [2:7574435927238969494:0]
2025-11-19T13:59:53.016557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435927238969508:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.016588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:53.016609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:53.017055Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:53.017119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435930110577113:2048] [node 1] ICS01 socket: 97 reason# EndOfStream
2025-11-19T13:59:53.017124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435930110577011:2048] [node 1] ICP30 unregister session Session# [3:7574435930110577113:2048] VirtualId# [3:7574435930110577103:0]
2025-11-19T13:59:53.017129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435930110577113:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.017155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:53.017171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:53.027011Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:53.027070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435927648109796:2048] [node 1] ICS01 socket: 52 reason# EndOfStream
2025-11-19T13:59:53.027074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435927648109694:2048] [node 1] ICP30 unregister session Session# [4:7574435927648109796:2048] VirtualId# [4:7574435927648109790:0]
2025-11-19T13:59:53.027078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435927648109796:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.027105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:53.027137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:53.026850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:53.026918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435927125525324:2048] [node 1] ICS01 socket: 67 reason# EndOfStream
2025-11-19T13:59:53.026922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435927125525233:2048] [node 1] ICP30 unregister session Session# [6:7574435927125525324:2048] VirtualId# [6:7574435927125525316:0]
2025-11-19T13:59:53.026926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435927125525324:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.026961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:53.026981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:53.026628Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:53.026723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435928154391636:2048] [node 1] ICS01 socket: 49 reason# EndOfStream
2025-11-19T13:59:53.026732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435928154391538:2048] [node 1] ICP30 unregister session Session# [5:7574435928154391636:2048] VirtualId# [5:7574435928154391621:0]
2025-11-19T13:59:53.026738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435928154391636:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.026788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:53.026813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:53.044634Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:53.044737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435930110577111:2049] [node 2] ICS01 socket: 78 reason# EndOfStream
2025-11-19T13:59:53.044747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435930110577012:2049] [node 2] ICP30 unregister session Session# [3:7574435930110577111:2049] VirtualId# [3:7574435930110577100:0]
2025-11-19T13:59:53.044755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435930110577111:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.044789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:53.044810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:53.065532Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:53.065632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435927648109791:2049] [node 2] ICS01 socket: 81 reason# EndOfStream
2025-11-19T13:59:53.065642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435927648109695:2049] [node 2] ICP30 unregister session Session# [4:7574435927648109791:2049] VirtualId# [4:7574435927648109785:0]
2025-11-19T13:59:53.065649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435927648109791:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.065785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:53.065847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435928154391641:2049] [node 2] ICS01 socket: 94 reason# EndOfStream
2025-11-19T13:59:53.065851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435928154391539:2049] [node 2] ICP30 unregister session Session# [5:7574435928154391641:2049] VirtualId# [5:7574435928154391639:0]
2025-11-19T13:59:53.065856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435928154391641:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.065880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:53.065901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:53.065933Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:53.065991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435931420492631:2049] [node 2] ICS01 socket: 80 reason# EndOfStream
2025-11-19T13:59:53.065995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435927125525234:2049] [node 2] ICP30 unregister session Session# [6:7574435931420492631:2049] VirtualId# [6:7574435927125525333:0]
2025-11-19T13:59:53.065999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435931420492631:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.066023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:53.066043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:53.065709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:53.065742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:53.089806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:53.089911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435927125525320:2050] [node 3] ICS01 socket: 60 reason# EndOfStream
2025-11-19T13:59:53.089921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435927125525235:2050] [node 3] ICP30 unregister session Session# [6:7574435927125525320:2050] VirtualId# [6:7574435927125525313:0]
2025-11-19T13:59:53.089927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435927125525320:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.089994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:53.090022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:53.090153Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:53.090254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435928154391634:2050] [node 3] ICS01 socket: 55 reason# EndOfStream
2025-11-19T13:59:53.090259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435928154391540:2050] [node 3] ICP30 unregister session Session# [5:7574435928154391634:2050] VirtualId# [5:7574435928154391615:0]
2025-11-19T13:59:53.090263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435928154391634:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.090299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:53.090322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:53.108046Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:53.108143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435927125525328:2068] [node 4] ICS01 socket: 79 reason# EndOfStream
2025-11-19T13:59:53.108154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435927125525258:2068] [node 4] ICP30 unregister session Session# [6:7574435927125525328:2068] VirtualId# [6:7574435927125525318:0]
2025-11-19T13:59:53.108161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435927125525328:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.108232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:53.108260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T13:59:53.195694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435931199843011:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:53.195713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435931199843012:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:53.252191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435934079071715:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:53.252204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435934079071716:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:53.303936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435934432774531:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:53.303956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435934432774532:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:53.369044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435931585418890:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:53.369055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435931585418891:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:53.369060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435931585418892:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:53.405217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435932901394349:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:53.405231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435932901394350:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:53.405236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435932901394351:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:53.456205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435932865986055:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:53.456218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435932865986056:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:53.467772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435932865986054:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:53.502932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435934079071776:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.504082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435931585418951:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60495
2025-11-19T13:59:53.504271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435932865986089:2079] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.505431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435931199843072:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:53.506015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435931199843073:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58117
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.507979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435931199843078:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:53.508690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435934079071782:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:53.508983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435932901394410:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31764
2025-11-19T13:59:53.512377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435932865986112:2089] [node 4] ICP01 ready to work
2025-11-19T13:59:53.511902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435934432774594:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20770
2025-11-19T13:59:53.512108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435934432774596:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.523978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435932865986126:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30975
2025-11-19T13:59:53.524063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435932865986127:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30975
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.526724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435934432774599:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20770
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.529516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435931585418952:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60495
2025-11-19T13:59:53.529084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435931199843012:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18521011ns) {Configure:104(18519733ns) {ProcessConfigured:135(18241377ns) {StartInitialHandshake:157(18241079ns) {DropHandshakes:469(492ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(293ns) PrepareNewSessionHandshake:476(1126ns) {SwitchToState:178(108ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.530912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435931585418954:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:53.530971Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435931585418951:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.531468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435931585418955:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60495
2025-11-19T13:59:53.531691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435931585418891:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.532071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435932901394413:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.530076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435934432774601:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:53.530639Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435934432774596:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16144900ns) {Configure:104(16143379ns) {ProcessConfigured:135(15818975ns) {StartInitialHandshake:157(15818603ns) {DropHandshakes:469(1010ns) {DropIncomingHandshake:441(560ns) DropOutgoingHandshake:456(93ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:476(656ns) {SwitchToState:178(203ns)}}}}}
2025-11-19T13:59:53.532166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435934432774607:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20770
2025-11-19T13:59:53.533251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435932901394411:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31764
2025-11-19T13:59:53.533596Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435931199843073:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.533630Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435931199843075:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.533633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435931199843075:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.528786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435932865986112:2089] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16323787ns) {Configure:104(16322609ns) {ProcessConfigured:135(15979184ns) {StartInitialHandshake:157(15978324ns) {DropHandshakes:469(473ns) {DropIncomingHandshake:441(113ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:476(1374ns) {SwitchToState:178(946ns)}}}}}
2025-11-19T13:59:53.532060Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435932865986126:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.533470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435932865986054:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:53.533800Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435932901394410:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.530149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435934079071792:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37399
2025-11-19T13:59:53.536011Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435934432774594:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.535771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435931199843085:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:53.536185Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435934432774599:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.536316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435934432774609:2115] [node 6] ICP01 ready to work
2025-11-19T13:59:53.535911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435934079071791:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:53.538199Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934079071792:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.538285Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435931199843075:2109] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:53.538386Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435932901394411:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.538644Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435931585418955:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.539972Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435932865986127:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.540345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435932865986055:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:53.540539Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934079071784:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.540550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435934079071784:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.541287Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435934079071784:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:53.541379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435934079071782:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.541392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:53.541399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435934079071782:2110] [node 6] ICP22 created new session: [2:7574435934079071796:2110]
2025-11-19T13:59:53.541406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435934079071796:2110] [node 6] ICS09 handshake done sender: [2:7574435934079071784:2111] self: [2:7574435934079071783:0] peer: [6:7574435932865986133:0] socket: 64 qp: -1
2025-11-19T13:59:53.541413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435934079071796:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:53.541430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:53.543868Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435931585418952:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.545608Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435931199843078:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(37570935ns) {Configure:104(37570146ns) {ProcessConfigured:135(37281717ns) {StartInitialHandshake:157(37280888ns) {DropHandshakes:469(526ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(64ns)} GenerateSessionVirtualId:988(62ns) PrepareNewSessionHandshake:476(1072ns) {SwitchToState:178(392ns)}}}}}
2025-11-19T13:59:53.545630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435931199843072:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.545641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:53.545647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435931199843072:2107] [node 6] ICP22 created new session: [1:7574435931199843092:2107]
2025-11-19T13:59:53.545653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435931199843092:2107] [node 6] ICS09 handshake done sender: [1:7574435931199843075:2109] self: [1:7574435931199843074:0] peer: [6:7574435932865986132:0] socket: 60 qp: -1
2025-11-19T13:59:53.545660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435931199843092:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:53.545678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:53.545851Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934079071778:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.545866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435934079071778:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.544954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435932865986134:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30975
2025-11-19T13:59:53.544979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435932865986135:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30975
2025-11-19T13:59:53.548645Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435932865986126:2112] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:53.548750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435932865986054:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.548767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:53.548774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435932865986054:2048] [node 1] ICP22 created new session: [6:7574435932865986137:2048]
2025-11-19T13:59:53.548779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435932865986137:2048] [node 1] ICS09 handshake done sender: [6:7574435932865986126:2112] self: [6:7574435932865986132:0] peer: [1:7574435931199843074:0] socket: 63 qp: -1
2025-11-19T13:59:53.548787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435932865986137:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:53.548796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.553049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435931199843082:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.553060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435931199843082:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.554167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435931199843094:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58117
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.554386Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435934432774603:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.554399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435934432774603:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.554940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435934432774611:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20770
2025-11-19T13:59:53.555209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435934079071799:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37399
2025-11-19T13:59:53.555227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435934079071800:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37399
2025-11-19T13:59:53.555480Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435934432774609:2115] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19119397ns) {Configure:104(19119211ns) {ProcessConfigured:135(18820444ns) {StartInitialHandshake:157(18808784ns) {DropHandshakes:469(425ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(658ns) PrepareNewSessionHandshake:476(293ns) {SwitchToState:178(21ns)}} IncomingHandshake:261(4921ns) {DropIncomingHandshake:441(166ns) PrepareNewSessionHandshake:476(185ns) {SwitchToState:178(130ns)} GenerateSessionVirtualId:988(232ns) IssueIncomingHandshakeReply:193(320ns)}}}}
2025-11-19T13:59:53.555302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435931199843085:2112] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19306074ns) {Configure:104(19305403ns) {ProcessConfigured:135(18959896ns) {StartInitialHandshake:157(18959407ns) {DropHandshakes:469(674ns) {DropIncomingHandshake:441(90ns) DropOutgoingHandshake:456(49ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:476(871ns) {SwitchToState:178(181ns)}}}}}
2025-11-19T13:59:53.555353Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435931199843082:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:53.555851Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435931199843094:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.555862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435931199843011:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.555874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: 2025-11-19T13:59:53.555880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435931199843011:2048] [node 2] ICP22 created new session: [1:7574435931199843098:2048]
::ffff:127.1.0.0
2025-11-19T13:59:53.555886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435931199843098:2048] [node 2] ICS09 handshake done sender: [1:7574435931199843082:2111] self: [1:7574435931199843081:0] peer: [2:7574435934079071794:0] socket: 66 qp: -1
2025-11-19T13:59:53.555892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435931199843098:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:53.555899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:53.555863Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435934432774603:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:53.556052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435934432774607:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.556079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435934432774531:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:53.556325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435934079071791:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19504749ns) {Configure:104(19504297ns) {ProcessConfigured:135(19162056ns) {StartInitialHandshake:157(19161552ns) {DropHandshakes:469(694ns) {DropIncomingHandshake:441(32ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:476(1215ns) {SwitchToState:178(568ns)}}}}}
2025-11-19T13:59:53.556146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435934432774601:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.556158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:53.556162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435934432774601:2111] [node 5] ICP22 created new session: [3:7574435934432774616:2111]
2025-11-19T13:59:53.556166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435934432774616:2111] [node 5] ICS09 handshake done sender: [3:7574435934432774603:2112] self: [3:7574435934432774602:0] peer: [5:7574435932901394421:0] socket: 73 qp: -1
2025-11-19T13:59:53.556170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435934432774616:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:53.556186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.556456Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435931585418892:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19459288ns) {Configure:104(19458862ns) {ProcessConfigured:135(19162897ns) {StartInitialHandshake:157(19162369ns) {DropHandshakes:469(435ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(325ns) PrepareNewSessionHandshake:476(727ns) {SwitchToState:178(20ns)}}}}}
2025-11-19T13:59:53.556491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435931585418892:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.556998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435932901394428:2114] [node 4] ICP01 ready to work
2025-11-19T13:59:53.558167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435932901394426:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31764
2025-11-19T13:59:53.558185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435932901394427:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31764
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.559427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435931585418967:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60495
2025-11-19T13:59:53.559683Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435931585418955:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:53.558338Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435934432774532:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(43418039ns) {Configure:104(43417236ns) {ProcessConfigured:135(43166974ns) {StartInitialHandshake:157(43166196ns) {DropHandshakes:469(437ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:476(598ns) {SwitchToState:178(202ns)}}}}}
2025-11-19T13:59:53.558381Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435934432774605:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.558384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435934432774605:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.558845Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435934432774611:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.558862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435934432774532:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:53.559017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435934432774620:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20770
2025-11-19T13:59:53.559521Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435934432774605:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:53.559762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435931585418892:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.559778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:53.559785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435931585418892:2050] [node 3] ICP22 created new session: [4:7574435931585418968:2050]
2025-11-19T13:59:53.559790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435931585418968:2050] [node 3] ICS09 handshake done sender: [4:7574435931585418955:2111] self: [4:7574435931585418965:0] peer: [3:7574435934432774597:0] socket: 76 qp: -1
2025-11-19T13:59:53.559796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435931585418968:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:53.559813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:53.560320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435934432774596:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.560335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:53.560341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435934432774596:2109] [node 4] ICP22 created new session: [3:7574435934432774623:2109]
2025-11-19T13:59:53.560346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435934432774623:2109] [node 4] ICS09 handshake done sender: [3:7574435934432774605:2113] self: [3:7574435934432774597:0] peer: [4:7574435931585418965:0] socket: 74 qp: -1
2025-11-19T13:59:53.560351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435934432774623:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:53.560367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:53.564393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435934079071715:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:53.564435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435934079071715:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(26187855ns) {IncomingHandshake:261(26187375ns) {DropIncomingHandshake:441(69ns) PrepareNewSessionHandshake:476(324ns) {SwitchToState:178(199ns)} GenerateSessionVirtualId:988(197ns) IssueIncomingHandshakeReply:193(26184196ns) {SwitchToState:178(508ns)}}}
2025-11-19T13:59:53.564287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435931585418961:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:53.564485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435932901394351:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:53.564510Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435932901394351:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(26080303ns) {IncomingHandshake:261(26080144ns) {DropIncomingHandshake:441(20ns) PrepareNewSessionHandshake:476(238ns) {SwitchToState:178(93ns)} GenerateSessionVirtualId:988(201ns) IssueIncomingHandshakeReply:193(26076717ns) {SwitchToState:178(104ns)}}}
2025-11-19T13:59:53.564962Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435932901394411:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:53.564911Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435931199843086:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.564927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435931199843086:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.565207Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934079071799:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.565226Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435934079071792:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:53.565295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435931585418963:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60495
2025-11-19T13:59:53.567842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435932865986136:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30975
2025-11-19T13:59:53.572137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435934079071715:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.572161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:53.572170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435934079071715:2048] [node 1] ICP22 created new session: [2:7574435934079071803:2048]
2025-11-19T13:59:53.572177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435934079071803:2048] [node 1] ICS09 handshake done sender: [2:7574435934079071792:2114] self: [2:7574435934079071794:0] peer: [1:7574435931199843081:0] socket: 68 qp: -1
2025-11-19T13:59:53.572186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435934079071803:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:53.572197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:53.573985Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934079071786:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.573995Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435934079071786:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.573532Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435932865986127:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.583399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435932865986055:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.583422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:53.583430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435932865986055:2049] [node 2] ICP22 created new session: [6:7574435932865986140:2049]
2025-11-19T13:59:53.583437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435932865986140:2049] [node 2] ICS09 handshake done sender: [6:7574435932865986127:2113] self: [6:7574435932865986133:0] peer: [2:7574435934079071783:0] socket: 69 qp: -1
2025-11-19T13:59:53.583446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435932865986140:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:53.583482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:53.584152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435931199843011:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(28186174ns) {HandleHandshakeStatus:329(28185528ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(199ns) SwitchToState:178(52ns) ProcessPendingSessionEvents:495(19202141ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19183179ns max# 19112727ns}}}
2025-11-19T13:59:53.584416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435931199843011:2048] [node 2] ICP18 handshake (actor [1:7574435931199843094:2115]) is too old
2025-11-19T13:59:53.584819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435934432774601:2111] [node 5] ICP35 event processing took too much time PendingConnection:278(28616323ns) {HandleHandshakeStatus:329(28616054ns) {DropIncomingHandshake:441(28ns) DropOutgoingHandshake:456(94ns) SwitchToState:178(52ns) ProcessPendingSessionEvents:495(19878132ns) {unordered ForwardSessionEventToSession:612 num# 50 sum# 19867170ns max# 19834043ns}}}
2025-11-19T13:59:53.584873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435934432774601:2111] [node 5] ICP18 handshake (actor [3:7574435934432774611:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.585581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435931585418974:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60495
2025-11-19T13:59:53.585672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435931199843101:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58117
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.592009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435931585418961:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26904959ns) {Configure:104(26904447ns) {ProcessConfigured:135(26607529ns) {StartInitialHandshake:157(26586141ns) {DropHandshakes:469(675ns) {DropIncomingHandshake:441(121ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(1119ns) PrepareNewSessionHandshake:476(1415ns) {SwitchToState:178(131ns)}} IncomingHandshake:261(9117ns) {DropIncomingHandshake:441(201ns) PrepareNewSessionHandshake:476(200ns) {SwitchToState:178(153ns)} GenerateSessionVirtualId:988(279ns) IssueIncomingHandshakeReply:193(381ns)}}}}
2025-11-19T13:59:53.593747Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435931199843086:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:53.594169Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435931585418963:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.594200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435931585418890:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:53.593914Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435931199843101:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.594007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435931199843012:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.594027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:53.594033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435931199843012:2049] [node 3] ICP22 created new session: [1:7574435931199843104:2049]
2025-11-19T13:59:53.594039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435931199843104:2049] [node 3] ICS09 handshake done sender: [1:7574435931199843086:2113] self: [1:7574435931199843080:0] peer: [3:7574435934432774615:0] socket: 71 qp: -1
2025-11-19T13:59:53.594045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435931199843104:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:53.594052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.595927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435932865986054:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(47107614ns) {HandleHandshakeStatus:329(47106995ns) {DropIncomingHandshake:441(356ns) DropOutgoingHandshake:456(1869ns) SwitchToState:178(157ns) ProcessPendingSessionEvents:495(29418703ns) {unordered ForwardSessionEventToSession:612 num# 50 sum# 29405279ns max# 29368852ns}}}
2025-11-19T13:59:53.596222Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435932865986134:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.596482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435932865986089:2079] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:53.596508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435932865986142:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30975
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.597396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435931585418979:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60495
2025-11-19T13:59:53.596456Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435931199843091:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.596470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435931199843091:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.597279Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435931199843091:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:53.597499Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435931199843072:2107] [node 6] ICP35 event processing took too much time PendingConnection:278(51746017ns) {HandleHandshakeStatus:329(51745182ns) {DropIncomingHandshake:441(113ns) DropOutgoingHandshake:456(175ns) SwitchToState:178(42ns) ProcessPendingSessionEvents:495(29551195ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29530867ns max# 29459235ns}}}
2025-11-19T13:59:53.598021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435934432774531:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1507) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435934432774607:2114] held: no
2025-11-19T13:59:53.598843Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435931585418963:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:53.600050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435932865986139:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30975
2025-11-19T13:59:53.600137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435934432774626:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20770
2025-11-19T13:59:53.598890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435932901394351:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.598930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:53.598939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435932901394351:2050] [node 3] ICP22 created new session: [5:7574435932901394434:2050]
2025-11-19T13:59:53.598946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435932901394434:2050] [node 3] ICS09 handshake done sender: [5:7574435932901394411:2109] self: [5:7574435932901394421:0] peer: [3:7574435934432774602:0] socket: 78 qp: -1
2025-11-19T13:59:53.598953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435932901394434:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:53.598974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:53.603797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435931199843103:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58117
2025-11-19T13:59:53.603924Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435934432774607:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:53.604132Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435932865986139:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.604175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435932865986056:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:53.604978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435931199843078:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.605006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:53.605014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435931199843078:2110] [node 4] ICP22 created new session: [1:7574435931199843108:2110]
2025-11-19T13:59:53.605020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435931199843108:2110] [node 4] ICS09 handshake done sender: [1:7574435931199843091:2114] self: [1:7574435931199843079:0] peer: [4:7574435931585418978:0] socket: 92 qp: -1
2025-11-19T13:59:53.605027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435931199843108:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:53.605035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:53.609599Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435934079071778:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:53.607828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435934432774531:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.607851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:53.607859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435934432774531:2048] [node 1] ICP22 created new session: [3:7574435934432774627:2048]
2025-11-19T13:59:53.607865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435934432774627:2048] [node 1] ICS09 handshake done sender: [3:7574435934432774607:2114] self: [3:7574435934432774615:0] peer: [1:7574435931199843080:0] socket: 75 qp: -1
2025-11-19T13:59:53.607873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435934432774627:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:53.607880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.612570Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435934079071786:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.612712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435934079071776:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.612728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:53.612735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435934079071776:2107] [node 4] ICP22 created new session: [2:7574435934079071808:2107]
2025-11-19T13:59:53.612741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435934079071808:2107] [node 4] ICS09 handshake done sender: [2:7574435934079071778:2108] self: [2:7574435934079071777:0] peer: [4:7574435931585418958:0] socket: 54 qp: -1
2025-11-19T13:59:53.612747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435934079071808:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:53.612754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:53.612333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435934432774629:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20770
2025-11-19T13:59:53.614887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435934079071716:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.614901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:53.614906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435934079071716:2049] [node 3] ICP22 created new session: [2:7574435934079071810:2049]
2025-11-19T13:59:53.614911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435934079071810:2049] [node 3] ICS09 handshake done sender: [2:7574435934079071786:2112] self: [2:7574435934079071785:0] peer: [3:7574435934432774621:0] socket: 65 qp: -1
2025-11-19T13:59:53.614917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435934079071810:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:53.614923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:53.615351Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435934432774599:2110] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:53.615472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435934432774532:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.615485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:53.615490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435934432774532:2049] [node 2] ICP22 created new session: [3:7574435934432774630:2049]
2025-11-19T13:59:53.615495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435934432774630:2049] [node 2] ICS09 handshake done sender: [3:7574435934432774599:2110] self: [3:7574435934432774621:0] peer: [2:7574435934079071785:0] socket: 67 qp: -1
2025-11-19T13:59:53.615499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435934432774630:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:53.615505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:53.614487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435932901394436:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31764
2025-11-19T13:59:53.614536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435932901394437:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31764
2025-11-19T13:59:53.614615Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435931199843103:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.614858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435931585418890:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.614876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:53.614882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435931585418890:2048] [node 1] ICP22 created new session: [4:7574435931585418980:2048]
2025-11-19T13:59:53.614889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435931585418980:2048] [node 1] ICS09 handshake done sender: [4:7574435931585418963:2114] self: [4:7574435931585418978:0] peer: [1:7574435931199843079:0] socket: 96 qp: -1
2025-11-19T13:59:53.614894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435931585418980:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:53.614911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.617194Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435932901394437:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.617239Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435932901394436:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.617264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435932901394350:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:53.617275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435932901394349:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:53.620052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435934432774613:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.620063Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435934432774613:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.619898Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435932901394418:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.619925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435932901394418:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.624631Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435931585418890:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:53.624841Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435931585418951:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:53.625956Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435932901394426:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.625991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435932901394428:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:53.627969Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435932865986142:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.628046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435932865986112:2089] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:53.628185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435934079071812:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37399
2025-11-19T13:59:53.628358Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435934079071716:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:53.628612Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934079071801:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.628616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435934079071801:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.628481Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435931199843097:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.628497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435931199843097:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.634115Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435931585418957:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.634129Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435931585418957:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:53.634602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435931585418891:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.634619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:53.634627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435931585418891:2049] [node 2] ICP22 created new session: [4:7574435931585418982:2049]
2025-11-19T13:59:53.634634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435931585418982:2049] [node 2] ICS09 handshake done sender: [4:7574435931585418951:2108] self: [4:7574435931585418958:0] peer: [2:7574435934079071777:0] socket: 55 qp: -1
2025-11-19T13:59:53.634642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435931585418982:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:53.634651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:53.635027Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435931585418975:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.635030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435931585418975:2119] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.636174Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435932901394351:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(71068161ns) {HandleHandshakeStatus:329(71067336ns) {DropIncomingHandshake:441(569ns) DropOutgoingHandshake:456(2369ns) SwitchToState:178(70ns) ProcessPendingSessionEvents:495(16251038ns) {unordered ForwardSessionEventToSession:612 num# 40 sum# 16241267ns max# 16214403ns}}}
2025-11-19T13:59:53.636226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435932901394441:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31764
2025-11-19T13:59:53.636247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435932901394351:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:53.638032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435934432774532:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(22467810ns) {HandleHandshakeStatus:329(22467384ns) {DropIncomingHandshake:441(179ns) DropOutgoingHandshake:456(1063ns) SwitchToState:178(71ns) ProcessPendingSessionEvents:495(22398179ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22376751ns max# 22300517ns}}}
2025-11-19T13:59:53.638784Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435931585418975:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:53.639576Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435931199843097:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:53.639804Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435932901394418:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:53.640507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435931585418961:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.640528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:53.640535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435931585418961:2113] [node 6] ICP22 created new session: [4:7574435931585418986:2113]
2025-11-19T13:59:53.640541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435931585418986:2113] [node 6] ICS09 handshake done sender: [4:7574435931585418975:2119] self: [4:7574435931585418973:0] peer: [6:7574435932865986145:0] socket: 90 qp: -1
2025-11-19T13:59:53.640547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435931585418986:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:53.640555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:53.640894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435931199843085:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.640917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:53.640979Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435934079071801:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:53.640925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435931199843085:2112] [node 5] ICP22 created new session: [1:7574435931199843112:2112]
2025-11-19T13:59:53.640936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435931199843112:2112] [node 5] ICS09 handshake done sender: [1:7574435931199843097:2116] self: [1:7574435931199843089:0] peer: [5:7574435932901394439:0] socket: 107 qp: -1
2025-11-19T13:59:53.640942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435931199843112:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:53.640949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:53.641685Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435934432774613:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:53.642022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435932901394413:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.642045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:53.642052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435932901394413:2110] [node 6] ICP22 created new session: [5:7574435932901394443:2110]
2025-11-19T13:59:53.642057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435932901394443:2110] [node 6] ICS09 handshake done sender: [5:7574435932901394418:2112] self: [5:7574435932901394417:0] peer: [6:7574435932865986143:0] socket: 81 qp: -1
2025-11-19T13:59:53.642062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435932901394443:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:53.642068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:53.643077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435934432774609:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.643096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:53.643101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435934432774609:2115] [node 6] ICP22 created new session: [3:7574435934432774633:2115]
2025-11-19T13:59:53.643107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435934432774633:2115] [node 6] ICS09 handshake done sender: [3:7574435934432774613:2117] self: [3:7574435934432774610:0] peer: [6:7574435932865986144:0] socket: 97 qp: -1
2025-11-19T13:59:53.643111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435934432774633:2115] [node 6] ICS10 traffic start
2025-11-19T13:59:53.643118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:53.643526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435934079071791:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.643545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:53.643549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435934079071791:2109] [node 5] ICP22 created new session: [2:7574435934079071814:2109]
2025-11-19T13:59:53.643555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435934079071814:2109] [node 5] ICS09 handshake done sender: [2:7574435934079071801:2117] self: [2:7574435934079071793:0] peer: [5:7574435932901394438:0] socket: 59 qp: -1
2025-11-19T13:59:53.643559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435934079071814:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:53.643564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.644176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435932865986146:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30975
2025-11-19T13:59:53.644466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435932865986147:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30975
2025-11-19T13:59:53.644511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435932865986148:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30975
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.644753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435932901394349:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1507) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435932901394436:2119] held: no
2025-11-19T13:59:53.644884Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435932901394350:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6034) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435932901394437:2120] held: no
2025-11-19T13:59:53.645554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435932901394446:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31764
2025-11-19T13:59:53.645614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435932901394447:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31764
2025-11-19T13:59:53.645763Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435932901394437:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:53.646902Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435932901394436:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:53.646983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435932901394350:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.646994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:53.647001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435932901394350:2049] [node 2] ICP22 created new session: [5:7574435932901394448:2049]
2025-11-19T13:59:53.647007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435932901394448:2049] [node 2] ICS09 handshake done sender: [5:7574435932901394437:2120] self: [5:7574435932901394438:0] peer: [2:7574435934079071793:0] socket: 114 qp: -1
2025-11-19T13:59:53.647014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435932901394448:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:53.647022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:53.647432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435931585418988:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60495
2025-11-19T13:59:53.647757Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435931585418988:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:53.648034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435932901394349:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.648049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:53.648055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435932901394349:2048] [node 1] ICP22 created new session: [5:7574435932901394450:2048]
2025-11-19T13:59:53.648062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435932901394450:2048] [node 1] ICS09 handshake done sender: [5:7574435932901394436:2119] self: [5:7574435932901394439:0] peer: [1:7574435931199843089:0] socket: 108 qp: -1
2025-11-19T13:59:53.648067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435932901394450:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:53.648077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:53.650393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435932865986089:2079] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5244) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435932865986134:2116] held: no
2025-11-19T13:59:53.650491Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435932865986056:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8785) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435932865986139:2119] held: no
2025-11-19T13:59:53.650568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435932865986112:2089] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20460) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435932865986142:2120] held: no
2025-11-19T13:59:53.650642Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435932865986134:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:53.650779Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435932865986139:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:53.650870Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435932865986142:2120] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:53.650982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435932865986112:2089] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.650997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:53.651003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435932865986112:2089] [node 4] ICP22 created new session: [6:7574435932865986149:2089]
2025-11-19T13:59:53.651008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435932865986149:2089] [node 4] ICS09 handshake done sender: [6:7574435932865986142:2120] self: [6:7574435932865986145:0] peer: [4:7574435931585418973:0] socket: 111 qp: -1
2025-11-19T13:59:53.651014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435932865986149:2089] [node 4] ICS10 traffic start
2025-11-19T13:59:53.651021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:53.651402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435932865986056:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.651415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:53.651420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435932865986056:2050] [node 3] ICP22 created new session: [6:7574435932865986151:2050]
2025-11-19T13:59:53.651425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435932865986151:2050] [node 3] ICS09 handshake done sender: [6:7574435932865986139:2119] self: [6:7574435932865986144:0] peer: [3:7574435934432774610:0] socket: 105 qp: -1
2025-11-19T13:59:53.651430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435932865986151:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:53.651449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:53.651909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435932865986089:2079] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.651921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:53.651926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435932865986089:2079] [node 5] ICP22 created new session: [6:7574435932865986153:2079]
2025-11-19T13:59:53.651931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435932865986153:2079] [node 5] ICS09 handshake done sender: [6:7574435932865986134:2116] self: [6:7574435932865986143:0] peer: [5:7574435932901394417:0] socket: 82 qp: -1
2025-11-19T13:59:53.651934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435932865986153:2079] [node 5] ICS10 traffic start
2025-11-19T13:59:53.651941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:53.652597Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435931585418957:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:53.652688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435931585418954:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:53.652705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:53.652710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435931585418954:2110] [node 5] ICP22 created new session: [4:7574435931585418990:2110]
2025-11-19T13:59:53.652715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435931585418990:2110] [node 5] ICS09 handshake done sender: [4:7574435931585418957:2112] self: [4:7574435931585418956:0] peer: [5:7574435932901394440:0] socket: 77 qp: -1
2025-11-19T13:59:53.652719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435931585418990:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:53.652724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:53.656949Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435932901394428:2114] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20460) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435932901394426:2115] held: no
2025-11-19T13:59:53.657061Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435932901394426:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:53.657295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435932901394428:2114] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:53.657309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:53.657316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435932901394428:2114] [node 4] ICP22 created new session: [5:7574435932901394452:2114]
2025-11-19T13:59:53.657322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435932901394452:2114] [node 4] ICS09 handshake done sender: [5:7574435932901394426:2115] self: [5:7574435932901394440:0] peer: [4:7574435931585418956:0] socket: 79 qp: -1
2025-11-19T13:59:53.657329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435932901394452:2114] [node 4] ICS10 traffic start
2025-11-19T13:59:53.657337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:53.670969Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:53.671057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435934079071803:2048] [node 1] ICS01 socket: 68 reason# EndOfStream
2025-11-19T13:59:53.671065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435934079071715:2048] [node 1] ICP30 unregister session Session# [2:7574435934079071803:2048] VirtualId# [2:7574435934079071794:0]
2025-11-19T13:59:53.671073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435934079071803:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.671109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:53.671140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:53.671727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:53.671867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435934432774627:2048] [node 1] ICS01 socket: 75 reason# EndOfStream
2025-11-19T13:59:53.671874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435934432774531:2048] [node 1] ICP30 unregister session Session# [3:7574435934432774627:2048] VirtualId# [3:7574435934432774615:0]
2025-11-19T13:59:53.671880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435934432774627:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.671925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:53.671946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:53.679933Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:53.680029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435932865986137:2048] [node 1] ICS01 socket: 63 reason# EndOfStream
2025-11-19T13:59:53.680038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435932865986054:2048] [node 1] ICP30 unregister session Session# [6:7574435932865986137:2048] VirtualId# [6:7574435932865986132:0]
2025-11-19T13:59:53.680046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435932865986137:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.680111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:53.680138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:53.680178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:53.680241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435931585418980:2048] [node 1] ICS01 socket: 96 reason# EndOfStream
2025-11-19T13:59:53.680245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435931585418890:2048] [node 1] ICP30 unregister session Session# [4:7574435931585418980:2048] VirtualId# [4:7574435931585418978:0]
2025-11-19T13:59:53.680250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435931585418980:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.680281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:53.680303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:53.680332Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:53.680391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435932901394450:2048] [node 1] ICS01 socket: 108 reason# EndOfStream
2025-11-19T13:59:53.680396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435932901394349:2048] [node 1] ICP30 unregister session Session# [5:7574435932901394450:2048] VirtualId# [5:7574435932901394439:0]
2025-11-19T13:59:53.680400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435932901394450:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.680432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:53.680453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:53.688684Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:53.688783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435934432774630:2049] [node 2] ICS07 socket disconnect 67 reason# EndOfStream
2025-11-19T13:59:53.688790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435934432774630:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.688842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:53.688863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435934432774630:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:53.700201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435934432774532:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6034) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T13:59:53.700220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435934432774630:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:53.700667Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435934432774532:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6034) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:53.700679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435934432774630:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:53.700684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435934432774532:2049] [node 2] ICP30 unregister session Session# [3:7574435934432774630:2049] VirtualId# [3:7574435934432774621:0]
2025-11-19T13:59:53.700728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:53.700733Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435934432774532:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6034) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:53.700736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6034) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:53.703731Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:53.703855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435931585418982:2049] [node 2] ICS01 socket: 55 reason# EndOfStream
2025-11-19T13:59:53.703863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435931585418891:2049] [node 2] ICP30 unregister session Session# [4:7574435931585418982:2049] VirtualId# [4:7574435931585418958:0]
2025-11-19T13:59:53.703869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435931585418982:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.703907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:53.703930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:53.703968Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:53.704030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435932901394448:2049] [node 2] ICS01 socket: 114 reason# EndOfStream
2025-11-19T13:59:53.704035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435932901394350:2049] [node 2] ICP30 unregister session Session# [5:7574435932901394448:2049] VirtualId# [5:7574435932901394438:0]
2025-11-19T13:59:53.704039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435932901394448:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.704067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:53.704087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:53.704228Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:53.704305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435932865986140:2049] [node 2] ICS07 socket disconnect 69 reason# EndOfStream
2025-11-19T13:59:53.704311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435932865986140:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.704343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:53.704365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435932865986140:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:53.705036Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435932865986055:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6034) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:53.705048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435932865986140:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:53.705053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435932865986055:2049] [node 2] ICP30 unregister session Session# [6:7574435932865986140:2049] VirtualId# [6:7574435932865986133:0]
2025-11-19T13:59:53.705067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:53.705073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435932865986055:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6034) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:53.705075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6034) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:53.715215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435932865986055:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:53.728445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:53.728543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435932865986151:2050] [node 3] ICS01 socket: 105 reason# EndOfStream
2025-11-19T13:59:53.728551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435932865986056:2050] [node 3] ICP30 unregister session Session# [6:7574435932865986151:2050] VirtualId# [6:7574435932865986144:0]
2025-11-19T13:59:53.728557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435932865986151:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.728603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:53.728623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:53.728656Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:53.728714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435932901394434:2050] [node 3] ICS01 socket: 78 reason# EndOfStream
2025-11-19T13:59:53.728719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435932901394351:2050] [node 3] ICP30 unregister session Session# [5:7574435932901394434:2050] VirtualId# [5:7574435932901394421:0]
2025-11-19T13:59:53.728724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435932901394434:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.728747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:53.728782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:53.728811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T13:59:53.728880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435931585418968:2050] [node 3] ICS07 socket disconnect 76 reason# EndOfStream
2025-11-19T13:59:53.728885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435931585418968:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.728910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:53.728936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435931585418968:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:53.750537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:53.750635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435932865986149:2089] [node 4] ICS01 socket: 111 reason# EndOfStream
2025-11-19T13:59:53.750644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435932865986112:2089] [node 4] ICP30 unregister session Session# [6:7574435932865986149:2089] VirtualId# [6:7574435932865986145:0]
2025-11-19T13:59:53.750652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435932865986149:2089] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:53.750705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:53.750732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T13:59:54.037273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435937029979555:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:54.037285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435937029979556:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:54.129932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435934806299562:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:54.129974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435934806299563:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:54.192592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435935811015170:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:54.192608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435935811015171:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:54.238082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435935417952166:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:54.238096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435935417952167:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:54.238101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435935417952168:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:54.271893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435937586151608:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:54.271911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435937586151609:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:54.271915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435937586151610:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:54.308489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435936059448600:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:54.308504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435936059448601:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:54.308509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435936059448602:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.372378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435935811015232:2106] [node 5] ICP01 ready to work
2025-11-19T13:59:54.372267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435936059448630:2074] [node 5] ICP01 ready to work
2025-11-19T13:59:54.372481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435935811015233:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:54.372190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435937029979618:2107] [node 5] ICP01 ready to work
2025-11-19T13:59:54.370625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435934806299622:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36684
2025-11-19T13:59:54.374883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435934806299624:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.375048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435937029979623:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:54.376836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435937029979625:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63576
2025-11-19T13:59:54.378352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435936059448647:2083] [node 4] ICP01 ready to work
2025-11-19T13:59:54.378100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435934806299630:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:54.378976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435934806299635:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:54.388334Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435937029979625:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.390793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435935811015232:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(18249316ns) {Configure:104(18248763ns) {ProcessConfigured:135(15869138ns) {StartInitialHandshake:157(15868068ns) {DropHandshakes:469(488ns) {DropIncomingHandshake:441(87ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:476(1403ns) {SwitchToState:178(1060ns)}}}}}
2025-11-19T13:59:54.391078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435935811015233:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18287322ns) {Configure:104(18277470ns) {ProcessConfigured:135(15983631ns) {StartInitialHandshake:157(15983109ns) {DropHandshakes:469(282ns) {DropIncomingHandshake:441(116ns) DropOutgoingHandshake:456(38ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(708ns) {SwitchToState:178(121ns)}}}}}
2025-11-19T13:59:54.391147Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435934806299624:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16176464ns) {Configure:104(16175814ns) {ProcessConfigured:135(15771166ns) {StartInitialHandshake:157(15770755ns) {DropHandshakes:469(476ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(159ns) PrepareNewSessionHandshake:476(1384ns) {SwitchToState:178(912ns)}}}}}
2025-11-19T13:59:54.395842Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934806299622:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.395893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435934806299562:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.403086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435937029979633:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63576
2025-11-19T13:59:54.403112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435937029979634:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63576
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.404174Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435937586151608:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24463403ns) {Configure:104(24462257ns) {ProcessConfigured:135(24114100ns) {StartInitialHandshake:157(24113807ns) {DropHandshakes:469(369ns) {DropIncomingHandshake:441(27ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:476(938ns) {SwitchToState:178(718ns)}}}}}
2025-11-19T13:59:54.403671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435934806299642:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36684
2025-11-19T13:59:54.403690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435934806299643:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36684
2025-11-19T13:59:54.403702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435934806299644:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36684
2025-11-19T13:59:54.404025Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435934806299563:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(29242916ns) {Configure:104(24243496ns) {ProcessConfigured:135(23953186ns) {StartInitialHandshake:157(23952071ns) {DropHandshakes:469(519ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(295ns) PrepareNewSessionHandshake:476(704ns) {SwitchToState:178(361ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.407488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435937029979635:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63576
2025-11-19T13:59:54.407508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435937029979636:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63576
2025-11-19T13:59:54.405032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435935417952166:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24818673ns) {Configure:104(24817447ns) {ProcessConfigured:135(24478656ns) {StartInitialHandshake:157(24478428ns) {DropHandshakes:469(478ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(663ns) {SwitchToState:178(405ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.412276Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435937029979615:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.412295Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435937029979615:2106] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:54.412302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435937029979627:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:54.412680Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435937029979634:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.409126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435935811015247:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11096
2025-11-19T13:59:54.409185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435935811015248:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11096
2025-11-19T13:59:54.409218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435935811015249:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11096
2025-11-19T13:59:54.413100Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435935811015248:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.413137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435935811015170:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:54.414341Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435935811015247:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.416003Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934806299643:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.415947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435935811015243:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:54.420188Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934806299644:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.420981Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435937029979635:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.421101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435936059448602:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(47640392ns) {Configure:104(47639155ns) {ProcessConfigured:135(47403470ns) {StartInitialHandshake:157(47403009ns) {DropHandshakes:469(202ns) {DropIncomingHandshake:441(34ns) DropOutgoingHandshake:456(23ns)} GenerateSessionVirtualId:988(122ns) PrepareNewSessionHandshake:476(1550ns) {SwitchToState:178(1255ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.421566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435935417952238:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39183
2025-11-19T13:59:54.423524Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435935417952238:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.423873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435935417952234:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:54.422449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435937586151676:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:54.422757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435937586151675:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:54.426686Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435937029979629:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.426702Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435937029979629:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:54.424333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435935417952235:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:54.428455Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435937029979629:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:54.428547Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435937029979615:2106] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.428721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435937029979555:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.428742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:54.428748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435937029979555:2048] [node 2] ICP22 created new session: [1:7574435937029979642:2048]
2025-11-19T13:59:54.428755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435937029979642:2048] [node 2] ICS09 handshake done sender: [1:7574435937029979615:2106] self: [1:7574435937029979614:0] peer: [2:7574435934806299641:0] socket: 60 qp: -1
2025-11-19T13:59:54.428761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435937029979642:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:54.428781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:54.429080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435937029979556:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.429091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:54.429096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435937029979556:2049] [node 3] ICP22 created new session: [1:7574435937029979644:2049]
2025-11-19T13:59:54.429101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435937029979644:2049] [node 3] ICS09 handshake done sender: [1:7574435937029979629:2112] self: [1:7574435937029979628:0] peer: [3:7574435935811015250:0] socket: 79 qp: -1
2025-11-19T13:59:54.429107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435937029979644:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:54.429119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.429643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435935417952241:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39183
2025-11-19T13:59:54.430730Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435937029979636:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.430835Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435935417952241:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.430863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435935417952166:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:54.430956Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435937029979641:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.430961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435937029979641:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.428775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435935811015253:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11096
2025-11-19T13:59:54.431798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435934806299648:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36684
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.432557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435936059448678:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43556
2025-11-19T13:59:54.431475Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435937029979623:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(56372763ns) {Configure:104(56372229ns) {ProcessConfigured:135(56030762ns) {StartInitialHandshake:157(56030167ns) {DropHandshakes:469(482ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:476(742ns) {SwitchToState:178(446ns)}}}}}
2025-11-19T13:59:54.431562Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435937029979641:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:54.431726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435937029979627:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.431738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:54.431759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435937029979627:2111] [node 4] ICP22 created new session: [1:7574435937029979651:2111]
2025-11-19T13:59:54.431765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435937029979651:2111] [node 4] ICS09 handshake done sender: [1:7574435937029979641:2118] self: [1:7574435937029979637:0] peer: [4:7574435935417952242:0] socket: 95 qp: -1
2025-11-19T13:59:54.431771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435937029979651:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:54.431789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:54.433087Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435934806299622:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:54.433256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435934806299562:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.433280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:54.433288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435934806299562:2048] [node 1] ICP22 created new session: [2:7574435934806299649:2048]
2025-11-19T13:59:54.433294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435934806299649:2048] [node 1] ICS09 handshake done sender: [2:7574435934806299622:2107] self: [2:7574435934806299641:0] peer: [1:7574435937029979614:0] socket: 61 qp: -1
2025-11-19T13:59:54.433301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435934806299649:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:54.433311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:54.431515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435935417952243:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39183
2025-11-19T13:59:54.435866Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435935811015248:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:54.435998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435935811015170:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.436013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:54.436020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435935811015170:2048] [node 1] ICP22 created new session: [3:7574435935811015254:2048]
2025-11-19T13:59:54.436026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435935811015254:2048] [node 1] ICS09 handshake done sender: [3:7574435935811015248:2115] self: [3:7574435935811015250:0] peer: [1:7574435937029979628:0] socket: 89 qp: -1
2025-11-19T13:59:54.436032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435935811015254:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:54.436039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.434129Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934806299642:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.434339Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435937029979633:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.434390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435937029979556:2049] [node 3] ICP18 handshake (actor [1:7574435937029979633:2114]) is too old
2025-11-19T13:59:54.435847Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435937029979649:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.435852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435937029979649:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:54.436578Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435937029979649:2119] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:54.434429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435936059448678:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.434468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435936059448600:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:54.436442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435936059448680:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43556
2025-11-19T13:59:54.436658Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435936059448678:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:54.436893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435937586151682:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55870
2025-11-19T13:59:54.437164Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435937586151682:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.437199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435937586151608:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:54.437170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435937029979623:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.437187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:54.437192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435937029979623:2109] [node 6] ICP22 created new session: [1:7574435937029979655:2109]
2025-11-19T13:59:54.437197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435937029979655:2109] [node 6] ICS09 handshake done sender: [1:7574435937029979649:2119] self: [1:7574435937029979624:0] peer: [6:7574435936059448679:0] socket: 100 qp: -1
2025-11-19T13:59:54.437202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435937029979655:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:54.437207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.440478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435937586151686:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55870
2025-11-19T13:59:54.440545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435937586151687:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55870
2025-11-19T13:59:54.440581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435937586151688:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55870
2025-11-19T13:59:54.440743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435937586151676:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18158742ns) {Configure:104(18158019ns) {ProcessConfigured:135(17791186ns) {StartInitialHandshake:157(17790802ns) {DropHandshakes:469(434ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(151ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:476(803ns) {SwitchToState:178(129ns)}}}}}
2025-11-19T13:59:54.440693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435935417952167:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(57998512ns) {Configure:104(57997912ns) {ProcessConfigured:135(57732765ns) {StartInitialHandshake:157(57732139ns) {DropHandshakes:469(340ns) {DropIncomingHandshake:441(95ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(1010ns) {SwitchToState:178(569ns)}}}}}
2025-11-19T13:59:54.440715Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435935417952241:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:54.441178Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435937586151686:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.441208Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435937586151687:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.441243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435937586151609:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:54.441429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435935417952166:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22776) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435935417952241:2113] held: no
2025-11-19T13:59:54.441514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435935417952166:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.441524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:54.441529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435935417952166:2048] [node 1] ICP22 created new session: [4:7574435935417952245:2048]
2025-11-19T13:59:54.441535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435935417952245:2048] [node 1] ICS09 handshake done sender: [4:7574435935417952241:2113] self: [4:7574435935417952242:0] peer: [1:7574435937029979637:0] socket: 96 qp: -1
2025-11-19T13:59:54.441541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435935417952245:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:54.441557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:54.441670Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435935417952234:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17667278ns) {Configure:104(17666853ns) {ProcessConfigured:135(17327743ns) {StartInitialHandshake:157(17327463ns) {DropHandshakes:469(435ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(286ns) PrepareNewSessionHandshake:476(574ns) {SwitchToState:178(149ns)}}}}}
2025-11-19T13:59:54.441737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435935417952167:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:54.441941Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435935417952235:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17515740ns) {Configure:104(17515552ns) {ProcessConfigured:135(17311751ns) {StartInitialHandshake:157(17311507ns) {DropHandshakes:469(261ns) {DropIncomingHandshake:441(27ns) DropOutgoingHandshake:456(22ns)} GenerateSessionVirtualId:988(36ns) PrepareNewSessionHandshake:476(909ns) {SwitchToState:178(624ns)}}}}}
2025-11-19T13:59:54.441569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435937586151675:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(18690078ns) {Configure:104(18688896ns) {ProcessConfigured:135(18402773ns) {StartInitialHandshake:157(18402463ns) {DropHandshakes:469(516ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(536ns) {SwitchToState:178(122ns)}}}}}
2025-11-19T13:59:54.442078Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435935811015249:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.442197Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435937586151688:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.442227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435937586151610:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:54.448311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435937029979555:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(19472144ns) {HandleHandshakeStatus:329(19471309ns) {DropIncomingHandshake:441(4178ns) DropOutgoingHandshake:456(223ns) SwitchToState:178(141ns) ProcessPendingSessionEvents:495(11405633ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11385060ns max# 11308425ns}}}
2025-11-19T13:59:54.449056Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435937029979620:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.449065Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435937029979620:2108] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.451987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435937586151694:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55870
2025-11-19T13:59:54.452176Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934806299632:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.452193Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435934806299632:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:54.452239Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934806299638:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.452247Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435934806299638:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:54.452390Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435935811015244:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.452405Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435935811015244:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:54.452578Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435937586151682:2113] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:54.452653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435937586151608:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.452669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:54.452674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435937586151608:2048] [node 1] ICP22 created new session: [5:7574435937586151695:2048]
2025-11-19T13:59:54.452680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435937586151695:2048] [node 1] ICS09 handshake done sender: [5:7574435937586151682:2113] self: [5:7574435937586151685:0] peer: [1:7574435937029979619:0] socket: 65 qp: -1
2025-11-19T13:59:54.452685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435937586151695:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:54.452695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:54.452288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435936059448600:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.452322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:54.452330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435936059448600:2048] [node 1] ICP22 created new session: [6:7574435936059448681:2048]
2025-11-19T13:59:54.452337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435936059448681:2048] [node 1] ICS09 handshake done sender: [6:7574435936059448678:2115] self: [6:7574435936059448679:0] peer: [1:7574435937029979624:0] socket: 101 qp: -1
2025-11-19T13:59:54.452343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435936059448681:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:54.452367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.453511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435934806299653:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36684
2025-11-19T13:59:54.452004Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435937029979620:2108] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:54.453795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435937029979618:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.453813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:54.453818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435937029979618:2107] [node 5] ICP22 created new session: [1:7574435937029979658:2107]
2025-11-19T13:59:54.453823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435937029979658:2107] [node 5] ICS09 handshake done sender: [1:7574435937029979620:2108] self: [1:7574435937029979619:0] peer: [5:7574435937586151685:0] socket: 63 qp: -1
2025-11-19T13:59:54.453828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435937029979658:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:54.453836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.456862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435937586151697:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55870
2025-11-19T13:59:54.456965Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435935811015244:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:54.457053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435935811015243:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(38713388ns) {Configure:104(38712494ns) {ProcessConfigured:135(38367770ns) {StartInitialHandshake:157(38366767ns) {DropHandshakes:469(584ns) {DropIncomingHandshake:441(91ns) DropOutgoingHandshake:456(209ns)} GenerateSessionVirtualId:988(301ns) PrepareNewSessionHandshake:476(1082ns) {SwitchToState:178(451ns)}}}}}
2025-11-19T13:59:54.457363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435935417952251:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39183
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.459883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435936059448683:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43556
2025-11-19T13:59:54.460585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435935417952252:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39183
2025-11-19T13:59:54.461833Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435937586151688:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:54.461340Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435936059448600:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(24192748ns) {HandleHandshakeStatus:329(24191805ns) {DropIncomingHandshake:441(947ns) DropOutgoingHandshake:456(9084ns) SwitchToState:178(261ns) ProcessPendingSessionEvents:495(3785751ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 3764060ns max# 3696243ns}}}
2025-11-19T13:59:54.461403Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435935417952238:2112] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:54.461998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435935811015232:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.462026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:54.462033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435935811015232:2106] [node 5] ICP22 created new session: [3:7574435935811015260:2106]
2025-11-19T13:59:54.462039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435935811015260:2106] [node 5] ICS09 handshake done sender: [3:7574435935811015244:2112] self: [3:7574435935811015234:0] peer: [5:7574435937586151693:0] socket: 80 qp: -1
2025-11-19T13:59:54.462045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435935811015260:2106] [node 5] ICS10 traffic start
2025-11-19T13:59:54.462052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.462429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435935417952167:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.462446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:54.462452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435935417952167:2049] [node 2] ICP22 created new session: [4:7574435935417952253:2049]
2025-11-19T13:59:54.462456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435935417952253:2049] [node 2] ICS09 handshake done sender: [4:7574435935417952238:2112] self: [4:7574435935417952248:0] peer: [2:7574435934806299631:0] socket: 76 qp: -1
2025-11-19T13:59:54.462464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435935417952253:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:54.462482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:54.461049Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435934806299632:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:54.461122Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435934806299638:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:54.461569Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934806299653:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.462540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435934806299635:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.462552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:54.462558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435934806299635:2112] [node 5] ICP22 created new session: [2:7574435934806299657:2112]
2025-11-19T13:59:54.462563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435934806299657:2112] [node 5] ICS09 handshake done sender: [2:7574435934806299638:2113] self: [2:7574435934806299636:0] peer: [5:7574435937586151690:0] socket: 78 qp: -1
2025-11-19T13:59:54.462567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435934806299657:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:54.462572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.463482Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435936059448683:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.463989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435936059448601:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:54.464026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435936059448685:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43556
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.465186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435936059448687:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43556
2025-11-19T13:59:54.463049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435937586151610:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.463071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:54.463078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435937586151610:2050] [node 3] ICP22 created new session: [5:7574435937586151699:2050]
2025-11-19T13:59:54.463084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435937586151699:2050] [node 3] ICS09 handshake done sender: [5:7574435937586151688:2116] self: [5:7574435937586151693:0] peer: [3:7574435935811015234:0] socket: 104 qp: -1
2025-11-19T13:59:54.463091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435937586151699:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:54.463097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:54.465180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435934806299630:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.465200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:54.465206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435934806299630:2110] [node 4] ICP22 created new session: [2:7574435934806299659:2110]
2025-11-19T13:59:54.465212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435934806299659:2110] [node 4] ICS09 handshake done sender: [2:7574435934806299632:2111] self: [2:7574435934806299631:0] peer: [4:7574435935417952248:0] socket: 70 qp: -1
2025-11-19T13:59:54.465217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435934806299659:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:54.465233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:54.466879Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934806299640:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.466884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435934806299640:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:54.468229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435937586151698:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55870
2025-11-19T13:59:54.469170Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435937586151687:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.469341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435937586151609:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.469357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:54.469362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435937586151609:2049] [node 2] ICP22 created new session: [5:7574435937586151703:2049]
2025-11-19T13:59:54.469367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435937586151703:2049] [node 2] ICS09 handshake done sender: [5:7574435937586151687:2115] self: [5:7574435937586151690:0] peer: [2:7574435934806299636:0] socket: 103 qp: -1
2025-11-19T13:59:54.469374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435937586151703:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:54.469379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:54.469588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435937586151701:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55870
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.472586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435935417952257:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39183
2025-11-19T13:59:54.472608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435935417952258:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39183
2025-11-19T13:59:54.471707Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435936059448685:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.471826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435936059448602:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.473464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435935811015263:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11096
2025-11-19T13:59:54.473980Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435935417952251:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.474019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435935417952168:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:54.474143Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435935811015246:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.474148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435935811015246:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:54.474778Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435935811015263:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.474913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435935811015171:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:54.475210Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435934806299645:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.475224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435934806299645:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:54.475961Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435934806299645:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:54.476295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435934806299563:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.476316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:54.476323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435934806299563:2049] [node 3] ICP22 created new session: [2:7574435934806299663:2049]
2025-11-19T13:59:54.476330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435934806299663:2049] [node 3] ICS09 handshake done sender: [2:7574435934806299645:2118] self: [2:7574435934806299637:0] peer: [3:7574435935811015264:0] socket: 83 qp: -1
2025-11-19T13:59:54.476337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435934806299663:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:54.476344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.476682Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435936059448687:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.476724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435936059448647:2083] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:54.478573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435936059448690:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43556
2025-11-19T13:59:54.478656Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435936059448687:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:54.478704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435936059448647:2083] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.478714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:54.478720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435936059448647:2083] [node 4] ICP22 created new session: [6:7574435936059448691:2083]
2025-11-19T13:59:54.478725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435936059448691:2083] [node 4] ICS09 handshake done sender: [6:7574435936059448687:2119] self: [6:7574435936059448689:0] peer: [4:7574435935417952240:0] socket: 119 qp: -1
2025-11-19T13:59:54.478731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435936059448691:2083] [node 4] ICS10 traffic start
2025-11-19T13:59:54.478737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:54.476851Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435935417952250:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.476861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435935417952250:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:54.477974Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435935417952250:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:54.478088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435935417952235:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.478098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:54.478104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435935417952235:2110] [node 6] ICP22 created new session: [4:7574435935417952261:2110]
2025-11-19T13:59:54.478110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435935417952261:2110] [node 6] ICS09 handshake done sender: [4:7574435935417952250:2117] self: [4:7574435935417952240:0] peer: [6:7574435936059448689:0] socket: 118 qp: -1
2025-11-19T13:59:54.478116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435935417952261:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:54.478123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.483072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435935811015265:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11096
2025-11-19T13:59:54.488713Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435935811015263:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:54.488962Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435935811015259:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.488972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435935811015259:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:54.489370Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435934806299640:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:54.489847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435934806299624:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.489874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:54.489884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435934806299624:2108] [node 6] ICP22 created new session: [2:7574435934806299666:2108]
2025-11-19T13:59:54.489893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435934806299666:2108] [node 6] ICS09 handshake done sender: [2:7574435934806299640:2114] self: [2:7574435934806299625:0] peer: [6:7574435936059448686:0] socket: 107 qp: -1
2025-11-19T13:59:54.489902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435934806299666:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:54.489913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:54.492087Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435935417952258:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.492384Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435935417952167:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(29855122ns) {HandleHandshakeStatus:329(29854440ns) {DropIncomingHandshake:441(184ns) DropOutgoingHandshake:456(1908ns) SwitchToState:178(80ns) ProcessPendingSessionEvents:495(24080172ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24054760ns max# 23978762ns}}}
2025-11-19T13:59:54.492348Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435937586151701:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.492388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435937586151676:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:54.492500Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435935417952247:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.492506Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435935417952247:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:54.494913Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435935417952247:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:54.495010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435935417952234:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.495034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:54.495041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435935417952234:2109] [node 5] ICP22 created new session: [4:7574435935417952265:2109]
2025-11-19T13:59:54.495048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435935417952265:2109] [node 5] ICS09 handshake done sender: [4:7574435935417952247:2116] self: [4:7574435935417952239:0] peer: [5:7574435937586151705:0] socket: 86 qp: -1
2025-11-19T13:59:54.495053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435935417952265:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:54.495085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:54.496883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435935811015171:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.496907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:54.496914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435935811015171:2049] [node 2] ICP22 created new session: [3:7574435935811015266:2049]
2025-11-19T13:59:54.496921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435935811015266:2049] [node 2] ICS09 handshake done sender: [3:7574435935811015263:2119] self: [3:7574435935811015264:0] peer: [2:7574435934806299637:0] socket: 105 qp: -1
2025-11-19T13:59:54.496929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435935811015266:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:54.496938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.500599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435935417952267:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39183
2025-11-19T13:59:54.500499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435936059448693:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43556
2025-11-19T13:59:54.500608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435936059448694:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43556
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.501490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435937586151706:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55870
2025-11-19T13:59:54.501708Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435935811015246:2113] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:54.501964Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435936059448685:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:54.501803Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435935811015259:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:54.502109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435935811015243:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.502124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:54.502131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435935811015243:2111] [node 6] ICP22 created new session: [3:7574435935811015270:2111]
2025-11-19T13:59:54.502137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435935811015270:2111] [node 6] ICS09 handshake done sender: [3:7574435935811015259:2118] self: [3:7574435935811015252:0] peer: [6:7574435936059448688:0] socket: 116 qp: -1
2025-11-19T13:59:54.502143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435935811015270:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:54.502160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:54.502612Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435936059448683:2117] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:54.502766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435935811015233:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.502793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:54.502798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435935811015233:2108] [node 4] ICP22 created new session: [3:7574435935811015272:2108]
2025-11-19T13:59:54.502804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435935811015272:2108] [node 4] ICS09 handshake done sender: [3:7574435935811015246:2113] self: [3:7574435935811015235:0] peer: [4:7574435935417952259:0] socket: 108 qp: -1
2025-11-19T13:59:54.502809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435935811015272:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:54.502815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:54.503176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435936059448602:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.503192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:54.503196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435936059448602:2050] [node 3] ICP22 created new session: [6:7574435936059448695:2050]
2025-11-19T13:59:54.503201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435936059448695:2050] [node 3] ICS09 handshake done sender: [6:7574435936059448685:2118] self: [6:7574435936059448688:0] peer: [3:7574435935811015252:0] socket: 117 qp: -1
2025-11-19T13:59:54.503206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435936059448695:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:54.503212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:54.504323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435936059448601:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.504344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:54.504350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435936059448601:2049] [node 2] ICP22 created new session: [6:7574435936059448697:2049]
2025-11-19T13:59:54.504356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435936059448697:2049] [node 2] ICS09 handshake done sender: [6:7574435936059448683:2117] self: [6:7574435936059448686:0] peer: [2:7574435934806299625:0] socket: 111 qp: -1
2025-11-19T13:59:54.504363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435936059448697:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:54.504384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:54.504454Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435935417952251:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:54.504509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435935417952168:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.504522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:54.504528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435935417952168:2050] [node 3] ICP22 created new session: [4:7574435935417952268:2050]
2025-11-19T13:59:54.504534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435935417952268:2050] [node 3] ICS09 handshake done sender: [4:7574435935417952251:2118] self: [4:7574435935417952259:0] peer: [3:7574435935811015235:0] socket: 110 qp: -1
2025-11-19T13:59:54.504539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435935417952268:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:54.504546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:54.504705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435937586151610:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(34730217ns) {HandleHandshakeStatus:329(34729664ns) {DropIncomingHandshake:441(186ns) DropOutgoingHandshake:456(1886ns) SwitchToState:178(156ns) ProcessPendingSessionEvents:495(29647272ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29631378ns max# 29566474ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.505621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435936059448699:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43556
2025-11-19T13:59:54.516986Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435937586151701:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:54.517276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435937586151676:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3993) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435937586151701:2122] held: no
2025-11-19T13:59:54.524507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435935417952235:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:54.528086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435937586151676:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.528113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:54.528122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435937586151676:2109] [node 4] ICP22 created new session: [5:7574435937586151708:2109]
2025-11-19T13:59:54.528129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435937586151708:2109] [node 4] ICS09 handshake done sender: [5:7574435937586151701:2122] self: [5:7574435937586151705:0] peer: [4:7574435935417952239:0] socket: 73 qp: -1
2025-11-19T13:59:54.528137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435937586151708:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:54.528146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:54.530908Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435936059448699:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.530968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435936059448630:2074] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:54.531197Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435937586151691:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:54.531207Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435937586151691:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:54.531867Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435937586151691:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:54.532049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435937586151675:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:54.532072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:54.532079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435937586151675:2110] [node 6] ICP22 created new session: [5:7574435937586151711:2110]
2025-11-19T13:59:54.532085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435937586151711:2110] [node 6] ICS09 handshake done sender: [5:7574435937586151691:2118] self: [5:7574435937586151684:0] peer: [6:7574435936059448700:0] socket: 87 qp: -1
2025-11-19T13:59:54.532092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435937586151711:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:54.532100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:54.532767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435936059448701:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43556
2025-11-19T13:59:54.533049Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435936059448699:2123] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:54.533184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435936059448630:2074] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:54.533202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:54.533209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435936059448630:2074] [node 5] ICP22 created new session: [6:7574435936059448702:2074]
2025-11-19T13:59:54.533214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435936059448702:2074] [node 5] ICS09 handshake done sender: [6:7574435936059448699:2123] self: [6:7574435936059448700:0] peer: [5:7574435937586151684:0] socket: 130 qp: -1
2025-11-19T13:59:54.533221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435936059448702:2074] [node 5] ICS10 traffic start
2025-11-19T13:59:54.533232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:54.546671Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:54.546773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435934806299649:2048] [node 1] ICS01 socket: 61 reason# EndOfStream
2025-11-19T13:59:54.546781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435934806299562:2048] [node 1] ICP30 unregister session Session# [2:7574435934806299649:2048] VirtualId# [2:7574435934806299641:0]
2025-11-19T13:59:54.546788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435934806299649:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:54.546821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:54.546840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:54.547739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:54.547877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435935811015254:2048] [node 1] ICS01 socket: 89 reason# EndOfStream
2025-11-19T13:59:54.547883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435935811015170:2048] [node 1] ICP30 unregister session Session# [3:7574435935811015254:2048] VirtualId# [3:7574435935811015250:0]
2025-11-19T13:59:54.547889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435935811015254:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:54.547919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:54.547937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:54.565933Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:54.566032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435937586151695:2048] [node 1] ICS07 socket disconnect 65 reason# EndOfStream
2025-11-19T13:59:54.566040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435937586151695:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:54.566095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:54.566117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435937586151695:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:54.566338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:54.566405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435936059448681:2048] [node 1] ICS07 socket disconnect 101 reason# EndOfStream
2025-11-19T13:59:54.566410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435936059448681:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:54.566438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:54.566457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435936059448681:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:54.566591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:54.566654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435935417952245:2048] [node 1] ICS01 socket: 96 reason# EndOfStream
2025-11-19T13:59:54.566662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435935417952166:2048] [node 1] ICP30 unregister session Session# [4:7574435935417952245:2048] VirtualId# [4:7574435935417952242:0]
2025-11-19T13:59:54.566668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435935417952245:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:54.566695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:54.566716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:54.571893Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:54.571983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435935811015266:2049] [node 2] ICS01 socket: 105 reason# EndOfStream
2025-11-19T13:59:54.571992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435935811015171:2049] [node 2] ICP30 unregister session Session# [3:7574435935811015266:2049] VirtualId# [3:7574435935811015264:0]
2025-11-19T13:59:54.571998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435935811015266:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:54.572028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:54.572043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:54.576563Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435936059448600:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22776) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:54.576583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435936059448681:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:54.576588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435936059448600:2048] [node 1] ICP30 unregister session Session# [6:7574435936059448681:2048] VirtualId# [6:7574435936059448679:0]
2025-11-19T13:59:54.576627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:54.576634Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435936059448600:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:54.576638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:54.578263Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435937586151608:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22776) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:54.578281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435937586151695:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:54.578286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435937586151608:2048] [node 1] ICP30 unregister session Session# [5:7574435937586151695:2048] VirtualId# [5:7574435937586151685:0]
2025-11-19T13:59:54.578325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:54.578329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435937586151608:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:54.578332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:54.580854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:54.580947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435936059448697:2049] [node 2] ICS07 socket disconnect 111 reason# EndOfStream
2025-11-19T13:59:54.580955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435936059448697:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:54.580993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:54.581024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435936059448697:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:54.581478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:54.581549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435937586151703:2049] [node 2] ICS01 socket: 103 reason# EndOfStream
2025-11-19T13:59:54.581555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435937586151609:2049] [node 2] ICP30 unregister session Session# [5:7574435937586151703:2049] VirtualId# [5:7574435937586151690:0]
2025-11-19T13:59:54.581561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435937586151703:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:54.581589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:54.581612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:54.581882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435936059448601:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19599) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:54.581893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435936059448697:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:54.581897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435936059448601:2049] [node 2] ICP30 unregister session Session# [6:7574435936059448697:2049] VirtualId# [6:7574435936059448686:0]
2025-11-19T13:59:54.581921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:54.581926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435936059448601:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19599) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:54.581928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19599) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:54.582715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:54.582792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435935417952253:2049] [node 2] ICS07 socket disconnect 76 reason# EndOfStream
2025-11-19T13:59:54.582798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435935417952253:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:54.582850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:54.582875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435935417952253:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:54.583222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435935417952167:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19599) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:54.583237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435935417952253:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:54.583242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435935417952167:2049] [node 2] ICP30 unregister session Session# [4:7574435935417952253:2049] VirtualId# [4:7574435935417952248:0]
2025-11-19T13:59:54.583275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:54.583280Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435935417952167:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19599) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:54.583282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19599) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:54.588233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435937586151608:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:54.591652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435936059448600:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:54.593723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435935417952167:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:54.594235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435936059448601:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:54.600152Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:54.600250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435937586151699:2050] [node 3] ICS07 socket disconnect 104 reason# EndOfStream
2025-11-19T13:59:54.600257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435937586151699:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:54.600320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:54.600345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435937586151699:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:54.600929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435937586151610:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22571) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:54.600946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435937586151699:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:54.600951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435937586151610:2050] [node 3] ICP30 unregister session Session# [5:7574435937586151699:2050] VirtualId# [5:7574435937586151693:0]
2025-11-19T13:59:54.600985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:54.600991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435937586151610:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22571) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:54.600996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22571) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:54.606041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:54.606142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435936059448695:2050] [node 3] ICS01 socket: 117 reason# EndOfStream
2025-11-19T13:59:54.606152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435936059448602:2050] [node 3] ICP30 unregister session Session# [6:7574435936059448695:2050] VirtualId# [6:7574435936059448688:0]
2025-11-19T13:59:54.606160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435936059448695:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:54.606225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:54.606254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:54.612245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435937586151610:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:59:54.619048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:54.619156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435936059448691:2083] [node 4] ICS01 socket: 119 reason# EndOfStream
2025-11-19T13:59:54.619167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435936059448647:2083] [node 4] ICP30 unregister session Session# [6:7574435936059448691:2083] VirtualId# [6:7574435936059448689:0]
2025-11-19T13:59:54.619175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435936059448691:2083] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:54.619242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:54.619266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T13:59:54.719950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435934900347035:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:54.719965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435934900347036:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:54.772666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435935610883227:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:54.772688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435935610883228:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:54.808472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435935332410722:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:54.808495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435935332410723:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:54.831957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435935723422944:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:54.831969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435935723422945:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:54.831973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435935723422946:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:54.907188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435938873698277:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:54.907205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435938873698278:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:54.907210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435938873698279:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:54.971692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435935651447941:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:54.971722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435935651447942:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:54.971727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435935651447943:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:55.009538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435939946415281:2090] [node 4] ICP01 ready to work
2025-11-19T13:59:55.009861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435939195314393:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:55.009962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435939195314394:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.011543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435943168665634:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54054
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.012097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435940018390301:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49212
2025-11-19T13:59:55.012118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435940018390302:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49212
2025-11-19T13:59:55.011645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435939905850583:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30989
2025-11-19T13:59:55.024261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435939946415293:2097] [node 5] ICP01 ready to work
2025-11-19T13:59:55.025325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435940018390301:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.025478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435940018390304:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:55.025872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435940018390304:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.026389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435943168665635:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54054
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.028137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435939627378079:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:55.028764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435939627378084:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:55.028804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435939627378080:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13598
2025-11-19T13:59:55.032317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435939905850586:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:55.032424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435939905850587:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.033289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435943168665636:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54054
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.033496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435940018390306:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49212
2025-11-19T13:59:55.033426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435939627378093:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:55.033643Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435939946415283:2091] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.033656Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435939946415283:2091] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.038071Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435943168665635:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.038239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435943168665638:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:55.038677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435943168665638:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.039137Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435939946415295:2100] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.039194Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435939946415295:2100] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.039757Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435939946415283:2091] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:55.039852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435939946415281:2090] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.039877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:55.039920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435939946415281:2090] [node 4] ICP22 created new session: [6:7574435939946415308:2090]
2025-11-19T13:59:55.039935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435939946415308:2090] [node 4] ICS09 handshake done sender: [6:7574435939946415283:2091] self: [6:7574435939946415282:0] peer: [4:7574435940018390305:0] socket: 66 qp: -1
2025-11-19T13:59:55.039951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435939946415308:2090] [node 4] ICS10 traffic start
2025-11-19T13:59:55.039968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:55.044271Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435943168665636:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.044661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435938873698279:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.049867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435939195314407:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:55.052207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435940018390307:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49212
2025-11-19T13:59:55.053198Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435943168665634:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.055779Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435939627378086:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.055805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435939627378086:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.056415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435939905850586:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(23928211ns) {Configure:104(23926971ns) {ProcessConfigured:135(23551754ns) {StartInitialHandshake:157(23550582ns) {DropHandshakes:469(731ns) {DropIncomingHandshake:441(94ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(260ns) PrepareNewSessionHandshake:476(1670ns) {SwitchToState:178(184ns)}}}}}
2025-11-19T13:59:55.068481Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435939627378080:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.069044Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435939627378086:2111] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:55.072642Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435935651447941:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16334926ns) {Configure:104(16332696ns) {ProcessConfigured:135(15929005ns) {StartInitialHandshake:157(15928239ns) {DropHandshakes:469(874ns) {DropIncomingHandshake:441(152ns) DropOutgoingHandshake:456(112ns)} GenerateSessionVirtualId:988(376ns) PrepareNewSessionHandshake:476(1368ns) {SwitchToState:178(900ns)}}}}}
2025-11-19T13:59:55.072749Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435939946415295:2100] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:55.075123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435939946415293:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.075160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:55.075170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435939946415293:2097] [node 5] ICP22 created new session: [6:7574435939946415317:2097]
2025-11-19T13:59:55.075177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435939946415317:2097] [node 5] ICS09 handshake done sender: [6:7574435939946415295:2100] self: [6:7574435939946415294:0] peer: [5:7574435943168665639:0] socket: 74 qp: -1
2025-11-19T13:59:55.075187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435939946415317:2097] [node 5] ICS10 traffic start
2025-11-19T13:59:55.075205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:55.075954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435939905850587:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19476593ns) {Configure:104(19474579ns) {ProcessConfigured:135(19116272ns) {StartInitialHandshake:157(19115560ns) {DropHandshakes:469(712ns) {DropIncomingHandshake:441(131ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(644ns) PrepareNewSessionHandshake:476(1289ns) {SwitchToState:178(832ns)}}}}}
2025-11-19T13:59:55.076270Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435940018390306:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.076581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435940018390309:2113] [node 5] ICP01 ready to work
2025-11-19T13:59:55.076725Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435940018390302:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.076810Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435940018390301:2108] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:55.081637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435939627378084:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.081678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:55.081695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435939627378084:2109] [node 5] ICP22 created new session: [3:7574435939627378100:2109]
2025-11-19T13:59:55.081708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435939627378100:2109] [node 5] ICS09 handshake done sender: [3:7574435939627378086:2111] self: [3:7574435939627378085:0] peer: [5:7574435943168665640:0] socket: 79 qp: -1
2025-11-19T13:59:55.081722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435939627378100:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:55.081739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:55.086559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435939905850596:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:55.087799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435940018390304:2110] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.087822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:55.087830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435940018390304:2110] [node 6] ICP22 created new session: [4:7574435940018390310:2110]
2025-11-19T13:59:55.087837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435940018390310:2110] [node 6] ICS09 handshake done sender: [4:7574435940018390301:2108] self: [4:7574435940018390305:0] peer: [6:7574435939946415282:0] socket: 67 qp: -1
2025-11-19T13:59:55.087847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435940018390310:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:55.087869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:55.088592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435935723422946:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.089393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435935723422944:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.092331Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435939627378083:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.092347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435939627378083:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.093332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435939627378102:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13598
2025-11-19T13:59:55.095364Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435939195314396:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.095382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435939195314396:2109] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.096644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435943168665648:2113] [node 4] ICP01 ready to work
2025-11-19T13:59:55.098450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435938873698277:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.098810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435943168665645:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54054
2025-11-19T13:59:55.098984Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435939627378083:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:55.096181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435939195314410:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6003
2025-11-19T13:59:55.096229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435939195314411:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6003
2025-11-19T13:59:55.099459Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435939905850583:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.099490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435939905850593:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30989
2025-11-19T13:59:55.099524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435939905850594:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30989
2025-11-19T13:59:55.099555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435939905850595:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30989
2025-11-19T13:59:55.099732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435935610883227:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.099767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435939905850603:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30989
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.106692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435939946415323:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50731
2025-11-19T13:59:55.106715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435939946415324:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50731
2025-11-19T13:59:55.107218Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435939195314396:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:55.104505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435939627378079:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.104529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:55.104538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435939627378079:2107] [node 4] ICP22 created new session: [3:7574435939627378105:2107]
2025-11-19T13:59:55.104544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435939627378105:2107] [node 4] ICS09 handshake done sender: [3:7574435939627378083:2110] self: [3:7574435939627378082:0] peer: [4:7574435940018390314:0] socket: 78 qp: -1
2025-11-19T13:59:55.104550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435939627378105:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:55.104561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:55.106399Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435939905850593:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.106433Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435939905850595:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.112316Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435939195314398:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.112333Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435939195314398:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.112991Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435939195314400:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.112996Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435939195314400:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.113873Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435939195314410:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.113885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435939195314393:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.113899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:55.113909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435939195314393:2107] [node 4] ICP22 created new session: [1:7574435939195314414:2107]
2025-11-19T13:59:55.113916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435939195314414:2107] [node 4] ICS09 handshake done sender: [1:7574435939195314396:2109] self: [1:7574435939195314395:0] peer: [4:7574435940018390321:0] socket: 68 qp: -1
2025-11-19T13:59:55.113923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435939195314414:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:55.113934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:55.114412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435939195314413:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6003
2025-11-19T13:59:55.116321Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435939195314398:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.116528Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435939195314400:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.116927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435939627378108:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13598
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.116424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435943168665646:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54054
2025-11-19T13:59:55.117327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435943168665647:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54054
2025-11-19T13:59:55.117497Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435943168665635:2109] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:55.116759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435939195314394:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.116777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:55.116785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435939195314394:2108] [node 5] ICP22 created new session: [1:7574435939195314420:2108]
2025-11-19T13:59:55.116792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435939195314420:2108] [node 5] ICS09 handshake done sender: [1:7574435939195314398:2110] self: [1:7574435939195314397:0] peer: [5:7574435943168665651:0] socket: 69 qp: -1
2025-11-19T13:59:55.116799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435939195314420:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:55.116809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:55.117047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435934900347035:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.117054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:55.117059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435934900347035:2048] [node 2] ICP22 created new session: [1:7574435939195314422:2048]
2025-11-19T13:59:55.117064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435939195314422:2048] [node 2] ICS09 handshake done sender: [1:7574435939195314400:2111] self: [1:7574435939195314399:0] peer: [2:7574435939905850606:0] socket: 70 qp: -1
2025-11-19T13:59:55.117069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435939195314422:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:55.117075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:55.119533Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435943168665636:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:55.119759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435943168665638:2111] [node 6] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.119773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:55.119779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435943168665638:2111] [node 6] ICP22 created new session: [5:7574435943168665657:2111]
2025-11-19T13:59:55.119785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435943168665657:2111] [node 6] ICS09 handshake done sender: [5:7574435943168665635:2109] self: [5:7574435943168665639:0] peer: [6:7574435939946415294:0] socket: 76 qp: -1
2025-11-19T13:59:55.119791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435943168665657:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:55.119799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:55.120182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435938873698279:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.120190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:55.120198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435938873698279:2050] [node 3] ICP22 created new session: [5:7574435943168665659:2050]
2025-11-19T13:59:55.120202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435943168665659:2050] [node 3] ICS09 handshake done sender: [5:7574435943168665636:2110] self: [5:7574435943168665640:0] peer: [3:7574435939627378085:0] socket: 81 qp: -1
2025-11-19T13:59:55.120206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435943168665659:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:55.120212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.122892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435940018390325:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49212
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.123100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435940018390326:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49212
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.123235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435940018390327:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49212
2025-11-19T13:59:55.123291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435940018390328:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49212
2025-11-19T13:59:55.123299Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435940018390306:2111] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:55.123393Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435940018390302:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:55.123452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435935723422946:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.123466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:55.123470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435935723422946:2050] [node 3] ICP22 created new session: [4:7574435940018390329:2050]
2025-11-19T13:59:55.123475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435940018390329:2050] [node 3] ICS09 handshake done sender: [4:7574435940018390306:2111] self: [4:7574435940018390314:0] peer: [3:7574435939627378082:0] socket: 82 qp: -1
2025-11-19T13:59:55.123480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435940018390329:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:55.123486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:55.123812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435935723422944:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.123820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:55.123824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435935723422944:2048] [node 1] ICP22 created new session: [4:7574435940018390331:2048]
2025-11-19T13:59:55.123828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435940018390331:2048] [node 1] ICS09 handshake done sender: [4:7574435940018390302:2109] self: [4:7574435940018390321:0] peer: [1:7574435939195314395:0] socket: 73 qp: -1
2025-11-19T13:59:55.123833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435940018390331:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:55.123840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:55.128542Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435939195314411:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.139328Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435939905850594:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.139359Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435939905850603:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.139460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435939905850610:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30989
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.139860Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435939627378102:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.140116Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435939627378108:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.140257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435939627378079:2107] [node 4] ICP18 handshake (actor [3:7574435939627378102:2116]) is too old
2025-11-19T13:59:55.140270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435935332410722:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.139733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435943168665654:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54054
2025-11-19T13:59:55.139858Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435939946415324:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.139896Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435939946415323:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.140002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435939946415325:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50731
2025-11-19T13:59:55.140033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435935651447943:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.140046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435935651447942:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.140705Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435939627378097:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.140713Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435939627378097:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.142331Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435939195314405:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.142347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435939195314405:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.143313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435939195314425:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6003
2025-11-19T13:59:55.143771Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435940018390327:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.144049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435935723422945:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.144269Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435939627378097:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:55.144466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435939627378093:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.144491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:55.144498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435939627378093:2113] [node 6] ICP22 created new session: [3:7574435939627378112:2113]
2025-11-19T13:59:55.144504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435939627378112:2113] [node 6] ICS09 handshake done sender: [3:7574435939627378097:2115] self: [3:7574435939627378096:0] peer: [6:7574435939946415326:0] socket: 112 qp: -1
2025-11-19T13:59:55.144510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435939627378112:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:55.144519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:55.145130Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435939905850592:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.145144Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435939905850592:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.146039Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435939905850590:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.146043Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435939905850590:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.146698Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435939905850583:2107] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.147967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435939627378111:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13598
2025-11-19T13:59:55.148099Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435939195314405:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:55.148273Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435939627378108:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:55.148330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435935332410722:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.148345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:55.148350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435935332410722:2048] [node 1] ICP22 created new session: [3:7574435939627378114:2048]
2025-11-19T13:59:55.148355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435939627378114:2048] [node 1] ICS09 handshake done sender: [3:7574435939627378108:2117] self: [3:7574435939627378109:0] peer: [1:7574435939195314404:0] socket: 119 qp: -1
2025-11-19T13:59:55.148361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435939627378114:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:55.148368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:55.149428Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435939195314413:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.149441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435934900347036:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.149455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:55.149460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435934900347036:2049] [node 3] ICP22 created new session: [1:7574435939195314428:2049]
2025-11-19T13:59:55.149465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435939195314428:2049] [node 3] ICS09 handshake done sender: [1:7574435939195314405:2112] self: [1:7574435939195314404:0] peer: [3:7574435939627378109:0] socket: 118 qp: -1
2025-11-19T13:59:55.149469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435939195314428:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:55.149475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:55.153771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435935651447943:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7733) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435939946415324:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.156242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435934900347035:2048] [node 2] ICP18 handshake (actor [1:7574435939195314413:2117]) is too old
2025-11-19T13:59:55.156487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435939946415328:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50731
2025-11-19T13:59:55.156567Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435943168665647:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.156677Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435943168665634:2108] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:55.157070Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435939946415325:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.157094Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435939946415324:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:55.157199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435935651447943:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.157217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:55.157223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435935651447943:2050] [node 3] ICP22 created new session: [6:7574435939946415329:2050]
2025-11-19T13:59:55.157228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435939946415329:2050] [node 3] ICS09 handshake done sender: [6:7574435939946415324:2116] self: [6:7574435939946415326:0] peer: [3:7574435939627378096:0] socket: 114 qp: -1
2025-11-19T13:59:55.157235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435939946415329:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:55.157243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:55.157450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435935651447941:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.157908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435935610883227:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.157932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:55.157939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435935610883227:2048] [node 1] ICP22 created new session: [2:7574435939905850615:2048]
2025-11-19T13:59:55.157945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435939905850615:2048] [node 1] ICS09 handshake done sender: [2:7574435939905850583:2107] self: [2:7574435939905850606:0] peer: [1:7574435939195314399:0] socket: 72 qp: -1
2025-11-19T13:59:55.157951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435939905850615:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:55.157958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.159438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435940018390334:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49212
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.159551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435939946415332:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50731
2025-11-19T13:59:55.159939Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435939195314409:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.159949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435939195314409:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.160954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435939946415333:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50731
2025-11-19T13:59:55.161289Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435939195314409:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:55.161507Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435939946415325:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:55.161582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435935651447941:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.161598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:55.161604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435935651447941:2048] [node 1] ICP22 created new session: [6:7574435939946415334:2048]
2025-11-19T13:59:55.161611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435939946415334:2048] [node 1] ICS09 handshake done sender: [6:7574435939946415325:2117] self: [6:7574435939946415331:0] peer: [1:7574435939195314408:0] socket: 124 qp: -1
2025-11-19T13:59:55.161617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435939946415334:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:55.161624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:55.163020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435939195314407:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.163039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:55.163044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435939195314407:2113] [node 6] ICP22 created new session: [1:7574435939195314431:2113]
2025-11-19T13:59:55.163049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435939195314431:2113] [node 6] ICS09 handshake done sender: [1:7574435939195314409:2114] self: [1:7574435939195314408:0] peer: [6:7574435939946415331:0] socket: 121 qp: -1
2025-11-19T13:59:55.163053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435939195314431:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:55.163061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:55.163797Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435939905850592:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:55.164170Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435939946415323:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:55.164308Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435939905850590:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:55.164492Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435940018390327:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:55.164568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435935723422945:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.164582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:55.164587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435935723422945:2049] [node 2] ICP22 created new session: [4:7574435940018390335:2049]
2025-11-19T13:59:55.164592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435940018390335:2049] [node 2] ICS09 handshake done sender: [4:7574435940018390327:2120] self: [4:7574435940018390333:0] peer: [2:7574435939905850588:0] socket: 111 qp: -1
2025-11-19T13:59:55.164597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435940018390335:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:55.164605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:55.165731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435935651447942:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.165746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:55.165752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435935651447942:2049] [node 2] ICP22 created new session: [6:7574435939946415336:2049]
2025-11-19T13:59:55.165757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435939946415336:2049] [node 2] ICS09 handshake done sender: [6:7574435939946415323:2115] self: [6:7574435939946415327:0] peer: [2:7574435939905850591:0] socket: 113 qp: -1
2025-11-19T13:59:55.165762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435939946415336:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:55.165767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:55.166865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435939905850587:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.166884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:55.166889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435939905850587:2109] [node 6] ICP22 created new session: [2:7574435939905850617:2109]
2025-11-19T13:59:55.166894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435939905850617:2109] [node 6] ICS09 handshake done sender: [2:7574435939905850592:2112] self: [2:7574435939905850591:0] peer: [6:7574435939946415327:0] socket: 107 qp: -1
2025-11-19T13:59:55.166898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435939905850617:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:55.166904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:55.167191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435939905850586:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.167199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:55.167202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435939905850586:2108] [node 4] ICP22 created new session: [2:7574435939905850619:2108]
2025-11-19T13:59:55.167206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435939905850619:2108] [node 4] ICS09 handshake done sender: [2:7574435939905850590:2111] self: [2:7574435939905850588:0] peer: [4:7574435940018390333:0] socket: 106 qp: -1
2025-11-19T13:59:55.167209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435939905850619:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:55.167214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:55.169574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435938873698277:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.169594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:55.169602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435938873698277:2048] [node 1] ICP22 created new session: [5:7574435943168665661:2048]
2025-11-19T13:59:55.169609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435943168665661:2048] [node 1] ICS09 handshake done sender: [5:7574435943168665634:2108] self: [5:7574435943168665651:0] peer: [1:7574435939195314397:0] socket: 71 qp: -1
2025-11-19T13:59:55.169615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435943168665661:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:55.169624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:55.171375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435943168665648:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.172160Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435939195314425:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.172238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435939195314394:2108] [node 5] ICP18 handshake (actor [1:7574435939195314425:2118]) is too old
2025-11-19T13:59:55.171519Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435940018390328:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.171558Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435940018390318:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.171562Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435940018390318:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.173069Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435940018390318:2116] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:55.173098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435939627378116:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13598
2025-11-19T13:59:55.173190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435940018390309:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.173207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:55.173213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435940018390309:2113] [node 5] ICP22 created new session: [4:7574435940018390339:2113]
2025-11-19T13:59:55.173218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435940018390339:2113] [node 5] ICS09 handshake done sender: [4:7574435940018390318:2116] self: [4:7574435940018390317:0] peer: [5:7574435943168665663:0] socket: 96 qp: -1
2025-11-19T13:59:55.173222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435940018390339:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:55.173232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.184011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435943168665648:2113] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15552) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435943168665647:2116] held: no
2025-11-19T13:59:55.184225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435943168665664:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54054
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.184412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435943168665665:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54054
2025-11-19T13:59:55.184462Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435943168665647:2116] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:55.184543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435943168665648:2113] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.184559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:55.184564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435943168665648:2113] [node 4] ICP22 created new session: [5:7574435943168665666:2113]
2025-11-19T13:59:55.184570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435943168665666:2113] [node 4] ICS09 handshake done sender: [5:7574435943168665647:2116] self: [5:7574435943168665663:0] peer: [4:7574435940018390317:0] socket: 104 qp: -1
2025-11-19T13:59:55.184576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435943168665666:2113] [node 4] ICS10 traffic start
2025-11-19T13:59:55.184584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:55.192045Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435939627378116:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.192092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435935332410723:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.192176Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435943168665665:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.192209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435938873698278:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.192354Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435939905850602:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.192366Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435939905850602:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.192943Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435939905850600:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.192946Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435939905850600:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.193911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435943168665669:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54054
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.194023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435939627378118:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13598
2025-11-19T13:59:55.194133Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435939905850602:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:55.194317Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435939627378116:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:55.194385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435935332410723:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.194402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:55.194408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435935332410723:2049] [node 2] ICP22 created new session: [3:7574435939627378119:2049]
2025-11-19T13:59:55.194414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435939627378119:2049] [node 2] ICS09 handshake done sender: [3:7574435939627378116:2119] self: [3:7574435939627378117:0] peer: [2:7574435939905850601:0] socket: 93 qp: -1
2025-11-19T13:59:55.194420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435939627378119:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:55.194429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:55.195866Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435939905850600:2117] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:55.196079Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435943168665665:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:55.196170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435938873698278:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.196187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:55.196193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435938873698278:2049] [node 2] ICP22 created new session: [5:7574435943168665670:2049]
2025-11-19T13:59:55.196198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435943168665670:2049] [node 2] ICS09 handshake done sender: [5:7574435943168665665:2121] self: [5:7574435943168665668:0] peer: [2:7574435939905850599:0] socket: 105 qp: -1
2025-11-19T13:59:55.196204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435943168665670:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:55.196214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:55.197513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435935610883228:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.197537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:55.197543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435935610883228:2049] [node 3] ICP22 created new session: [2:7574435939905850625:2049]
2025-11-19T13:59:55.197549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435939905850625:2049] [node 3] ICS09 handshake done sender: [2:7574435939905850602:2118] self: [2:7574435939905850601:0] peer: [3:7574435939627378117:0] socket: 92 qp: -1
2025-11-19T13:59:55.197554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435939905850625:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:55.197563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:55.197909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435939905850596:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.197936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:55.197940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435939905850596:2110] [node 5] ICP22 created new session: [2:7574435939905850627:2110]
2025-11-19T13:59:55.197943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435939905850627:2110] [node 5] ICS09 handshake done sender: [2:7574435939905850600:2117] self: [2:7574435939905850599:0] peer: [5:7574435943168665668:0] socket: 91 qp: -1
2025-11-19T13:59:55.197948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435939905850627:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:55.197954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:55.216203Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:55.216296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435939905850615:2048] [node 1] ICS01 socket: 72 reason# EndOfStream
2025-11-19T13:59:55.216305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435935610883227:2048] [node 1] ICP30 unregister session Session# [2:7574435939905850615:2048] VirtualId# [2:7574435939905850606:0]
2025-11-19T13:59:55.216314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435939905850615:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:55.216348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:55.216369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:55.221422Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:55.221504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435939627378114:2048] [node 1] ICS01 socket: 119 reason# EndOfStream
2025-11-19T13:59:55.221512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435935332410722:2048] [node 1] ICP30 unregister session Session# [3:7574435939627378114:2048] VirtualId# [3:7574435939627378109:0]
2025-11-19T13:59:55.221520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435939627378114:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:55.221554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:55.221574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:55.230322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:55.230422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435943168665661:2048] [node 1] ICS01 socket: 71 reason# EndOfStream
2025-11-19T13:59:55.230431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435938873698277:2048] [node 1] ICP30 unregister session Session# [5:7574435943168665661:2048] VirtualId# [5:7574435943168665651:0]
2025-11-19T13:59:55.230438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435943168665661:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:55.230497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:55.230524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:55.230567Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:55.230628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435939946415334:2048] [node 1] ICS01 socket: 124 reason# EndOfStream
2025-11-19T13:59:55.230632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435935651447941:2048] [node 1] ICP30 unregister session Session# [6:7574435939946415334:2048] VirtualId# [6:7574435939946415331:0]
2025-11-19T13:59:55.230636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435939946415334:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:55.230662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:55.230684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:55.230839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:55.230898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435940018390331:2048] [node 1] ICS01 socket: 73 reason# EndOfStream
2025-11-19T13:59:55.230902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435935723422944:2048] [node 1] ICP30 unregister session Session# [4:7574435940018390331:2048] VirtualId# [4:7574435940018390321:0]
2025-11-19T13:59:55.230906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435940018390331:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:55.230931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:55.230951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:55.240553Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:55.240657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435939627378119:2049] [node 2] ICS01 socket: 93 reason# EndOfStream
2025-11-19T13:59:55.240666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435935332410723:2049] [node 2] ICP30 unregister session Session# [3:7574435939627378119:2049] VirtualId# [3:7574435939627378117:0]
2025-11-19T13:59:55.240674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435939627378119:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:55.240719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:55.240740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:55.247899Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:55.248016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435939946415336:2049] [node 2] ICS01 socket: 113 reason# EndOfStream
2025-11-19T13:59:55.248026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435935651447942:2049] [node 2] ICP30 unregister session Session# [6:7574435939946415336:2049] VirtualId# [6:7574435939946415327:0]
2025-11-19T13:59:55.248034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435939946415336:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:55.248087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:55.248113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:55.248152Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:55.248221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435943168665670:2049] [node 2] ICS01 socket: 105 reason# EndOfStream
2025-11-19T13:59:55.248227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435938873698278:2049] [node 2] ICP30 unregister session Session# [5:7574435943168665670:2049] VirtualId# [5:7574435943168665668:0]
2025-11-19T13:59:55.248232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435943168665670:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:55.248255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:55.248275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:55.248384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:55.248445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435940018390335:2049] [node 2] ICS01 socket: 111 reason# EndOfStream
2025-11-19T13:59:55.248449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435935723422945:2049] [node 2] ICP30 unregister session Session# [4:7574435940018390335:2049] VirtualId# [4:7574435940018390333:0]
2025-11-19T13:59:55.248454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435940018390335:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:55.248479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:55.248498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:55.275721Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:55.275847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435943168665659:2050] [node 3] ICS01 socket: 81 reason# EndOfStream
2025-11-19T13:59:55.275859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435938873698279:2050] [node 3] ICP30 unregister session Session# [5:7574435943168665659:2050] VirtualId# [5:7574435943168665640:0]
2025-11-19T13:59:55.275868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435943168665659:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:55.275944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:55.275973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:55.276025Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:55.276171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435939946415329:2050] [node 3] ICS01 socket: 114 reason# EndOfStream
2025-11-19T13:59:55.276176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435935651447943:2050] [node 3] ICP30 unregister session Session# [6:7574435939946415329:2050] VirtualId# [6:7574435939946415326:0]
2025-11-19T13:59:55.276181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435939946415329:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:55.276211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:55.276234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:55.283932Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:55.284033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435939946415308:2090] [node 4] ICS01 socket: 66 reason# EndOfStream
2025-11-19T13:59:55.284045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435939946415281:2090] [node 4] ICP30 unregister session Session# [6:7574435939946415308:2090] VirtualId# [6:7574435939946415282:0]
2025-11-19T13:59:55.284055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435939946415308:2090] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:55.284116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:55.284150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T13:59:55.715336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435943090734845:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:55.715358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435943090734846:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:55.770503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435941731912261:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:55.770519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435941731912262:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:55.808362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435943110091316:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:55.808375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435943110091317:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:55.808179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435941282863198:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:55.808193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435941282863199:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:55.808197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435941282863200:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:55.828141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435942188112842:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:55.828153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435942188112843:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:55.828158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435942188112844:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:55.852412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435939578719665:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:55.852426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435939578719666:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:55.852431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435939578719667:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:55.900187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435939578719700:2077] [node 4] ICP01 ready to work
2025-11-19T13:59:55.900259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435939578719701:2079] [node 5] ICP01 ready to work
2025-11-19T13:59:55.900468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435943090734907:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:55.900562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435943090734908:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.901764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435941282863257:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29292
2025-11-19T13:59:55.904326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435941731912323:2107] [node 4] ICP01 ready to work
2025-11-19T13:59:55.901871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435942188112901:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50522
2025-11-19T13:59:55.907728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435943090734916:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.908818Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435941282863257:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.912030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435942188112904:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:55.914189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435943110091377:2106] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.920133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435941282863263:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:55.920594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435941282863262:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29292
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.920839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435941731912323:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16426962ns) {Configure:104(16426374ns) {ProcessConfigured:135(16042320ns) {StartInitialHandshake:157(16042010ns) {DropHandshakes:469(818ns) {DropIncomingHandshake:441(148ns) DropOutgoingHandshake:456(123ns)} GenerateSessionVirtualId:988(69ns) PrepareNewSessionHandshake:476(919ns) {SwitchToState:178(413ns)}}}}}
2025-11-19T13:59:55.921024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435941731912328:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:55.921128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435941731912329:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:55.923218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435939578719667:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18830501ns) {Configure:104(18829840ns) {ProcessConfigured:135(18491248ns) {StartInitialHandshake:157(18490980ns) {DropHandshakes:469(794ns) {DropIncomingHandshake:441(182ns) DropOutgoingHandshake:456(118ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:476(1387ns) {SwitchToState:178(121ns)}}}}}
2025-11-19T13:59:55.922959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435942188112907:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50522
2025-11-19T13:59:55.923428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435941731912322:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53018
2025-11-19T13:59:55.934261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435942188112904:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(22152605ns) {Configure:104(22152020ns) {ProcessConfigured:135(21734274ns) {StartInitialHandshake:157(21733588ns) {DropHandshakes:469(781ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(140ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:476(1158ns) {SwitchToState:178(656ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.935499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435939578719740:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53093
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.935183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435943110091383:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:55.935329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435943110091384:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:55.937745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435943110091385:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15467
2025-11-19T13:59:55.937761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435943110091386:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15467
2025-11-19T13:59:55.940885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435941282863198:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.940948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435941282863198:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(32065416ns) {Configure:104(32065012ns) {ProcessConfigured:135(31693884ns) {IncomingHandshake:261(31681011ns) {DropIncomingHandshake:441(82ns) PrepareNewSessionHandshake:476(437ns) {SwitchToState:178(246ns)} GenerateSessionVirtualId:988(326ns) IssueIncomingHandshakeReply:193(31675642ns) {SwitchToState:178(177ns)}}}}}
2025-11-19T13:59:55.941131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435941282863268:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:55.944062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435939578719741:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53093
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.944614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435941731912341:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53018
2025-11-19T13:59:55.944639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435941731912342:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53018
2025-11-19T13:59:55.944738Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435943110091316:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(39596594ns) {Configure:104(39595289ns) {ProcessConfigured:135(39192533ns) {StartInitialHandshake:157(39191810ns) {DropHandshakes:469(452ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(45ns)} GenerateSessionVirtualId:988(296ns) PrepareNewSessionHandshake:476(1218ns) {SwitchToState:178(495ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.946323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435941282863269:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29292
2025-11-19T13:59:55.946344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435941282863270:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29292
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.947022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435941731912343:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53018
2025-11-19T13:59:55.949520Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435939578719740:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.949556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435939578719667:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.949792Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435941282863262:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.952095Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435941731912342:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.953316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435942188112842:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16589895ns) {Configure:104(16589006ns) {ProcessConfigured:135(16223738ns) {StartInitialHandshake:157(16223136ns) {DropHandshakes:469(897ns) {DropIncomingHandshake:441(123ns) DropOutgoingHandshake:456(157ns)} GenerateSessionVirtualId:988(314ns) PrepareNewSessionHandshake:476(659ns) {SwitchToState:178(568ns)}}}}}
2025-11-19T13:59:55.954135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435942188112920:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:55.954562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435942188112918:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50522
2025-11-19T13:59:55.954614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435942188112919:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50522
2025-11-19T13:59:55.954795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435943090734916:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(46532833ns) {Configure:104(46532186ns) {ProcessConfigured:135(46319052ns) {StartInitialHandshake:157(46318836ns) {DropHandshakes:469(246ns) {DropIncomingHandshake:441(15ns) DropOutgoingHandshake:456(31ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(626ns) {SwitchToState:178(503ns)}}}}}
2025-11-19T13:59:55.955059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435943090734924:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24160
2025-11-19T13:59:55.955072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435943090734925:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24160
2025-11-19T13:59:55.955699Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435943110091385:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.955730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435943110091316:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.956172Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435943090734910:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.956186Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435943090734910:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.956232Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435942188112901:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.956263Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435942188112907:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.956283Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435942188112918:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.956323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435942188112842:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.956342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435942188112844:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.956830Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435941731912322:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.960225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435943110091379:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.960248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435943110091379:2107] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.961558Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435943090734918:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.961574Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435943090734918:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.961612Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435943110091388:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.961620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435943110091388:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.966137Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435943090734912:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.966150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435943090734912:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.968368Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435943090734910:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:55.970263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435941282863280:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29292
2025-11-19T13:59:55.976575Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435943090734912:2110] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:55.978760Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435943110091388:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:55.979595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435943110091384:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.979615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:55.979622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435943110091384:2109] [node 5] ICP22 created new session: [3:7574435943110091399:2109]
2025-11-19T13:59:55.979629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435943110091399:2109] [node 5] ICS09 handshake done sender: [3:7574435943110091388:2112] self: [3:7574435943110091387:0] peer: [5:7574435942188112926:0] socket: 96 qp: -1
2025-11-19T13:59:55.979636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435943110091399:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:55.979645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:55.977934Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435941282863257:2106] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:55.979966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435943090734907:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.979983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:55.979989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435943090734907:2107] [node 4] ICP22 created new session: [1:7574435943090734928:2107]
2025-11-19T13:59:55.979994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435943090734928:2107] [node 4] ICS09 handshake done sender: [1:7574435943090734910:2109] self: [1:7574435943090734909:0] peer: [4:7574435941282863258:0] socket: 72 qp: -1
2025-11-19T13:59:55.979998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435943090734928:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:55.980004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:55.980215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435943090734908:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.980220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:55.980225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435943090734908:2108] [node 5] ICP22 created new session: [1:7574435943090734931:2108]
2025-11-19T13:59:55.980229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435943090734931:2108] [node 5] ICS09 handshake done sender: [1:7574435943090734912:2110] self: [1:7574435943090734911:0] peer: [5:7574435942188112924:0] socket: 73 qp: -1
2025-11-19T13:59:55.980232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435943090734931:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:55.980236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:55.989349Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435943110091379:2107] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:55.989472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435943110091377:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.989491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:55.989499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435943110091377:2106] [node 6] ICP22 created new session: [3:7574435943110091403:2106]
2025-11-19T13:59:55.989505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435943110091403:2106] [node 6] ICS09 handshake done sender: [3:7574435943110091379:2107] self: [3:7574435943110091378:0] peer: [6:7574435939578719743:0] socket: 82 qp: -1
2025-11-19T13:59:55.989512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435943110091403:2106] [node 6] ICS10 traffic start
2025-11-19T13:59:55.989529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:55.991371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435941282863198:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:55.991410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:55.991417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435941282863198:2048] [node 1] ICP22 created new session: [4:7574435941282863282:2048]
2025-11-19T13:59:55.991424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435941282863282:2048] [node 1] ICS09 handshake done sender: [4:7574435941282863257:2106] self: [4:7574435941282863258:0] peer: [1:7574435943090734909:0] socket: 74 qp: -1
2025-11-19T13:59:55.991429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435941282863282:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:55.991446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.992870Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435939578719741:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.992960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435939578719700:2077] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:55.992978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435941282863281:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29292
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:55.992840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435943090734933:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24160
2025-11-19T13:59:55.995252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435942188112927:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50522
2025-11-19T13:59:55.995507Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435941282863265:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:55.995521Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435941282863265:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:55.996304Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435941282863265:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:55.996395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435941282863263:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.996415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:55.996421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435941282863263:2108] [node 6] ICP22 created new session: [4:7574435941282863285:2108]
2025-11-19T13:59:55.996426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435941282863285:2108] [node 6] ICS09 handshake done sender: [4:7574435941282863265:2111] self: [4:7574435941282863264:0] peer: [6:7574435939578719746:0] socket: 84 qp: -1
2025-11-19T13:59:55.996431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435941282863285:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:55.996438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:55.998548Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435943090734918:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:55.998296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435939578719700:2077] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27762) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435939578719741:2114] held: no
2025-11-19T13:59:55.998744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435943090734846:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:55.998760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:55.998766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435943090734846:2049] [node 3] ICP22 created new session: [1:7574435943090734936:2049]
2025-11-19T13:59:55.998772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435943090734936:2049] [node 3] ICS09 handshake done sender: [1:7574435943090734918:2112] self: [1:7574435943090734917:0] peer: [3:7574435943110091397:0] socket: 83 qp: -1
2025-11-19T13:59:55.998777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435943090734936:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:55.998785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.001985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435946026879642:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53018
2025-11-19T13:59:56.003110Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435943090734908:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(22767966ns) {HandleHandshakeStatus:329(22767409ns) {DropIncomingHandshake:441(25ns) DropOutgoingHandshake:456(61ns) SwitchToState:178(30ns) ProcessPendingSessionEvents:495(22726122ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22704564ns max# 22634177ns}}}
2025-11-19T13:59:56.003506Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435943090734924:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.003755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435947405058702:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15467
2025-11-19T13:59:56.004020Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435943090734933:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.004038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435943090734846:2049] [node 3] ICP18 handshake (actor [1:7574435943090734933:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.004368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435947405058703:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15467
2025-11-19T13:59:56.004386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435947405058704:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15467
2025-11-19T13:59:56.005045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435943110091316:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24670) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435943110091385:2110] held: no
2025-11-19T13:59:56.005148Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435943110091385:2110] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:56.005346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435943110091316:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.005360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:56.005365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435943110091316:2048] [node 1] ICP22 created new session: [3:7574435947405058705:2048]
2025-11-19T13:59:56.005370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435947405058705:2048] [node 1] ICS09 handshake done sender: [3:7574435943110091385:2110] self: [3:7574435943110091397:0] peer: [1:7574435943090734917:0] socket: 94 qp: -1
2025-11-19T13:59:56.005376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435947405058705:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:56.005382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:56.005967Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435943090734921:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.005977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435943090734921:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.006762Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435943090734921:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:56.006840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435943090734845:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.006855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:56.006860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435943090734845:2048] [node 2] ICP22 created new session: [1:7574435947385702236:2048]
2025-11-19T13:59:56.006865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435947385702236:2048] [node 2] ICS09 handshake done sender: [1:7574435943090734921:2113] self: [1:7574435943090734920:0] peer: [2:7574435946026879643:0] socket: 119 qp: -1
2025-11-19T13:59:56.006871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435947385702236:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:56.006877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:56.003451Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435946026879642:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.003898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435943873687044:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53093
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.008372Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435943110091386:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.008399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435943110091317:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.008547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435942188112928:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50522
2025-11-19T13:59:56.008645Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435941731912331:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.008656Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435941731912331:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.009192Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435941282863281:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.009221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435941282863200:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.009371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435943873687045:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53093
2025-11-19T13:59:56.009659Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435939578719740:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:56.009562Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435947405058703:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.009717Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435947405058704:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.009730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435943110091377:2106] [node 6] ICP18 handshake (actor [3:7574435947405058704:2119]) is too old
2025-11-19T13:59:56.009564Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435942188112919:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.009702Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435942188112901:2106] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:56.009819Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435943090734925:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.009839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435943090734908:2108] [node 5] ICP18 handshake (actor [1:7574435943090734925:2116]) is too old
2025-11-19T13:59:56.009985Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435942188112918:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:56.010029Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435941282863270:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.010106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435942188112842:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.010126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:56.010132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435942188112842:2048] [node 1] ICP22 created new session: [5:7574435946483080225:2048]
2025-11-19T13:59:56.010137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435946483080225:2048] [node 1] ICS09 handshake done sender: [5:7574435942188112901:2106] self: [5:7574435942188112924:0] peer: [1:7574435943090734911:0] socket: 75 qp: -1
2025-11-19T13:59:56.010142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435946483080225:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:56.010148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:56.010148Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435943873687045:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.010761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435939578719667:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.010776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:56.010781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435939578719667:2050] [node 3] ICP22 created new session: [6:7574435943873687047:2050]
2025-11-19T13:59:56.010786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435943873687047:2050] [node 3] ICS09 handshake done sender: [6:7574435939578719740:2113] self: [6:7574435939578719743:0] peer: [3:7574435943110091378:0] socket: 91 qp: -1
2025-11-19T13:59:56.010792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435943873687047:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:56.010799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:56.010932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435939578719667:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:56.011191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435939578719665:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.015951Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435943090734923:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.015963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435943090734923:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.016830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435947385702238:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24160
2025-11-19T13:59:56.017010Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435943090734923:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:56.017101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435943090734916:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.017115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:56.017122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435943090734916:2111] [node 6] ICP22 created new session: [1:7574435947385702240:2111]
2025-11-19T13:59:56.017128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435947385702240:2111] [node 6] ICS09 handshake done sender: [1:7574435943090734923:2114] self: [1:7574435943090734919:0] peer: [6:7574435943873687049:0] socket: 120 qp: -1
2025-11-19T13:59:56.017133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435947385702240:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:56.017142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:56.017366Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435943110091392:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.017390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435943110091392:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.018158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435943873687046:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53093
2025-11-19T13:59:56.018319Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435941282863269:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.018358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435941282863199:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.017731Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435941731912343:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.017760Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435941731912341:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.017802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435941731912261:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.017894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435941731912261:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24670) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435946026879642:2119] held: no
2025-11-19T13:59:56.018340Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435941731912331:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.018802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435942188112844:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.018822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:56.018827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435942188112844:2050] [node 3] ICP22 created new session: [5:7574435946483080227:2050]
2025-11-19T13:59:56.018833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435946483080227:2050] [node 3] ICS09 handshake done sender: [5:7574435942188112918:2116] self: [5:7574435942188112926:0] peer: [3:7574435943110091387:0] socket: 103 qp: -1
2025-11-19T13:59:56.018838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435946483080227:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:56.018869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:56.019178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435942188112904:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.019361Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435942188112842:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:56.019726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435946483080230:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50522
2025-11-19T13:59:56.019785Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435941282863272:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.019792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435941282863272:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.020788Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435939578719741:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.021344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435939578719700:2077] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.021358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:56.021364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435939578719700:2077] [node 4] ICP22 created new session: [6:7574435943873687053:2077]
2025-11-19T13:59:56.021370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435943873687053:2077] [node 4] ICS09 handshake done sender: [6:7574435939578719741:2114] self: [6:7574435939578719746:0] peer: [4:7574435941282863264:0] socket: 92 qp: -1
2025-11-19T13:59:56.021375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435943873687053:2077] [node 4] ICS10 traffic start
2025-11-19T13:59:56.021383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:56.021608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435943873687051:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53093
2025-11-19T13:59:56.021913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435947405058709:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15467
2025-11-19T13:59:56.022281Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435943110091386:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.022475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435943873687052:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53093
2025-11-19T13:59:56.022566Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435943873687045:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:56.022627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435943110091317:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.022641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:56.022646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435943110091317:2049] [node 2] ICP22 created new session: [3:7574435947405058710:2049]
2025-11-19T13:59:56.022650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435947405058710:2049] [node 2] ICS09 handshake done sender: [3:7574435943110091386:2111] self: [3:7574435947405058707:0] peer: [2:7574435941731912330:0] socket: 95 qp: -1
2025-11-19T13:59:56.022655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435947405058710:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:56.022661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:56.021088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435946026879644:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53018
2025-11-19T13:59:56.024110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435939578719665:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.024128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:56.024133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435939578719665:2048] [node 1] ICP22 created new session: [6:7574435943873687055:2048]
2025-11-19T13:59:56.024139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435943873687055:2048] [node 1] ICS09 handshake done sender: [6:7574435943873687045:2118] self: [6:7574435943873687049:0] peer: [1:7574435943090734919:0] socket: 128 qp: -1
2025-11-19T13:59:56.024145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435943873687055:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:56.024152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:56.025432Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435941282863272:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:56.025646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435941282863268:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.025664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:56.025670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435941282863268:2112] [node 5] ICP22 created new session: [4:7574435945577830588:2112]
2025-11-19T13:59:56.025675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435945577830588:2112] [node 5] ICS09 handshake done sender: [4:7574435941282863272:2115] self: [4:7574435941282863271:0] peer: [5:7574435946483080229:0] socket: 100 qp: -1
2025-11-19T13:59:56.025679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435945577830588:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:56.025686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:56.027113Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435941731912327:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.027130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435941731912327:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.026407Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435947385702238:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.026674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435943090734916:2111] [node 6] ICP18 handshake (actor [1:7574435947385702238:2118]) is too old
2025-11-19T13:59:56.028348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435941731912262:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.028368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:56.028373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435941731912262:2049] [node 3] ICP22 created new session: [2:7574435946026879649:2049]
2025-11-19T13:59:56.028379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435946026879649:2049] [node 3] ICS09 handshake done sender: [2:7574435941731912331:2112] self: [2:7574435941731912330:0] peer: [3:7574435947405058707:0] socket: 86 qp: -1
2025-11-19T13:59:56.028384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435946026879649:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:56.028391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.031325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435945577830590:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29292
2025-11-19T13:59:56.031647Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435941282863281:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:56.031491Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435943110091392:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:56.031828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435941282863200:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.031842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:56.031847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435941282863200:2050] [node 3] ICP22 created new session: [4:7574435945577830591:2050]
2025-11-19T13:59:56.031852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435945577830591:2050] [node 3] ICS09 handshake done sender: [4:7574435941282863281:2119] self: [4:7574435945577830584:0] peer: [3:7574435943110091391:0] socket: 116 qp: -1
2025-11-19T13:59:56.031859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435945577830591:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:56.031866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:56.032993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435943110091383:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.033013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:56.033017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435943110091383:2108] [node 4] ICP22 created new session: [3:7574435947405058713:2108]
2025-11-19T13:59:56.033023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435947405058713:2108] [node 4] ICS09 handshake done sender: [3:7574435943110091392:2114] self: [3:7574435943110091391:0] peer: [4:7574435945577830584:0] socket: 113 qp: -1
2025-11-19T13:59:56.033028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435947405058713:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:56.033034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.033672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435942188112904:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27762) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435942188112919:2117] held: no
2025-11-19T13:59:56.033966Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435943873687052:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.033999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435939578719701:2079] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.034458Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435946026879642:2119] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:56.036078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435941731912261:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.036094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:56.036100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435941731912261:2048] [node 1] ICP22 created new session: [2:7574435946026879653:2048]
2025-11-19T13:59:56.036106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435946026879653:2048] [node 1] ICS09 handshake done sender: [2:7574435946026879642:2119] self: [2:7574435946026879643:0] peer: [1:7574435943090734920:0] socket: 125 qp: -1
2025-11-19T13:59:56.036111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435946026879653:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:56.036116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:56.036888Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435941731912327:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:56.036976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435941731912323:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.037011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:56.037016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435941731912323:2107] [node 4] ICP22 created new session: [2:7574435946026879655:2107]
2025-11-19T13:59:56.037020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435946026879655:2107] [node 4] ICS09 handshake done sender: [2:7574435941731912327:2111] self: [2:7574435941731912324:0] peer: [4:7574435945577830586:0] socket: 85 qp: -1
2025-11-19T13:59:56.037025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435946026879655:2107] [node 4] ICS10 traffic start
2025-11-19T13:59:56.037030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:56.037404Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435942188112922:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.037417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435942188112922:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.037495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435946483080232:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50522
2025-11-19T13:59:56.037669Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435942188112919:2117] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:56.037763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435942188112904:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.037775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:56.037781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435942188112904:2108] [node 4] ICP22 created new session: [5:7574435946483080233:2108]
2025-11-19T13:59:56.037785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435946483080233:2108] [node 4] ICS09 handshake done sender: [5:7574435942188112919:2117] self: [5:7574435946483080229:0] peer: [4:7574435941282863271:0] socket: 104 qp: -1
2025-11-19T13:59:56.037790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435946483080233:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:56.037794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.038129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435943873687058:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53093
2025-11-19T13:59:56.038808Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435942188112922:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:56.038884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435942188112920:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.038898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:56.038902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435942188112920:2113] [node 6] ICP22 created new session: [5:7574435946483080236:2113]
2025-11-19T13:59:56.038907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435946483080236:2113] [node 6] ICS09 handshake done sender: [5:7574435942188112922:2118] self: [5:7574435942188112921:0] peer: [6:7574435943873687057:0] socket: 132 qp: -1
2025-11-19T13:59:56.038911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435946483080236:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:56.038917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.039284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435943873687059:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53093
2025-11-19T13:59:56.039436Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435943873687052:2121] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.040837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435941282863199:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6863) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435941282863269:2113] held: no
2025-11-19T13:59:56.041379Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435946483080230:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.041409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435942188112843:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.041782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435939578719701:2079] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23237) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435943873687052:2121] held: no
2025-11-19T13:59:56.041879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435939578719701:2079] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.041888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:56.041892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435939578719701:2079] [node 5] ICP22 created new session: [6:7574435943873687060:2079]
2025-11-19T13:59:56.041898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435943873687060:2079] [node 5] ICS09 handshake done sender: [6:7574435943873687052:2121] self: [6:7574435943873687057:0] peer: [5:7574435942188112921:0] socket: 135 qp: -1
2025-11-19T13:59:56.041903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435943873687060:2079] [node 5] ICS10 traffic start
2025-11-19T13:59:56.041910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:56.042605Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435941731912335:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.042617Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435941731912335:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.043661Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435943873687058:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.043691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435939578719666:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.043787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435945577830593:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29292
2025-11-19T13:59:56.043945Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435941282863269:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:56.044021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435941282863199:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.044040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:56.044046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435941282863199:2049] [node 2] ICP22 created new session: [4:7574435945577830594:2049]
2025-11-19T13:59:56.044051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435945577830594:2049] [node 2] ICS09 handshake done sender: [4:7574435941282863269:2113] self: [4:7574435945577830586:0] peer: [2:7574435941731912324:0] socket: 90 qp: -1
2025-11-19T13:59:56.044056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435945577830594:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:56.044062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:56.043530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435946483080239:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50522
2025-11-19T13:59:56.046790Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435941731912335:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:56.047026Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435946483080230:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:56.047899Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435941731912337:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.047911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435941731912337:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.047139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435942188112843:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.047156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:56.047173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435942188112843:2049] [node 2] ICP22 created new session: [5:7574435946483080240:2049]
2025-11-19T13:59:56.047180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435946483080240:2049] [node 2] ICS09 handshake done sender: [5:7574435946483080230:2121] self: [5:7574435946483080238:0] peer: [2:7574435941731912334:0] socket: 131 qp: -1
2025-11-19T13:59:56.047186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435946483080240:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:56.047192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:56.048382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435941731912328:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.048401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:56.048407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435941731912328:2109] [node 5] ICP22 created new session: [2:7574435946026879658:2109]
2025-11-19T13:59:56.048412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435946026879658:2109] [node 5] ICS09 handshake done sender: [2:7574435941731912335:2114] self: [2:7574435941731912334:0] peer: [5:7574435946483080238:0] socket: 127 qp: -1
2025-11-19T13:59:56.048417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435946026879658:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:56.048423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.060035Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435941731912337:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:56.059914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435943873687063:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53093
2025-11-19T13:59:56.060605Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435943873687058:2122] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:56.060409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435941731912329:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.060432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:56.060439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435941731912329:2110] [node 6] ICP22 created new session: [2:7574435946026879661:2110]
2025-11-19T13:59:56.060445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435946026879661:2110] [node 6] ICS09 handshake done sender: [2:7574435941731912337:2115] self: [2:7574435941731912336:0] peer: [6:7574435943873687062:0] socket: 93 qp: -1
2025-11-19T13:59:56.060451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435946026879661:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:56.060461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:56.060699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435939578719666:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.060716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:56.060731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435939578719666:2049] [node 2] ICP22 created new session: [6:7574435943873687064:2049]
2025-11-19T13:59:56.060737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435943873687064:2049] [node 2] ICS09 handshake done sender: [6:7574435943873687058:2122] self: [6:7574435943873687062:0] peer: [2:7574435941731912336:0] socket: 98 qp: -1
2025-11-19T13:59:56.060743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435943873687064:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:56.060751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:56.066834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:56.066918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435946026879653:2048] [node 1] ICS01 socket: 125 reason# EndOfStream
2025-11-19T13:59:56.066926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435941731912261:2048] [node 1] ICP30 unregister session Session# [2:7574435946026879653:2048] VirtualId# [2:7574435946026879643:0]
2025-11-19T13:59:56.066932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435946026879653:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.066964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:56.066983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:56.067370Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:56.067445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435947405058705:2048] [node 1] ICS01 socket: 94 reason# EndOfStream
2025-11-19T13:59:56.067450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435943110091316:2048] [node 1] ICP30 unregister session Session# [3:7574435947405058705:2048] VirtualId# [3:7574435943110091397:0]
2025-11-19T13:59:56.067454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435947405058705:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.067478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:56.067494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:56.074651Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:56.074740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435946483080225:2048] [node 1] ICS01 socket: 75 reason# EndOfStream
2025-11-19T13:59:56.074747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435942188112842:2048] [node 1] ICP30 unregister session Session# [5:7574435946483080225:2048] VirtualId# [5:7574435942188112924:0]
2025-11-19T13:59:56.074754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435946483080225:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.074807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:56.074833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:56.074875Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:56.074940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435943873687055:2048] [node 1] ICS01 socket: 128 reason# EndOfStream
2025-11-19T13:59:56.074945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435939578719665:2048] [node 1] ICP30 unregister session Session# [6:7574435943873687055:2048] VirtualId# [6:7574435943873687049:0]
2025-11-19T13:59:56.074949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435943873687055:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.074975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:56.074996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:56.076023Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:56.076112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435941282863282:2048] [node 1] ICS01 socket: 74 reason# EndOfStream
2025-11-19T13:59:56.076120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435941282863198:2048] [node 1] ICP30 unregister session Session# [4:7574435941282863282:2048] VirtualId# [4:7574435941282863258:0]
2025-11-19T13:59:56.076127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435941282863282:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.076187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:56.076220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:56.080242Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:56.080328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435947405058710:2049] [node 2] ICS01 socket: 95 reason# EndOfStream
2025-11-19T13:59:56.080336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435943110091317:2049] [node 2] ICP30 unregister session Session# [3:7574435947405058710:2049] VirtualId# [3:7574435947405058707:0]
2025-11-19T13:59:56.080341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435947405058710:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.080373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:56.080391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:56.102963Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:56.103029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435946483080240:2049] [node 2] ICS01 socket: 131 reason# EndOfStream
2025-11-19T13:59:56.103035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435942188112843:2049] [node 2] ICP30 unregister session Session# [5:7574435946483080240:2049] VirtualId# [5:7574435946483080238:0]
2025-11-19T13:59:56.103039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435946483080240:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.103074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:56.103096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:56.103136Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:56.103226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435943873687064:2049] [node 2] ICS07 socket disconnect 98 reason# EndOfStream
2025-11-19T13:59:56.103234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435943873687064:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.103267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:56.103288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435943873687064:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:56.106234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435939578719666:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6863) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:56.106252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435943873687064:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:56.106258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435939578719666:2049] [node 2] ICP30 unregister session Session# [6:7574435943873687064:2049] VirtualId# [6:7574435943873687062:0]
2025-11-19T13:59:56.106297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:56.106303Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435939578719666:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6863) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:56.106305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6863) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:56.102693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:56.102797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435945577830594:2049] [node 2] ICS01 socket: 90 reason# EndOfStream
2025-11-19T13:59:56.102807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435941282863199:2049] [node 2] ICP30 unregister session Session# [4:7574435945577830594:2049] VirtualId# [4:7574435945577830586:0]
2025-11-19T13:59:56.102816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435945577830594:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.102887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:56.102913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:56.117522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435939578719666:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:56.117660Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:56.117754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435946483080227:2050] [node 3] ICS01 socket: 103 reason# EndOfStream
2025-11-19T13:59:56.117764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435942188112844:2050] [node 3] ICP30 unregister session Session# [5:7574435946483080227:2050] VirtualId# [5:7574435942188112926:0]
2025-11-19T13:59:56.117771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435946483080227:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.117830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:56.117852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:56.118454Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:56.118519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435943873687047:2050] [node 3] ICS01 socket: 91 reason# EndOfStream
2025-11-19T13:59:56.118524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435939578719667:2050] [node 3] ICP30 unregister session Session# [6:7574435943873687047:2050] VirtualId# [6:7574435939578719743:0]
2025-11-19T13:59:56.118529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435943873687047:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.118556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:56.118576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:56.147375Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:56.147464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435943873687053:2077] [node 4] ICS01 socket: 92 reason# EndOfStream
2025-11-19T13:59:56.147473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435939578719700:2077] [node 4] ICP30 unregister session Session# [6:7574435943873687053:2077] VirtualId# [6:7574435939578719746:0]
2025-11-19T13:59:56.147479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435943873687053:2077] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.147531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:56.147553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T13:59:56.267937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435946103012765:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:56.267951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435946103012766:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:56.295958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435944167514073:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:56.295970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435944167514074:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:56.368049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435946273035467:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:56.368063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435946273035468:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:56.459901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435944924530413:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:56.459920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435944924530414:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:56.459925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435944924530415:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:56.536126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435945281862079:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:56.536154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435945281862080:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:56.536159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435945281862081:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:56.599966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435946955428700:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:56.599983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435946955428701:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:56.599988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435946955428702:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:56.640348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435945281862127:2093] [node 6] ICP01 ready to work
2025-11-19T13:59:56.640059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435946955428715:2060] [node 5] ICP01 ready to work
2025-11-19T13:59:56.640537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435946273035522:2101] [node 6] ICP01 ready to work
2025-11-19T13:59:56.640756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435945281862130:2096] [node 4] ICP01 ready to work
2025-11-19T13:59:56.640805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435946955428721:2066] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.645007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435944167514133:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14877
2025-11-19T13:59:56.645296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435944167514134:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14877
2025-11-19T13:59:56.655408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435946103012830:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:56.655569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435946103012835:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.655807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435946103012836:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.655937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435944924530480:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29754
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.658017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435944167514141:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:56.658110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435944167514142:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:56.658999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435944167514140:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14877
2025-11-19T13:59:56.659518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435946273035526:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21999
2025-11-19T13:59:56.659582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435946273035527:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21999
2025-11-19T13:59:56.659592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435946273035528:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21999
2025-11-19T13:59:56.659602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435946273035529:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21999
2025-11-19T13:59:56.659906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435946955428721:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19056342ns) {Configure:104(19055932ns) {ProcessConfigured:135(15283282ns) {StartInitialHandshake:157(15282656ns) {DropHandshakes:469(341ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(35ns)} GenerateSessionVirtualId:988(82ns) PrepareNewSessionHandshake:476(631ns) {SwitchToState:178(456ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.660490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435945281862154:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14870
2025-11-19T13:59:56.660529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435945281862155:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14870
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.661325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435946273035526:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.661379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435946273035532:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21999
2025-11-19T13:59:56.662909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435946273035467:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.664761Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435944167514073:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16018869ns) {Configure:104(16018257ns) {ProcessConfigured:135(15636812ns) {StartInitialHandshake:157(15636488ns) {DropHandshakes:469(390ns) {DropIncomingHandshake:441(41ns) DropOutgoingHandshake:456(112ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:476(1106ns) {SwitchToState:178(423ns)}}}}}
2025-11-19T13:59:56.668241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435944924530486:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:56.668360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435944924530487:2113] [node 6] ICP01 ready to work
2025-11-19T13:59:56.668125Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435945281862080:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(25191291ns) {Configure:104(25190810ns) {ProcessConfigured:135(24975836ns) {StartInitialHandshake:157(24975482ns) {DropHandshakes:469(286ns) {DropIncomingHandshake:441(34ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:476(765ns) {SwitchToState:178(338ns)}}}}}
2025-11-19T13:59:56.672004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435946273035542:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:56.672271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435946273035522:2101] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(31639610ns) {Configure:104(31639044ns) {ProcessConfigured:135(31213170ns) {StartInitialHandshake:157(31212654ns) {DropHandshakes:469(487ns) {DropIncomingHandshake:441(87ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:476(929ns) {SwitchToState:178(468ns)}}}}}
2025-11-19T13:59:56.674091Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435946273035527:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.675325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435946103012826:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.675354Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435946103012826:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.675664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435944167514139:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:56.675807Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435944167514140:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.680034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435946955428778:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45591
2025-11-19T13:59:56.679780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435946103012844:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30034
2025-11-19T13:59:56.679802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435946103012845:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30034
2025-11-19T13:59:56.679827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435946103012846:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30034
2025-11-19T13:59:56.681394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435946955428780:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45591
2025-11-19T13:59:56.683970Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435946955428778:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.684023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435946955428700:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.682017Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435944924530480:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.689119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435944924530487:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19075588ns) {Configure:104(19073720ns) {ProcessConfigured:135(18792480ns) {StartInitialHandshake:157(18792109ns) {DropHandshakes:469(416ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(271ns) PrepareNewSessionHandshake:476(1417ns) {SwitchToState:178(988ns)}}}}}
2025-11-19T13:59:56.689254Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435945281862154:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.687337Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435946103012845:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.687376Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435946103012844:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.689620Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435946103012838:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.689630Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435946103012838:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.691659Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435946103012846:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.689567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435946955428779:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45591
2025-11-19T13:59:56.689661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435946273035540:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:56.689721Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435946273035529:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.696379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435946273035468:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.696447Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435946273035532:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.698247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435944167514073:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.698331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435944167514154:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14877
2025-11-19T13:59:56.698943Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435946103012826:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:56.699049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435945281862155:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.699101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435945281862080:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.701323Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435946273035528:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.702044Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435944167514146:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.702057Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435944167514146:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.702576Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435944167514134:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.703645Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435944167514133:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.703700Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435944167514146:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:56.702909Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435946955428780:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.702985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435946955428702:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.711670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435945281862161:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14870
2025-11-19T13:59:56.707873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435946273035540:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(17907189ns) {Configure:104(17905999ns) {ProcessConfigured:135(17589867ns) {StartInitialHandshake:157(17589389ns) {DropHandshakes:469(435ns) {DropIncomingHandshake:441(27ns) DropOutgoingHandshake:456(38ns)} GenerateSessionVirtualId:988(174ns) PrepareNewSessionHandshake:476(1427ns) {SwitchToState:178(644ns)}}}}}
2025-11-19T13:59:56.707944Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435946273035543:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.707948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435946273035543:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.711892Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435945281862155:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:56.710285Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435944167514139:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(21751733ns) {Configure:104(21750750ns) {ProcessConfigured:135(21394755ns) {StartInitialHandshake:157(21394355ns) {DropHandshakes:469(797ns) {DropIncomingHandshake:441(132ns) DropOutgoingHandshake:456(172ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:476(1694ns) {SwitchToState:178(613ns)}}}}}
2025-11-19T13:59:56.710415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435944167514141:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.710428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:56.710437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435944167514141:2110] [node 5] ICP22 created new session: [2:7574435944167514159:2110]
2025-11-19T13:59:56.710443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435944167514159:2110] [node 5] ICS09 handshake done sender: [2:7574435944167514146:2114] self: [2:7574435944167514145:0] peer: [5:7574435945281862160:0] socket: 97 qp: -1
2025-11-19T13:59:56.710453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435944167514159:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:56.710465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:56.712276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435945281862080:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.712299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:56.712308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435945281862080:2049] [node 2] ICP22 created new session: [5:7574435945281862162:2049]
2025-11-19T13:59:56.712315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435945281862162:2049] [node 2] ICS09 handshake done sender: [5:7574435945281862155:2113] self: [5:7574435945281862160:0] peer: [2:7574435944167514145:0] socket: 104 qp: -1
2025-11-19T13:59:56.712324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435945281862162:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:56.712346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:56.712065Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435946103012831:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.712086Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435946103012831:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.712245Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435946955428779:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.712296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435946955428701:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.712519Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435944167514144:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.712533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435944167514144:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.713302Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435946273035543:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.716222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435946273035557:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21999
2025-11-19T13:59:56.716563Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435946103012838:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:56.719736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435946103012766:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.719799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:56.719813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435946103012766:2049] [node 3] ICP22 created new session: [1:7574435946103012853:2049]
2025-11-19T13:59:56.719823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435946103012853:2049] [node 3] ICS09 handshake done sender: [1:7574435946103012826:2107] self: [1:7574435946103012825:0] peer: [3:7574435946273035533:0] socket: 79 qp: -1
2025-11-19T13:59:56.719834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435946103012853:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:56.719862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.722310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435946955428784:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45591
2025-11-19T13:59:56.722798Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435944167514148:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.722816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435944167514148:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.721086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435945281862164:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14870
2025-11-19T13:59:56.719844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435946273035522:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.719875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:56.719883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435946273035522:2101] [node 6] ICP22 created new session: [3:7574435946273035558:2101]
2025-11-19T13:59:56.719889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435946273035558:2101] [node 6] ICS09 handshake done sender: [3:7574435946273035543:2116] self: [3:7574435946273035523:0] peer: [6:7574435946955428782:0] socket: 107 qp: -1
2025-11-19T13:59:56.719896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435946273035558:2101] [node 6] ICS10 traffic start
2025-11-19T13:59:56.719906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:56.721759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435946103012830:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.721779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:56.721785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435946103012830:2108] [node 6] ICP22 created new session: [1:7574435946103012855:2108]
2025-11-19T13:59:56.721791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435946103012855:2108] [node 6] ICS09 handshake done sender: [1:7574435946103012838:2112] self: [1:7574435946103012837:0] peer: [6:7574435946955428781:0] socket: 95 qp: -1
2025-11-19T13:59:56.721798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435946103012855:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:56.721806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:56.724531Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435946103012766:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(20926695ns) {HandleHandshakeStatus:329(20925150ns) {DropIncomingHandshake:441(13235ns) DropOutgoingHandshake:456(236ns) SwitchToState:178(288ns) ProcessPendingSessionEvents:495(128793ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 111980ns max# 36365ns}}}
2025-11-19T13:59:56.726365Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435946103012831:2109] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.728232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435946103012765:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.728254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:56.728264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435946103012765:2048] [node 2] ICP22 created new session: [1:7574435946103012859:2048]
2025-11-19T13:59:56.728270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435946103012859:2048] [node 2] ICS09 handshake done sender: [1:7574435946103012831:2109] self: [1:7574435946103012827:0] peer: [2:7574435944167514155:0] socket: 89 qp: -1
2025-11-19T13:59:56.728278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435946103012859:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:56.728287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:56.727951Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435946273035526:2103] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.728798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435946103012861:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30034
2025-11-19T13:59:56.728257Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435944167514141:2110] [node 5] ICP35 event processing took too much time PendingConnection:278(17733109ns) {HandleHandshakeStatus:329(17731881ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(307ns) SwitchToState:178(91ns) ProcessPendingSessionEvents:495(8169806ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8150587ns max# 8072867ns}}}
2025-11-19T13:59:56.728647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435944167514162:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14877
2025-11-19T13:59:56.730890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435946273035467:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.730909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:56.730914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435946273035467:2048] [node 1] ICP22 created new session: [3:7574435946273035560:2048]
2025-11-19T13:59:56.730920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435946273035560:2048] [node 1] ICS09 handshake done sender: [3:7574435946273035526:2103] self: [3:7574435946273035533:0] peer: [1:7574435946103012825:0] socket: 81 qp: -1
2025-11-19T13:59:56.730925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435946273035560:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:56.730932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:56.731075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435946273035467:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.728974Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435945281862164:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.729008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435945281862079:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.731674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435945281862167:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14870
2025-11-19T13:59:56.732033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435945281862080:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(19656440ns) {HandleHandshakeStatus:329(19655930ns) {DropIncomingHandshake:441(327ns) DropOutgoingHandshake:456(2175ns) SwitchToState:178(64ns) ProcessPendingSessionEvents:495(11840220ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 11820869ns max# 11756408ns}}}
2025-11-19T13:59:56.732335Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435944167514154:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.733491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435946955428785:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45591
2025-11-19T13:59:56.733549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435946955428786:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45591
2025-11-19T13:59:56.734648Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435944167514140:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.736170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435946273035564:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21999
2025-11-19T13:59:56.736375Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435946955428780:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:56.737010Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435944167514144:2113] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:56.737113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435944167514073:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.737129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:56.737135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435944167514073:2048] [node 1] ICP22 created new session: [2:7574435944167514167:2048]
2025-11-19T13:59:56.737141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435944167514167:2048] [node 1] ICS09 handshake done sender: [2:7574435944167514140:2112] self: [2:7574435944167514155:0] peer: [1:7574435946103012827:0] socket: 92 qp: -1
2025-11-19T13:59:56.737146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435944167514167:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:56.737152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:56.736621Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435946955428702:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61269) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435946955428780:2117] held: no
2025-11-19T13:59:56.736706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435946955428702:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.736717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:56.736722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435946955428702:2050] [node 3] ICP22 created new session: [6:7574435946955428787:2050]
2025-11-19T13:59:56.736728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435946955428787:2050] [node 3] ICS09 handshake done sender: [6:7574435946955428780:2117] self: [6:7574435946955428782:0] peer: [3:7574435946273035523:0] socket: 113 qp: -1
2025-11-19T13:59:56.736735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435946955428787:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:56.736743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:56.736965Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435946955428786:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.737094Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435946955428778:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:56.737373Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435946103012840:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.737388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435946103012840:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.738102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435944167514074:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.738118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:56.738123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435944167514074:2049] [node 3] ICP22 created new session: [2:7574435944167514169:2049]
2025-11-19T13:59:56.738127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435944167514169:2049] [node 3] ICS09 handshake done sender: [2:7574435944167514144:2113] self: [2:7574435944167514143:0] peer: [3:7574435946273035552:0] socket: 96 qp: -1
2025-11-19T13:59:56.738131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435944167514169:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:56.738136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:56.739639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435946955428700:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.739658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:56.739666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435946955428700:2048] [node 1] ICP22 created new session: [6:7574435946955428789:2048]
2025-11-19T13:59:56.739672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435946955428789:2048] [node 1] ICS09 handshake done sender: [6:7574435946955428778:2115] self: [6:7574435946955428781:0] peer: [1:7574435946103012837:0] socket: 106 qp: -1
2025-11-19T13:59:56.739678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435946955428789:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:56.739685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:56.740021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435946955428715:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.741254Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435946273035529:2106] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:56.742916Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435944167514148:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:56.743122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435944167514142:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.743141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:56.743146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435944167514142:2111] [node 6] ICP22 created new session: [2:7574435944167514172:2111]
2025-11-19T13:59:56.743151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435944167514172:2111] [node 6] ICS09 handshake done sender: [2:7574435944167514148:2115] self: [2:7574435944167514147:0] peer: [6:7574435946955428783:0] socket: 98 qp: -1
2025-11-19T13:59:56.743156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435944167514172:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:56.743163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:56.743628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435946103012765:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:56.741439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435944924530493:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29754
2025-11-19T13:59:56.741514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435944924530494:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29754
2025-11-19T13:59:56.741550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435944924530495:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29754
2025-11-19T13:59:56.744464Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435945281862148:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.744482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435945281862148:2107] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.744300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435946273035468:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435946273035529:2106] held: no
2025-11-19T13:59:56.746647Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435944924530493:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.746682Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435944924530494:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.746938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435946273035468:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.746957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:56.746964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435946273035468:2049] [node 2] ICP22 created new session: [3:7574435946273035565:2049]
2025-11-19T13:59:56.746970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435946273035565:2049] [node 2] ICS09 handshake done sender: [3:7574435946273035529:2106] self: [3:7574435946273035552:0] peer: [2:7574435944167514143:0] socket: 101 qp: -1
2025-11-19T13:59:56.746977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435946273035565:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:56.746984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:56.747547Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435945281862167:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.747604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435945281862081:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.747873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435944924530413:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.747922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435944924530415:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.747837Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435946273035549:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.747860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435946273035549:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.751830Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435944924530495:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.752355Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435946103012842:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.752371Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435946103012842:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.752593Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435946103012840:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.757253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435945281862169:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14870
2025-11-19T13:59:56.758403Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435946273035553:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.758417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435946273035553:2119] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.759311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435946103012835:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.759340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:56.759351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435946103012835:2110] [node 5] ICP22 created new session: [1:7574435946103012863:2110]
2025-11-19T13:59:56.759358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435946103012863:2110] [node 5] ICS09 handshake done sender: [1:7574435946103012840:2113] self: [1:7574435946103012839:0] peer: [5:7574435945281862166:0] socket: 121 qp: -1
2025-11-19T13:59:56.759365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435946103012863:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:56.759374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.760102Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435945281862164:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:56.761109Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435946955428701:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435946955428779:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.762595Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435946273035553:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.763170Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435946273035549:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:56.763448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435946955428793:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45591
2025-11-19T13:59:56.761154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435945281862079:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.761168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:56.761175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435945281862079:2048] [node 1] ICP22 created new session: [5:7574435945281862170:2048]
2025-11-19T13:59:56.761183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435945281862170:2048] [node 1] ICS09 handshake done sender: [5:7574435945281862164:2118] self: [5:7574435945281862166:0] peer: [1:7574435946103012839:0] socket: 122 qp: -1
2025-11-19T13:59:56.761189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435945281862170:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:56.761196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:56.764626Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435946103012842:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:56.764617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435946273035540:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.764640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:56.764647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435946273035540:2109] [node 4] ICP22 created new session: [3:7574435946273035569:2109]
2025-11-19T13:59:56.764662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435946273035569:2109] [node 4] ICS09 handshake done sender: [3:7574435946273035553:2119] self: [3:7574435946273035551:0] peer: [4:7574435944924530497:0] socket: 103 qp: -1
2025-11-19T13:59:56.764668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435946273035569:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:56.764676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:56.764409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435944924530499:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29754
2025-11-19T13:59:56.764749Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435946955428779:2116] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:56.764899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435946103012836:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.764915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:56.764920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435946103012836:2111] [node 4] ICP22 created new session: [1:7574435946103012866:2111]
2025-11-19T13:59:56.764926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435946103012866:2111] [node 4] ICS09 handshake done sender: [1:7574435946103012842:2114] self: [1:7574435946103012841:0] peer: [4:7574435944924530496:0] socket: 111 qp: -1
2025-11-19T13:59:56.764930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435946103012866:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:56.764937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:56.765064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435946955428701:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.765075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:56.765081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435946955428701:2049] [node 2] ICP22 created new session: [6:7574435946955428794:2049]
2025-11-19T13:59:56.765085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435946955428794:2049] [node 2] ICS09 handshake done sender: [6:7574435946955428779:2116] self: [6:7574435946955428783:0] peer: [2:7574435944167514147:0] socket: 112 qp: -1
2025-11-19T13:59:56.765089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435946955428794:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:56.765096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.768038Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435944167514158:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.768052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435944167514158:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.766413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435944924530500:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29754
2025-11-19T13:59:56.766627Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435944924530499:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.766671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435944924530501:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29754
2025-11-19T13:59:56.766812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435944924530414:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.766864Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435944924530494:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:56.766961Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435944924530493:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:56.767005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435944924530415:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.767017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:56.767021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435944924530415:2050] [node 3] ICP22 created new session: [4:7574435944924530503:2050]
2025-11-19T13:59:56.767027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435944924530503:2050] [node 3] ICS09 handshake done sender: [4:7574435944924530494:2117] self: [4:7574435944924530497:0] peer: [3:7574435946273035551:0] socket: 137 qp: -1
2025-11-19T13:59:56.767033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435944924530503:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:56.767040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:56.771960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435944924530413:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.771978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:56.771986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435944924530413:2048] [node 1] ICP22 created new session: [4:7574435944924530505:2048]
2025-11-19T13:59:56.771992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435944924530505:2048] [node 1] ICS09 handshake done sender: [4:7574435944924530493:2116] self: [4:7574435944924530496:0] peer: [1:7574435946103012841:0] socket: 127 qp: -1
2025-11-19T13:59:56.771999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435944924530505:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:56.772020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:56.773707Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435945281862148:2107] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:56.775142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435946273035542:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.775165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:56.775186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435946273035542:2110] [node 5] ICP22 created new session: [3:7574435946273035571:2110]
2025-11-19T13:59:56.775192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435946273035571:2110] [node 5] ICS09 handshake done sender: [3:7574435946273035549:2118] self: [3:7574435946273035547:0] peer: [5:7574435945281862168:0] socket: 100 qp: -1
2025-11-19T13:59:56.775198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435946273035571:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:56.775205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:56.775630Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435945281862079:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:56.775879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435945281862127:2093] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.775892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:56.775896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435945281862127:2093] [node 6] ICP22 created new session: [5:7574435945281862174:2093]
2025-11-19T13:59:56.775901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435945281862174:2093] [node 6] ICS09 handshake done sender: [5:7574435945281862148:2107] self: [5:7574435945281862147:0] peer: [6:7574435946955428791:0] socket: 123 qp: -1
2025-11-19T13:59:56.775905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435945281862174:2093] [node 6] ICS10 traffic start
2025-11-19T13:59:56.775914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:56.777258Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435944167514158:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:56.777395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435944167514139:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.777415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:56.777422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435944167514139:2109] [node 4] ICP22 created new session: [2:7574435944167514175:2109]
2025-11-19T13:59:56.777426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435944167514175:2109] [node 4] ICS09 handshake done sender: [2:7574435944167514158:2118] self: [2:7574435944167514153:0] peer: [4:7574435944924530502:0] socket: 109 qp: -1
2025-11-19T13:59:56.777431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435944167514175:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:56.777440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:56.779845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435945281862173:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14870
2025-11-19T13:59:56.781983Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435946103012836:2111] [node 4] ICP35 event processing took too much time PendingConnection:278(16984873ns) {HandleHandshakeStatus:329(16983688ns) {DropIncomingHandshake:441(3586ns) DropOutgoingHandshake:456(185ns) SwitchToState:178(117ns) ProcessPendingSessionEvents:495(16921816ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16905093ns max# 16827882ns}}}
2025-11-19T13:59:56.783520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435945281862081:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61269) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435945281862167:2119] held: no
2025-11-19T13:59:56.783638Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435945281862167:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:56.784067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435945281862081:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.784081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:56.784089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435945281862081:2050] [node 3] ICP22 created new session: [5:7574435945281862176:2050]
2025-11-19T13:59:56.784093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435945281862176:2050] [node 3] ICS09 handshake done sender: [5:7574435945281862167:2119] self: [5:7574435945281862168:0] peer: [3:7574435946273035547:0] socket: 131 qp: -1
2025-11-19T13:59:56.784100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435945281862176:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:56.784106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.786671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435944924530507:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29754
2025-11-19T13:59:56.787127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435946955428796:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45591
2025-11-19T13:59:56.788216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435946273035540:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(23496880ns) {HandleHandshakeStatus:329(23495817ns) {DropIncomingHandshake:441(4911ns) DropOutgoingHandshake:456(220ns) SwitchToState:178(266ns) ProcessPendingSessionEvents:495(23404813ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23385190ns max# 23315998ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.789053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435945281862178:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14870
2025-11-19T13:59:56.789132Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435946955428786:2120] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:56.789677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435946955428715:2060] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.789694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:56.789700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435946955428715:2060] [node 5] ICP22 created new session: [6:7574435946955428797:2060]
2025-11-19T13:59:56.789706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435946955428797:2060] [node 5] ICS09 handshake done sender: [6:7574435946955428786:2120] self: [6:7574435946955428791:0] peer: [5:7574435945281862147:0] socket: 125 qp: -1
2025-11-19T13:59:56.789713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435946955428797:2060] [node 5] ICS10 traffic start
2025-11-19T13:59:56.789721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:56.794600Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435944924530499:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:56.794919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435944924530414:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435944924530499:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.795323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435946955428799:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45591
2025-11-19T13:59:56.795922Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435946955428799:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.795961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435946955428721:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:56.795617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435944924530414:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.795641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:56.795651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435944924530414:2049] [node 2] ICP22 created new session: [4:7574435944924530510:2049]
2025-11-19T13:59:56.795656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435944924530510:2049] [node 2] ICS09 handshake done sender: [4:7574435944924530499:2119] self: [4:7574435944924530502:0] peer: [2:7574435944167514153:0] socket: 108 qp: -1
2025-11-19T13:59:56.795664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435944924530510:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:56.795673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:56.796817Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435945281862178:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.796862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435945281862130:2096] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.796177Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435944924530492:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.796184Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435944924530492:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.797077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435946955428801:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45591
2025-11-19T13:59:56.797224Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435944924530490:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:56.797230Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435944924530490:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:56.798009Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435944924530492:2115] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:56.798280Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435946955428799:2123] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:56.798375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435946955428721:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.798389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:56.798396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435946955428721:2066] [node 4] ICP22 created new session: [6:7574435946955428802:2066]
2025-11-19T13:59:56.798403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435946955428802:2066] [node 4] ICS09 handshake done sender: [6:7574435946955428799:2123] self: [6:7574435946955428800:0] peer: [4:7574435944924530489:0] socket: 84 qp: -1
2025-11-19T13:59:56.798410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435946955428802:2066] [node 4] ICS10 traffic start
2025-11-19T13:59:56.798419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:56.799990Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435944924530490:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:56.800315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435944924530486:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.800342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:56.800349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435944924530486:2112] [node 5] ICP22 created new session: [4:7574435944924530514:2112]
2025-11-19T13:59:56.800355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435944924530514:2112] [node 5] ICS09 handshake done sender: [4:7574435944924530490:2114] self: [4:7574435944924530488:0] peer: [5:7574435945281862179:0] socket: 78 qp: -1
2025-11-19T13:59:56.800364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435944924530514:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:56.800372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:56.800865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435945281862180:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14870
2025-11-19T13:59:56.800982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435944924530487:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:56.801001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:56.801006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435944924530487:2113] [node 6] ICP22 created new session: [4:7574435944924530516:2113]
2025-11-19T13:59:56.801129Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435945281862178:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:56.801255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435944924530516:2113] [node 6] ICS09 handshake done sender: [4:7574435944924530492:2115] self: [4:7574435944924530489:0] peer: [6:7574435946955428800:0] socket: 82 qp: -1
2025-11-19T13:59:56.801265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435944924530516:2113] [node 6] ICS10 traffic start
2025-11-19T13:59:56.801276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:56.801309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435945281862130:2096] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:56.801327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:56.801332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435945281862130:2096] [node 4] ICP22 created new session: [5:7574435945281862181:2096]
2025-11-19T13:59:56.801337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435945281862181:2096] [node 4] ICS09 handshake done sender: [5:7574435945281862178:2122] self: [5:7574435945281862179:0] peer: [4:7574435944924530488:0] socket: 99 qp: -1
2025-11-19T13:59:56.801341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435945281862181:2096] [node 4] ICS10 traffic start
2025-11-19T13:59:56.801348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:56.823104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:56.823223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435944167514167:2048] [node 1] ICS01 socket: 92 reason# EndOfStream
2025-11-19T13:59:56.823232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435944167514073:2048] [node 1] ICP30 unregister session Session# [2:7574435944167514167:2048] VirtualId# [2:7574435944167514155:0]
2025-11-19T13:59:56.823239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435944167514167:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.823276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:56.823295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:56.823814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:56.823882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435946273035560:2048] [node 1] ICS01 socket: 81 reason# EndOfStream
2025-11-19T13:59:56.823888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435946273035467:2048] [node 1] ICP30 unregister session Session# [3:7574435946273035560:2048] VirtualId# [3:7574435946273035533:0]
2025-11-19T13:59:56.823901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435946273035560:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.823930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:56.823947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:56.831435Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:56.831501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435944924530505:2048] [node 1] ICS01 socket: 127 reason# EndOfStream
2025-11-19T13:59:56.831511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435944924530413:2048] [node 1] ICP30 unregister session Session# [4:7574435944924530505:2048] VirtualId# [4:7574435944924530496:0]
2025-11-19T13:59:56.831516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435944924530505:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.831541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:56.830874Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:56.830975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435946955428789:2048] [node 1] ICS01 socket: 106 reason# EndOfStream
2025-11-19T13:59:56.830984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435946955428700:2048] [node 1] ICP30 unregister session Session# [6:7574435946955428789:2048] VirtualId# [6:7574435946955428781:0]
2025-11-19T13:59:56.830991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435946955428789:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.831039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:56.831063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:56.831097Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:56.831158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435945281862170:2048] [node 1] ICS07 socket disconnect 122 reason# EndOfStream
2025-11-19T13:59:56.831162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435945281862170:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.831204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:56.831223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435945281862170:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:56.831561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:56.835564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435945281862079:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21109) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:56.835580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435945281862170:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:56.835584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435945281862079:2048] [node 1] ICP30 unregister session Session# [5:7574435945281862170:2048] VirtualId# [5:7574435945281862166:0]
2025-11-19T13:59:56.835620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:56.835626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435945281862079:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21109) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:56.835628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21109) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:56.840849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:56.840939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435946273035565:2049] [node 2] ICS01 socket: 101 reason# EndOfStream
2025-11-19T13:59:56.840947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435946273035468:2049] [node 2] ICP30 unregister session Session# [3:7574435946273035565:2049] VirtualId# [3:7574435946273035552:0]
2025-11-19T13:59:56.840954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435946273035565:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.840998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:56.841018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:56.847469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:56.847568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435944924530510:2049] [node 2] ICS01 socket: 108 reason# EndOfStream
2025-11-19T13:59:56.847576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435944924530414:2049] [node 2] ICP30 unregister session Session# [4:7574435944924530510:2049] VirtualId# [4:7574435944924530502:0]
2025-11-19T13:59:56.847583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435944924530510:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.847637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:56.847662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:56.847885Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:56.847967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435945281862162:2049] [node 2] ICS07 socket disconnect 104 reason# EndOfStream
2025-11-19T13:59:56.847973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435945281862162:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.848006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:56.848025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435945281862162:2049] [node 2] ICS15 start handshake
2025-11-19T13:59:56.848238Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:56.848299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435946955428794:2049] [node 2] ICS01 socket: 112 reason# EndOfStream
2025-11-19T13:59:56.848304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435946955428701:2049] [node 2] ICP30 unregister session Session# [6:7574435946955428794:2049] VirtualId# [6:7574435946955428783:0]
2025-11-19T13:59:56.848310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435946955428794:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.848334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:56.848352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:56.848882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435945281862080:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7482) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:56.848894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435945281862162:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:56.848898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435945281862080:2049] [node 2] ICP30 unregister session Session# [5:7574435945281862162:2049] VirtualId# [5:7574435945281862160:0]
2025-11-19T13:59:56.848932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:56.848937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435945281862080:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7482) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:56.848941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7482) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:56.855789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435945281862079:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:56.856339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435945281862080:2049] [node 2] ICP33 wake up from error state
2025-11-19T13:59:56.900025Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:56.900135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435945281862176:2050] [node 3] ICS01 socket: 131 reason# EndOfStream
2025-11-19T13:59:56.900145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435945281862081:2050] [node 3] ICP30 unregister session Session# [5:7574435945281862176:2050] VirtualId# [5:7574435945281862168:0]
2025-11-19T13:59:56.900153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435945281862176:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.900215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:56.900237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:56.900328Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:56.900386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435946955428787:2050] [node 3] ICS01 socket: 113 reason# EndOfStream
2025-11-19T13:59:56.900390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435946955428702:2050] [node 3] ICP30 unregister session Session# [6:7574435946955428787:2050] VirtualId# [6:7574435946955428782:0]
2025-11-19T13:59:56.900394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435946955428787:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.900420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:56.900440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:56.938627Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:56.938764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435946955428802:2066] [node 4] ICS01 socket: 84 reason# EndOfStream
2025-11-19T13:59:56.938774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435946955428721:2066] [node 4] ICP30 unregister session Session# [6:7574435946955428802:2066] VirtualId# [6:7574435946955428800:0]
2025-11-19T13:59:56.938781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435946955428802:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:56.938842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:56.938868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T13:59:57.164074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435947782144153:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:57.164092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435947782144154:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:57.183867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435951403150947:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:57.183886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435951403150948:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:57.207882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435950977279535:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:57.207901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435950977279536:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:57.233053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435950871491208:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:57.233052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435950871491209:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:57.233067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435950871491210:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:57.223141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435950741519236:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:57.223160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435950741519237:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:57.223166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435950741519238:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:57.259660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435948702367345:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:57.259670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435948702367346:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:57.259675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435948702367347:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:57.328100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435948702367376:2077] [node 5] ICP01 ready to work
2025-11-19T13:59:57.328651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435951403151010:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:57.328651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435951403151009:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.329894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435948702367382:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21517
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.330932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435950977279595:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.328883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435947782144213:2106] [node 5] ICP01 ready to work
2025-11-19T13:59:57.331839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435947782144219:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56880
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.331973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435950871491263:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52760
2025-11-19T13:59:57.332307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435950871491266:2104] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.333197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435950741519297:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62461
2025-11-19T13:59:57.334197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435947782144225:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.336394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435947782144227:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56880
2025-11-19T13:59:57.336531Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435950977279595:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.336671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435950977279600:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:57.336705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435950977279535:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.337380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435948702367397:2089] [node 4] ICP01 ready to work
2025-11-19T13:59:57.338062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435948702367398:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21517
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.334242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435950741519302:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:57.334814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435950871491272:2107] [node 6] ICP01 ready to work
2025-11-19T13:59:57.336517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435951403151019:2111] [node 4] ICP01 ready to work
2025-11-19T13:59:57.338668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435951403151018:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30759
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.343885Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435947782144215:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.343901Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435947782144215:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:57.344591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435947782144229:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56880
2025-11-19T13:59:57.344603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435947782144230:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56880
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.344760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435950977279607:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-11-19T13:59:57.348535Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435947782144215:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:57.348799Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435950977279595:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:57.348885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435950977279535:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.348908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:57.348916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435950977279535:2048] [node 1] ICP22 created new session: [3:7574435950977279609:2048]
2025-11-19T13:59:57.348924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435950977279609:2048] [node 1] ICS09 handshake done sender: [3:7574435950977279595:2107] self: [3:7574435950977279601:0] peer: [1:7574435947782144214:0] socket: 89 qp: -1
2025-11-19T13:59:57.348932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435950977279609:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:57.348959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:57.349303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435950977279611:2113] [node 5] ICP01 ready to work
2025-11-19T13:59:57.349252Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435950871491263:2102] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.349643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435950871491209:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.349605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435950741519307:2111] [node 5] ICP01 ready to work
2025-11-19T13:59:57.349683Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435950741519297:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.350346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435950741519305:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62461
2025-11-19T13:59:57.350624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435947782144154:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.350640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:57.350645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435947782144154:2049] [node 3] ICP22 created new session: [1:7574435947782144232:2049]
2025-11-19T13:59:57.350661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435947782144232:2049] [node 3] ICS09 handshake done sender: [1:7574435947782144215:2107] self: [1:7574435947782144214:0] peer: [3:7574435950977279601:0] socket: 88 qp: -1
2025-11-19T13:59:57.350666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435947782144232:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:57.350673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:57.351455Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435947782144227:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.351720Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435947782144229:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.356305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435950871491277:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52760
2025-11-19T13:59:57.361754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435950741519306:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62461
2025-11-19T13:59:57.363941Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435951403151012:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.363974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435951403151012:2109] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.368120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435950977279536:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(27786224ns) {Configure:104(27785090ns) {ProcessConfigured:135(27469036ns) {StartInitialHandshake:157(27467643ns) {DropHandshakes:469(708ns) {DropIncomingHandshake:441(81ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(444ns) PrepareNewSessionHandshake:476(1312ns) {SwitchToState:178(781ns)}}}}}
2025-11-19T13:59:57.368462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435950977279617:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-11-19T13:59:57.369940Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435950741519306:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.371898Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435950977279617:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.371397Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435948702367398:2096] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.376421Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435948702367376:2077] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(48244882ns) {Configure:104(48244036ns) {ProcessConfigured:135(33302259ns) {StartInitialHandshake:157(33299774ns) {DropHandshakes:469(713ns) {DropIncomingHandshake:441(248ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:476(1075ns) {SwitchToState:178(247ns)}}}}}
2025-11-19T13:59:57.376521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435948702367376:2077] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:57.376918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435948702367382:2082] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.379784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435947782144223:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:57.380422Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435950741519302:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(46104560ns) {Configure:104(46103891ns) {ProcessConfigured:135(45857072ns) {StartInitialHandshake:157(45856331ns) {DropHandshakes:469(539ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(117ns) PrepareNewSessionHandshake:476(1017ns) {SwitchToState:178(503ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.382890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435950741519318:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62461
2025-11-19T13:59:57.383378Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435950741519305:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.383408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435950741519238:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:57.383568Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435950977279603:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.383580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435950977279603:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:57.381108Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435947782144225:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(46850077ns) {Configure:104(46849704ns) {ProcessConfigured:135(46574336ns) {StartInitialHandshake:157(46574150ns) {DropHandshakes:469(361ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(49ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(616ns) {SwitchToState:178(274ns)}}}}}
2025-11-19T13:59:57.383701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435947782144213:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(54765632ns) {Configure:104(54765216ns) {ProcessConfigured:135(47693678ns) {StartInitialHandshake:157(47693012ns) {DropHandshakes:469(640ns) {DropIncomingHandshake:441(228ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:476(582ns) {SwitchToState:178(140ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.383980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435950871491285:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52760
2025-11-19T13:59:57.384772Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435947782144230:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.385926Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435947782144238:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.385937Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435947782144238:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:57.387083Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435947782144238:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.384907Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435950741519318:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.384938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435950741519236:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:57.387579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435947782144225:2112] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.387604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:57.387612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435947782144225:2112] [node 4] ICP22 created new session: [1:7574435947782144244:2112]
2025-11-19T13:59:57.387619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435947782144244:2112] [node 4] ICS09 handshake done sender: [1:7574435947782144238:2117] self: [1:7574435947782144226:0] peer: [4:7574435950741519320:0] socket: 121 qp: -1
2025-11-19T13:59:57.387627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435947782144244:2112] [node 4] ICS10 traffic start
2025-11-19T13:59:57.387653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:57.387940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435951403151027:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30759
2025-11-19T13:59:57.388012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435951403151028:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30759
2025-11-19T13:59:57.390985Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435951403151012:2109] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:57.391089Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435951403151028:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.391109Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435951403151027:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.391148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435951403150947:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:57.391266Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435947782144218:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.391276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435947782144218:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:57.392054Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435947782144218:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:57.392118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435951403151009:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.392138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:57.392146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435951403151009:2108] [node 5] ICP22 created new session: [2:7574435951403151032:2108]
2025-11-19T13:59:57.392154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435951403151032:2108] [node 5] ICS09 handshake done sender: [2:7574435951403151012:2109] self: [2:7574435951403151011:0] peer: [5:7574435950871491276:0] socket: 84 qp: -1
2025-11-19T13:59:57.392164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435951403151032:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:57.392177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:57.392242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435947782144153:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.392257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:57.392261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435947782144153:2048] [node 2] ICP22 created new session: [1:7574435947782144248:2048]
2025-11-19T13:59:57.392267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435947782144248:2048] [node 2] ICS09 handshake done sender: [1:7574435947782144218:2108] self: [1:7574435947782144217:0] peer: [2:7574435951403151031:0] socket: 93 qp: -1
2025-11-19T13:59:57.392271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435947782144248:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:57.392278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:57.392270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435950977279615:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:57.392808Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435951403151015:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.392823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435951403151015:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:57.392882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435948702367426:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21517
2025-11-19T13:59:57.393045Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435950871491285:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.393060Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435950871491274:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.393068Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435950871491274:2108] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:57.393092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435950871491266:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:57.393223Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435950871491277:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.393247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435950871491210:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.402208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435947782144250:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56880
2025-11-19T13:59:57.403051Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435950977279603:2110] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:57.403207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435950977279600:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.403235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:57.403245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435950977279600:2109] [node 4] ICP22 created new session: [3:7574435950977279622:2109]
2025-11-19T13:59:57.403253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435950977279622:2109] [node 4] ICS09 handshake done sender: [3:7574435950977279603:2110] self: [3:7574435950977279602:0] peer: [4:7574435950741519319:0] socket: 101 qp: -1
2025-11-19T13:59:57.403260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435950977279622:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:57.403269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:57.403892Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435948702367426:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.403945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435948702367345:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.407216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435950871491290:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52760
2025-11-19T13:59:57.407284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435950871491291:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52760
2025-11-19T13:59:57.407961Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435947782144235:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.407976Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435947782144235:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:57.409024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435947782144250:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.409750Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435947782144235:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.409387Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435950871491290:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.409420Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435950871491263:2102] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:57.409559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435950871491208:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:57.409618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435950871491209:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.409644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:57.409653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435950871491209:2049] [node 2] ICP22 created new session: [5:7574435950871491293:2049]
2025-11-19T13:59:57.409660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435950871491293:2049] [node 2] ICS09 handshake done sender: [5:7574435950871491263:2102] self: [5:7574435950871491276:0] peer: [2:7574435951403151011:0] socket: 92 qp: -1
2025-11-19T13:59:57.409668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435950871491293:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:57.409687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.412422Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435950741519309:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.412441Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435950741519309:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:57.410141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435950977279624:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-11-19T13:59:57.413004Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435950977279624:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.413037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435950977279536:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:57.411918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435951403151035:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30759
2025-11-19T13:59:57.412931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435951403151018:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.412951Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435951403151027:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:57.415845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435948702367346:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:57.415892Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435948702367346:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(38926290ns) {IncomingHandshake:261(38925671ns) {DropIncomingHandshake:441(263ns) PrepareNewSessionHandshake:476(518ns) {SwitchToState:178(304ns)} GenerateSessionVirtualId:988(200ns) IssueIncomingHandshakeReply:193(38921231ns) {SwitchToState:178(820ns)}}}
2025-11-19T13:59:57.416040Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435950977279613:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.416059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435950977279613:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:57.425358Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435947782144241:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.425381Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435947782144241:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:57.425913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435947782144223:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.425940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:57.425951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435947782144223:2111] [node 6] ICP22 created new session: [1:7574435947782144253:2111]
2025-11-19T13:59:57.425960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435947782144253:2111] [node 6] ICS09 handshake done sender: [1:7574435947782144235:2116] self: [1:7574435947782144234:0] peer: [6:7574435948702367427:0] socket: 119 qp: -1
2025-11-19T13:59:57.425970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435947782144253:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:57.425984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:57.426663Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435947782144153:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(34321877ns) {HandleHandshakeStatus:329(34321739ns) {DropIncomingHandshake:441(3760ns) DropOutgoingHandshake:456(75ns) SwitchToState:178(36ns) ProcessPendingSessionEvents:495(34255271ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 34238499ns max# 34168294ns}}}
2025-11-19T13:59:57.427876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435950977279615:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(35441049ns) {Configure:104(35439493ns) {ProcessConfigured:135(35061469ns) {StartInitialHandshake:157(35060343ns) {DropHandshakes:469(881ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(587ns) PrepareNewSessionHandshake:476(1618ns) {SwitchToState:178(378ns)}}}}}
2025-11-19T13:59:57.428936Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435951403151023:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.428949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435951403151023:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:57.430248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435948702367347:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(101842483ns) {Configure:104(101841899ns) {ProcessConfigured:135(87014307ns) {StartInitialHandshake:157(87013986ns) {DropHandshakes:469(493ns) {DropIncomingHandshake:441(16ns) DropOutgoingHandshake:456(120ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(1543ns) {SwitchToState:178(177ns)}}}}}
2025-11-19T13:59:57.431463Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435950871491274:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.431799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435950871491272:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.431816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:57.431824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435950871491272:2107] [node 6] ICP22 created new session: [5:7574435950871491296:2107]
2025-11-19T13:59:57.431832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435950871491296:2107] [node 6] ICS09 handshake done sender: [5:7574435950871491274:2108] self: [5:7574435950871491273:0] peer: [6:7574435948702367424:0] socket: 100 qp: -1
2025-11-19T13:59:57.429834Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435947782144241:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:57.431838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435950871491296:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:57.431863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:57.431965Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435948702367345:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12510) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435948702367426:2116] held: no
2025-11-19T13:59:57.436143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435950741519236:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12510) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435950741519318:2117] held: no
2025-11-19T13:59:57.438733Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435950741519238:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15791) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435950741519305:2112] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.441268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435947782144213:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.441294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:57.441300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435947782144213:2106] [node 5] ICP22 created new session: [1:7574435947782144256:2106]
2025-11-19T13:59:57.441307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435947782144256:2106] [node 5] ICS09 handshake done sender: [1:7574435947782144241:2118] self: [1:7574435947782144228:0] peer: [5:7574435950871491292:0] socket: 125 qp: -1
2025-11-19T13:59:57.441313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435947782144256:2106] [node 5] ICS10 traffic start
2025-11-19T13:59:57.441320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:57.442149Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435950741519309:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:57.442317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435950741519307:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.442334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:57.442338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435950741519307:2111] [node 5] ICP22 created new session: [4:7574435950741519328:2111]
2025-11-19T13:59:57.442343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435950741519328:2111] [node 5] ICS09 handshake done sender: [4:7574435950741519309:2114] self: [4:7574435950741519308:0] peer: [5:7574435950871491288:0] socket: 112 qp: -1
2025-11-19T13:59:57.442347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435950741519328:2111] [node 5] ICS10 traffic start
2025-11-19T13:59:57.442368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:57.440073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435951403151009:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(47838191ns) {HandleHandshakeStatus:329(47836946ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(487ns) SwitchToState:178(92ns) ProcessPendingSessionEvents:495(27258570ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27232640ns max# 27150010ns}}}
2025-11-19T13:59:57.440101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435951403150947:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.440119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:57.440126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435951403150947:2048] [node 1] ICP22 created new session: [2:7574435951403151036:2048]
2025-11-19T13:59:57.440131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435951403151036:2048] [node 1] ICS09 handshake done sender: [2:7574435951403151027:2116] self: [2:7574435951403151031:0] peer: [1:7574435947782144217:0] socket: 99 qp: -1
2025-11-19T13:59:57.440140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435951403151036:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:57.440150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:57.440331Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435951403150947:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:57.441097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435951403151041:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30759
2025-11-19T13:59:57.444090Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435951403151015:2110] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:57.444597Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435950977279613:2114] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:57.444703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435950977279611:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.444717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:57.444724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435950977279611:2113] [node 5] ICP22 created new session: [3:7574435950977279629:2113]
2025-11-19T13:59:57.444729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435950977279629:2113] [node 5] ICS09 handshake done sender: [3:7574435950977279613:2114] self: [3:7574435950977279612:0] peer: [5:7574435950871491289:0] socket: 90 qp: -1
2025-11-19T13:59:57.444745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435950977279629:2113] [node 5] ICS10 traffic start
2025-11-19T13:59:57.444752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:57.444556Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435951403151023:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:57.445637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435951403150948:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.445656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:57.445662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435951403150948:2049] [node 3] ICP22 created new session: [2:7574435951403151043:2049]
2025-11-19T13:59:57.445668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435951403151043:2049] [node 3] ICS09 handshake done sender: [2:7574435951403151023:2114] self: [2:7574435951403151022:0] peer: [3:7574435950977279625:0] socket: 131 qp: -1
2025-11-19T13:59:57.445673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435951403151043:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:57.445679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:57.447559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435951403151010:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.447579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:57.447586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435951403151010:2107] [node 6] ICP22 created new session: [2:7574435951403151045:2107]
2025-11-19T13:59:57.447592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435951403151045:2107] [node 6] ICS09 handshake done sender: [2:7574435951403151015:2110] self: [2:7574435951403151014:0] peer: [6:7574435948702367425:0] socket: 85 qp: -1
2025-11-19T13:59:57.447598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435951403151045:2107] [node 6] ICS10 traffic start
2025-11-19T13:59:57.447606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.452226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435950741519237:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(51025261ns) {Configure:104(51024268ns) {ProcessConfigured:135(50678587ns) {StartInitialHandshake:157(50677417ns) {DropHandshakes:469(691ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(198ns)} GenerateSessionVirtualId:988(133ns) PrepareNewSessionHandshake:476(1548ns) {SwitchToState:178(42ns)}}}}}
2025-11-19T13:59:57.452291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435948702367429:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21517
2025-11-19T13:59:57.455071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435948702367346:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10104) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435948702367382:2082] held: no
2025-11-19T13:59:57.455250Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435948702367426:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:57.455330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435948702367345:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.455352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:57.455360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435948702367345:2048] [node 1] ICP22 created new session: [6:7574435948702367430:2048]
2025-11-19T13:59:57.455366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435948702367430:2048] [node 1] ICS09 handshake done sender: [6:7574435948702367426:2116] self: [6:7574435948702367427:0] peer: [1:7574435947782144234:0] socket: 120 qp: -1
2025-11-19T13:59:57.455374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435948702367430:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:57.455384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:57.456469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435950977279536:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10104) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435950977279624:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.456906Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435950871491208:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12510) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435950871491290:2117] held: no
2025-11-19T13:59:57.457007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435950871491266:2104] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65011) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435950871491285:2115] held: no
2025-11-19T13:59:57.457210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435950871491299:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52760
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.460282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435950977279631:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-11-19T13:59:57.460304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435950977279632:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-11-19T13:59:57.460684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435950977279633:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-11-19T13:59:57.460924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435950871491300:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52760
2025-11-19T13:59:57.461129Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435950977279624:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.461354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435950871491301:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52760
2025-11-19T13:59:57.461222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435950977279536:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.461250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:57.461257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435950977279536:2049] [node 2] ICP22 created new session: [3:7574435950977279634:2049]
2025-11-19T13:59:57.461265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435950977279634:2049] [node 2] ICS09 handshake done sender: [3:7574435950977279624:2117] self: [3:7574435950977279625:0] peer: [2:7574435951403151022:0] socket: 134 qp: -1
2025-11-19T13:59:57.461272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435950977279634:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:57.461284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:57.461666Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435950871491290:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:57.461782Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435950871491285:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:57.461844Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435950871491277:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.461904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435950871491208:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.461918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:57.461923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435950871491208:2048] [node 1] ICP22 created new session: [5:7574435950871491303:2048]
2025-11-19T13:59:57.461928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435950871491303:2048] [node 1] ICS09 handshake done sender: [5:7574435950871491290:2117] self: [5:7574435950871491292:0] peer: [1:7574435947782144228:0] socket: 126 qp: -1
2025-11-19T13:59:57.461933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435950871491303:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:57.461938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:57.462093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435950871491266:2104] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.462100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:57.462104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435950871491266:2104] [node 4] ICP22 created new session: [5:7574435950871491305:2104]
2025-11-19T13:59:57.462108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435950871491305:2104] [node 4] ICS09 handshake done sender: [5:7574435950871491285:2115] self: [5:7574435950871491288:0] peer: [4:7574435950741519308:0] socket: 114 qp: -1
2025-11-19T13:59:57.462113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435950871491305:2104] [node 4] ICS10 traffic start
2025-11-19T13:59:57.462119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:57.462279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435950871491210:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.462285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:57.462289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435950871491210:2050] [node 3] ICP22 created new session: [5:7574435950871491307:2050]
2025-11-19T13:59:57.462293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435950871491307:2050] [node 3] ICS09 handshake done sender: [5:7574435950871491277:2109] self: [5:7574435950871491289:0] peer: [3:7574435950977279612:0] socket: 91 qp: -1
2025-11-19T13:59:57.462297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435950871491307:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:57.462303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:57.462584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435950871491302:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52760
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.464051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435950741519326:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62461
2025-11-19T13:59:57.464572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435950741519327:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62461
2025-11-19T13:59:57.465081Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435950741519327:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.465108Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435950741519305:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:57.465212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435950741519238:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.465232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:57.465238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435950741519238:2050] [node 3] ICP22 created new session: [4:7574435950741519330:2050]
2025-11-19T13:59:57.465243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435950741519330:2050] [node 3] ICS09 handshake done sender: [4:7574435950741519305:2112] self: [4:7574435950741519319:0] peer: [3:7574435950977279602:0] socket: 110 qp: -1
2025-11-19T13:59:57.465249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435950741519330:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:57.465258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:57.465450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435950741519237:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:57.472648Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435950871491302:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.472534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435950741519325:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62461
2025-11-19T13:59:57.477659Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435950741519318:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:57.477748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435950741519236:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.477771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:57.477780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435950741519236:2048] [node 1] ICP22 created new session: [4:7574435950741519333:2048]
2025-11-19T13:59:57.477787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435950741519333:2048] [node 1] ICS09 handshake done sender: [4:7574435950741519318:2117] self: [4:7574435950741519320:0] peer: [1:7574435947782144226:0] socket: 122 qp: -1
2025-11-19T13:59:57.477797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435950741519333:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:57.477816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:57.479939Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435951403151025:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.479963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435951403151025:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:57.480283Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435951403151041:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.484346Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435950871491272:2107] [node 6] ICP35 event processing took too much time PendingConnection:278(52409514ns) {HandleHandshakeStatus:329(52408265ns) {DropIncomingHandshake:441(453ns) DropOutgoingHandshake:456(287ns) SwitchToState:178(117ns) ProcessPendingSessionEvents:495(31838272ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 31816762ns max# 31719748ns}}}
2025-11-19T13:59:57.484422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435950871491272:2107] [node 6] ICP18 handshake (actor [5:7574435950871491302:2122]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.491784Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435950977279633:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.492660Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435951403151025:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:57.492816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435951403151019:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.492839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:57.492847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435951403151019:2111] [node 4] ICP22 created new session: [2:7574435951403151049:2111]
2025-11-19T13:59:57.492854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435951403151049:2111] [node 4] ICS09 handshake done sender: [2:7574435951403151025:2115] self: [2:7574435951403151024:0] peer: [4:7574435950741519332:0] socket: 143 qp: -1
2025-11-19T13:59:57.492861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435951403151049:2111] [node 4] ICS10 traffic start
2025-11-19T13:59:57.492871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:57.496007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435948702367434:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21517
2025-11-19T13:59:57.496356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435950977279611:2113] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.497863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435950741519237:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10104) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435950741519327:2121] held: no
2025-11-19T13:59:57.498036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435950741519335:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62461
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.497092Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435948702367434:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.497762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435948702367376:2077] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6350) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435948702367398:2096] held: no
2025-11-19T13:59:57.497925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435948702367347:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:57.498401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435948702367436:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21517
2025-11-19T13:59:57.498564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435948702367437:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21517
2025-11-19T13:59:57.498606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435948702367438:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21517
2025-11-19T13:59:57.498787Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435948702367437:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.498795Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435948702367398:2096] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:57.498901Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435948702367382:2082] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:57.498961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435948702367397:2089] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:57.498970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435948702367376:2077] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.498982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:57.498988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435948702367376:2077] [node 5] ICP22 created new session: [6:7574435948702367440:2077]
2025-11-19T13:59:57.498994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435948702367440:2077] [node 5] ICS09 handshake done sender: [6:7574435948702367398:2096] self: [6:7574435948702367424:0] peer: [5:7574435950871491273:0] socket: 102 qp: -1
2025-11-19T13:59:57.499001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435948702367440:2077] [node 5] ICS10 traffic start
2025-11-19T13:59:57.499009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:57.500079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435948702367346:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.500094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:57.500100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435948702367346:2049] [node 2] ICP22 created new session: [6:7574435948702367442:2049]
2025-11-19T13:59:57.500107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435948702367442:2049] [node 2] ICS09 handshake done sender: [6:7574435948702367382:2082] self: [6:7574435948702367425:0] peer: [2:7574435951403151014:0] socket: 86 qp: -1
2025-11-19T13:59:57.500113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435948702367442:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:57.500125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:57.498190Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435950741519327:2121] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:57.499376Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435950741519316:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.499380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435950741519316:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:57.499953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435950741519237:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.499965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:57.499972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435950741519237:2049] [node 2] ICP22 created new session: [4:7574435950741519336:2049]
2025-11-19T13:59:57.499977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435950741519336:2049] [node 2] ICS09 handshake done sender: [4:7574435950741519327:2121] self: [4:7574435950741519332:0] peer: [2:7574435951403151024:0] socket: 145 qp: -1
2025-11-19T13:59:57.499983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435950741519336:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:57.499990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:57.501499Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435950741519316:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:57.501570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435950741519302:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.501583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:57.501589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435950741519302:2110] [node 6] ICP22 created new session: [4:7574435950741519339:2110]
2025-11-19T13:59:57.501594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435950741519339:2110] [node 6] ICS09 handshake done sender: [4:7574435950741519316:2116] self: [4:7574435950741519303:0] peer: [6:7574435948702367439:0] socket: 113 qp: -1
2025-11-19T13:59:57.501599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435950741519339:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:57.501605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:57.504103Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435950977279626:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:57.504120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435950977279626:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.505131Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435950977279626:2118] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:57.505810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435950977279615:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:57.505828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:57.505834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435950977279615:2111] [node 6] ICP22 created new session: [3:7574435950977279638:2111]
2025-11-19T13:59:57.505839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435950977279638:2111] [node 6] ICS09 handshake done sender: [3:7574435950977279626:2118] self: [3:7574435950977279620:0] peer: [6:7574435948702367435:0] socket: 107 qp: -1
2025-11-19T13:59:57.505844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435950977279638:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:57.505852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:57.505655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435948702367444:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21517
2025-11-19T13:59:57.511853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435948702367445:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21517
2025-11-19T13:59:57.512807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435948702367397:2089] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65011) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435948702367437:2121] held: no
2025-11-19T13:59:57.512907Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435948702367347:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15791) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435948702367434:2119] held: no
2025-11-19T13:59:57.512994Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435948702367437:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:57.513325Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435948702367434:2119] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:57.513973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435948702367397:2089] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.513991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:57.513998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435948702367397:2089] [node 4] ICP22 created new session: [6:7574435948702367446:2089]
2025-11-19T13:59:57.514005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435948702367446:2089] [node 4] ICS09 handshake done sender: [6:7574435948702367437:2121] self: [6:7574435948702367439:0] peer: [4:7574435950741519303:0] socket: 116 qp: -1
2025-11-19T13:59:57.514014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435948702367446:2089] [node 4] ICS10 traffic start
2025-11-19T13:59:57.514023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:57.514390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435948702367347:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:57.514403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:57.514408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435948702367347:2050] [node 3] ICP22 created new session: [6:7574435948702367448:2050]
2025-11-19T13:59:57.514413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435948702367448:2050] [node 3] ICS09 handshake done sender: [6:7574435948702367434:2119] self: [6:7574435948702367435:0] peer: [3:7574435950977279620:0] socket: 137 qp: -1
2025-11-19T13:59:57.514418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435948702367448:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:57.514424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:57.525507Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:57.525613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435951403151036:2048] [node 1] ICS01 socket: 99 reason# EndOfStream
2025-11-19T13:59:57.526041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:57.526104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435950977279609:2048] [node 1] ICS01 socket: 89 reason# EndOfStream
2025-11-19T13:59:57.526108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435950977279535:2048] [node 1] ICP30 unregister session Session# [3:7574435950977279609:2048] VirtualId# [3:7574435950977279601:0]
2025-11-19T13:59:57.526112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435950977279609:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.526137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:57.526154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:57.525631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435951403150947:2048] [node 1] ICP30 unregister session Session# [2:7574435951403151036:2048] VirtualId# [2:7574435951403151031:0]
2025-11-19T13:59:57.525645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435951403151036:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.525678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:57.525702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:57.559797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:57.559813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:57.559933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435948702367430:2048] [node 1] ICS01 socket: 120 reason# EndOfStream
2025-11-19T13:59:57.559944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435948702367345:2048] [node 1] ICP30 unregister session Session# [6:7574435948702367430:2048] VirtualId# [6:7574435948702367427:0]
2025-11-19T13:59:57.559954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435948702367430:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.560015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:57.560042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:57.560060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:57.560146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435950741519333:2048] [node 1] ICS01 socket: 122 reason# EndOfStream
2025-11-19T13:59:57.560154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435950741519236:2048] [node 1] ICP30 unregister session Session# [4:7574435950741519333:2048] VirtualId# [4:7574435950741519320:0]
2025-11-19T13:59:57.560162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435950741519333:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.560217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:57.560242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:57.560189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435950871491303:2048] [node 1] ICS01 socket: 126 reason# EndOfStream
2025-11-19T13:59:57.560203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435950871491208:2048] [node 1] ICP30 unregister session Session# [5:7574435950871491303:2048] VirtualId# [5:7574435950871491292:0]
2025-11-19T13:59:57.560207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435950871491303:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.560239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:57.560264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:57.571868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:57.571984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435950977279634:2049] [node 2] ICS01 socket: 134 reason# EndOfStream
2025-11-19T13:59:57.571998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435950977279536:2049] [node 2] ICP30 unregister session Session# [3:7574435950977279634:2049] VirtualId# [3:7574435950977279625:0]
2025-11-19T13:59:57.572010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435950977279634:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.572057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:57.572079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:57.595126Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:57.595250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435950741519336:2049] [node 2] ICS01 socket: 145 reason# EndOfStream
2025-11-19T13:59:57.595263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435950741519237:2049] [node 2] ICP30 unregister session Session# [4:7574435950741519336:2049] VirtualId# [4:7574435950741519332:0]
2025-11-19T13:59:57.595272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435950741519336:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.595334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:57.595361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:57.597904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:57.598141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:57.598207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435948702367442:2049] [node 2] ICS01 socket: 86 reason# EndOfStream
2025-11-19T13:59:57.598213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435948702367346:2049] [node 2] ICP30 unregister session Session# [6:7574435948702367442:2049] VirtualId# [6:7574435948702367425:0]
2025-11-19T13:59:57.598217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435948702367442:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.598243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:57.598264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:57.598010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435950871491293:2049] [node 2] ICS01 socket: 92 reason# EndOfStream
2025-11-19T13:59:57.598023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435950871491209:2049] [node 2] ICP30 unregister session Session# [5:7574435950871491293:2049] VirtualId# [5:7574435950871491276:0]
2025-11-19T13:59:57.598030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435950871491293:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.598080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:57.598105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:57.646971Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:57.647159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435948702367448:2050] [node 3] ICS01 socket: 137 reason# EndOfStream
2025-11-19T13:59:57.647172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435948702367347:2050] [node 3] ICP30 unregister session Session# [6:7574435948702367448:2050] VirtualId# [6:7574435948702367435:0]
2025-11-19T13:59:57.647182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435948702367448:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.647262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:57.647292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:57.647334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:57.647407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435950871491307:2050] [node 3] ICS01 socket: 91 reason# EndOfStream
2025-11-19T13:59:57.647413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435950871491210:2050] [node 3] ICP30 unregister session Session# [5:7574435950871491307:2050] VirtualId# [5:7574435950871491289:0]
2025-11-19T13:59:57.647418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435950871491307:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.647451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:57.647473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:57.660797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:57.660921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435948702367446:2089] [node 4] ICS01 socket: 116 reason# EndOfStream
2025-11-19T13:59:57.660933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435948702367397:2089] [node 4] ICP30 unregister session Session# [6:7574435948702367446:2089] VirtualId# [6:7574435948702367439:0]
2025-11-19T13:59:57.660942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435948702367446:2089] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.660986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:57.661010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:57.661048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T13:59:57.661109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435950871491305:2104] [node 4] ICS01 socket: 114 reason# EndOfStream
2025-11-19T13:59:57.661114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435950871491266:2104] [node 4] ICP30 unregister session Session# [5:7574435950871491305:2104] VirtualId# [5:7574435950871491288:0]
2025-11-19T13:59:57.661117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435950871491305:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.661137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:57.661153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:57.683800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:59:57.683922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435948702367440:2077] [node 5] ICS01 socket: 102 reason# EndOfStream
2025-11-19T13:59:57.683935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435948702367376:2077] [node 5] ICP30 unregister session Session# [6:7574435948702367440:2077] VirtualId# [6:7574435948702367424:0]
2025-11-19T13:59:57.683944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435948702367440:2077] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:57.683981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:59:57.684004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T13:59:57.795970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435948983274550:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:57.795992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435948983274551:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:57.810166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435951874247559:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:57.810184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435951874247560:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:57.831085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435948734423549:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:57.831100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435948734423550:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:57.876137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435951425958901:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:57.876161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435951425958902:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:57.876167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435951425958903:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:57.916033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435951214247375:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:57.916047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435951214247376:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:57.916052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435951214247377:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:57.967928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435949158595189:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:57.967943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435949158595191:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:57.967955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435949158595190:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:58.020370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435955509214700:2075] [node 6] ICP01 ready to work
2025-11-19T13:59:58.028039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435953453562535:2085] [node 4] ICP01 ready to work
2025-11-19T13:59:58.032207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435955509214718:2088] [node 4] ICP01 ready to work
2025-11-19T13:59:58.031978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435953278241915:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:58.033023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435956169214924:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:58.033055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435956169214917:2108] [node 6] ICP01 ready to work
2025-11-19T13:59:58.032919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435953278241911:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:58.033306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435953278241912:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:58.035771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435956169214918:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.044374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435953029390908:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53373
2025-11-19T13:59:58.044431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435953029390909:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53373
2025-11-19T13:59:58.045763Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435953029390908:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.052435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435953029390916:2112] [node 5] ICP01 ready to work
2025-11-19T13:59:58.051978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435953453562524:2079] [node 5] ICP01 ready to work
2025-11-19T13:59:58.052691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435953278241911:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19658684ns) {Configure:104(19658352ns) {ProcessConfigured:135(19391122ns) {StartInitialHandshake:157(19390758ns) {DropHandshakes:469(426ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:476(850ns) {SwitchToState:178(147ns)}}}}}
2025-11-19T13:59:58.059972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435953029390921:2114] [node 4] ICP01 ready to work
2025-11-19T13:59:58.059296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435956169214924:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(26190023ns) {Configure:104(26189495ns) {ProcessConfigured:135(25889431ns) {StartInitialHandshake:157(25888643ns) {DropHandshakes:469(512ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(88ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:476(1516ns) {SwitchToState:178(771ns)}}}}}
2025-11-19T13:59:58.060288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435956169214917:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27144747ns) {Configure:104(27143844ns) {ProcessConfigured:135(26880847ns) {StartInitialHandshake:157(26880399ns) {DropHandshakes:469(454ns) {DropIncomingHandshake:441(86ns) DropOutgoingHandshake:456(38ns)} GenerateSessionVirtualId:988(259ns) PrepareNewSessionHandshake:476(1103ns) {SwitchToState:178(340ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.064014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435956169214933:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36920
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.064095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435956169214934:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36920
2025-11-19T13:59:58.064110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435956169214935:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36920
2025-11-19T13:59:58.066896Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435956169214933:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.067874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435953278241912:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(34496002ns) {Configure:104(34495333ns) {ProcessConfigured:135(34298228ns) {StartInitialHandshake:157(34297371ns) {DropHandshakes:469(305ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(22ns)} GenerateSessionVirtualId:988(88ns) PrepareNewSessionHandshake:476(951ns) {SwitchToState:178(275ns)}}}}}
2025-11-19T13:59:58.068126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435953278241925:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39449
2025-11-19T13:59:58.068139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435953278241926:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39449
2025-11-19T13:59:58.068306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435949158595189:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(34785430ns) {Configure:104(34784775ns) {ProcessConfigured:135(34503667ns) {StartInitialHandshake:157(34503497ns) {DropHandshakes:469(312ns) {DropIncomingHandshake:441(170ns) DropOutgoingHandshake:456(36ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:476(966ns) {SwitchToState:178(767ns)}}}}}
2025-11-19T13:59:58.071909Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435956169214935:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.072095Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435953278241926:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.072239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435953453562562:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41461
2025-11-19T13:59:58.072171Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435956169214934:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.072201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435951874247559:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.072832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435951425958901:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(36712889ns) {Configure:104(36711715ns) {ProcessConfigured:135(36338490ns) {StartInitialHandshake:157(36337633ns) {DropHandshakes:469(515ns) {DropIncomingHandshake:441(81ns) DropOutgoingHandshake:456(76ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:476(995ns) {SwitchToState:178(136ns)}}}}}
2025-11-19T13:59:58.072995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955720926264:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60278
2025-11-19T13:59:58.072338Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435953278241907:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.072345Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435953278241907:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.073253Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435953453562562:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.073297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435949158595189:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.073513Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435953278241919:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.073519Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435953278241919:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.074150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435953278241928:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39449
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.074605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435953453562565:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41461
2025-11-19T13:59:58.074764Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435953278241919:2112] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:58.074921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435953278241915:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.074939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:58.074946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435953278241915:2110] [node 6] ICP22 created new session: [1:7574435953278241930:2110]
2025-11-19T13:59:58.074952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435953278241930:2110] [node 6] ICS09 handshake done sender: [1:7574435953278241919:2112] self: [1:7574435953278241918:0] peer: [6:7574435953453562564:0] socket: 96 qp: -1
2025-11-19T13:59:58.074959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435953278241930:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:58.074969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:58.075992Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435953029390909:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.076736Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435953453562562:2114] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:58.076180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435951214247375:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(39521760ns) {Configure:104(39520312ns) {ProcessConfigured:135(39223524ns) {StartInitialHandshake:157(39222880ns) {DropHandshakes:469(431ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(1595ns) {SwitchToState:178(733ns)}}}}}
2025-11-19T13:59:58.076579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435951214247377:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(40729225ns) {Configure:104(40727154ns) {ProcessConfigured:135(39465219ns) {StartInitialHandshake:157(39464768ns) {DropHandshakes:469(348ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(19ns)} GenerateSessionVirtualId:988(159ns) PrepareNewSessionHandshake:476(583ns) {SwitchToState:178(197ns)}}}}}
2025-11-19T13:59:58.076599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955509214746:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18991
2025-11-19T13:59:58.076663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955509214747:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18991
2025-11-19T13:59:58.076749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955509214748:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18991
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.077253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435956169214938:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36920
2025-11-19T13:59:58.076857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435949158595189:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.076876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:58.076882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435949158595189:2048] [node 1] ICP22 created new session: [6:7574435953453562566:2048]
2025-11-19T13:59:58.076888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435953453562566:2048] [node 1] ICS09 handshake done sender: [6:7574435953453562562:2114] self: [6:7574435953453562564:0] peer: [1:7574435953278241918:0] socket: 97 qp: -1
2025-11-19T13:59:58.076895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435953453562566:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:58.076902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:58.077333Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435955509214746:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.077372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435951214247377:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.077832Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435955509214748:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.077859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435951214247375:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.077483Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435953029390918:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.077492Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435953029390918:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.078017Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435953029390918:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:58.078044Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435953278241923:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.078052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435953278241923:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.078154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435953278241933:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39449
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.078246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955509214754:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18991
2025-11-19T13:59:58.078464Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435955509214746:2114] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:58.078110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435953029390916:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.078124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:58.078129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435953029390916:2112] [node 5] ICP22 created new session: [3:7574435953029390925:2112]
2025-11-19T13:59:58.078134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435953029390925:2112] [node 5] ICS09 handshake done sender: [3:7574435953029390918:2113] self: [3:7574435953029390917:0] peer: [5:7574435955509214752:0] socket: 105 qp: -1
2025-11-19T13:59:58.078139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435953029390925:2112] [node 5] ICS10 traffic start
2025-11-19T13:59:58.078145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:58.078530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435951214247377:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.078543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:58.078548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435951214247377:2050] [node 3] ICP22 created new session: [5:7574435955509214755:2050]
2025-11-19T13:59:58.078553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435955509214755:2050] [node 3] ICS09 handshake done sender: [5:7574435955509214746:2114] self: [5:7574435955509214752:0] peer: [3:7574435953029390917:0] socket: 119 qp: -1
2025-11-19T13:59:58.078558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435955509214755:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:58.078574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:58.079738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435953453562563:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41461
2025-11-19T13:59:58.084093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955509214745:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18991
2025-11-19T13:59:58.086037Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435951874247560:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(65909367ns) {Configure:104(65908233ns) {ProcessConfigured:135(63960286ns) {StartInitialHandshake:157(63959665ns) {DropHandshakes:469(431ns) {DropIncomingHandshake:441(16ns) DropOutgoingHandshake:456(186ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:476(1370ns) {SwitchToState:178(221ns)}}}}}
2025-11-19T13:59:58.097969Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435953278241907:2107] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:58.098110Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435953278241923:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:58.100047Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435955509214745:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.100090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435951214247376:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(36482985ns) {Configure:104(36481527ns) {ProcessConfigured:135(36151685ns) {StartInitialHandshake:157(36151262ns) {DropHandshakes:469(452ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(1513ns) {SwitchToState:178(928ns)}}}}}
2025-11-19T13:59:58.099721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435953029390911:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:58.097013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435955720926260:2105] [node 6] ICP01 ready to work
2025-11-19T13:59:58.099802Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435953278241916:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.099827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435953278241916:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.100026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435948983274550:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.100043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:58.100051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435948983274550:2048] [node 2] ICP22 created new session: [1:7574435953278241936:2048]
2025-11-19T13:59:58.100058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435953278241936:2048] [node 2] ICS09 handshake done sender: [1:7574435953278241907:2107] self: [1:7574435953278241906:0] peer: [2:7574435956169214937:0] socket: 92 qp: -1
2025-11-19T13:59:58.100067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435953278241936:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:58.100076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:58.103166Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435953453562535:2085] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(75035439ns) {Configure:104(75034127ns) {ProcessConfigured:135(74692824ns) {StartInitialHandshake:157(74691511ns) {DropHandshakes:469(1507ns) {DropIncomingHandshake:441(221ns) DropOutgoingHandshake:456(350ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:476(1141ns) {SwitchToState:178(472ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.103679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955720926268:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60278
2025-11-19T13:59:58.103706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955720926271:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60278
2025-11-19T13:59:58.115322Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955720926268:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.115677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435948734423549:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.115737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435948734423549:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(35705862ns) {IncomingHandshake:261(35704985ns) {DropIncomingHandshake:441(102ns) PrepareNewSessionHandshake:476(521ns) {SwitchToState:178(300ns)} GenerateSessionVirtualId:988(593ns) IssueIncomingHandshakeReply:193(35695995ns) {SwitchToState:178(946ns)}}}
2025-11-19T13:59:58.116049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435953278241911:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.116075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:58.116084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435953278241911:2108] [node 5] ICP22 created new session: [1:7574435953278241938:2108]
2025-11-19T13:59:58.116092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435953278241938:2108] [node 5] ICS09 handshake done sender: [1:7574435953278241923:2113] self: [1:7574435953278241921:0] peer: [5:7574435955509214753:0] socket: 117 qp: -1
2025-11-19T13:59:58.116101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435953278241938:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:58.116130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:58.115584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435955720926266:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:58.120035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435951425958902:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(71896908ns) {Configure:104(71895792ns) {ProcessConfigured:135(71572060ns) {StartInitialHandshake:157(71570757ns) {DropHandshakes:469(1262ns) {DropIncomingHandshake:441(218ns) DropOutgoingHandshake:456(189ns)} GenerateSessionVirtualId:988(253ns) PrepareNewSessionHandshake:476(936ns) {SwitchToState:178(116ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.122488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955720926276:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60278
2025-11-19T13:59:58.121705Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435953278241928:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.125400Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435953029390911:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25483087ns) {Configure:104(25482500ns) {ProcessConfigured:135(25158425ns) {StartInitialHandshake:157(25147148ns) {DropHandshakes:469(727ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(119ns)} GenerateSessionVirtualId:988(467ns) PrepareNewSessionHandshake:476(1421ns) {SwitchToState:178(1063ns)}} IncomingHandshake:261(5439ns) {DropIncomingHandshake:441(251ns) PrepareNewSessionHandshake:476(170ns) {SwitchToState:178(126ns)} GenerateSessionVirtualId:988(169ns) IssueIncomingHandshakeReply:193(207ns)}}}}
2025-11-19T13:59:58.123966Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435955509214718:2088] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(85912214ns) {Configure:104(85909999ns) {ProcessConfigured:135(85637886ns) {StartInitialHandshake:157(85636552ns) {DropHandshakes:469(483ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(104ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(690ns) {SwitchToState:178(443ns)}}}}}
2025-11-19T13:59:58.126777Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435955509214747:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.125198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435948983274550:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(25070575ns) {HandleHandshakeStatus:329(25070083ns) {DropIncomingHandshake:441(168ns) DropOutgoingHandshake:456(452ns) SwitchToState:178(306ns) ProcessPendingSessionEvents:495(24966192ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24945907ns max# 24864153ns}}}
2025-11-19T13:59:58.125248Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435953278241933:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.125291Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435953278241916:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.128204Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955720926264:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.128250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955720926277:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60278
2025-11-19T13:59:58.128288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435951425958902:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.129576Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435956169214929:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.129596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435956169214929:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.126822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435951214247376:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.126975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955509214761:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18991
2025-11-19T13:59:58.130028Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435955509214748:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:58.130195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435956169214941:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36920
2025-11-19T13:59:58.130109Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955720926277:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.127978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435953029390932:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53373
2025-11-19T13:59:58.128518Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435953029390909:2109] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:58.128624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435948734423549:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.128644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:58.128652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435948734423549:2048] [node 1] ICP22 created new session: [3:7574435953029390933:2048]
2025-11-19T13:59:58.128659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435953029390933:2048] [node 1] ICS09 handshake done sender: [3:7574435953029390909:2109] self: [3:7574435953029390928:0] peer: [1:7574435953278241913:0] socket: 101 qp: -1
2025-11-19T13:59:58.128666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435953029390933:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:58.128678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:58.130428Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955720926271:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.130642Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435956169214929:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:58.130676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435951214247375:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.130695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:58.130701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435951214247375:2048] [node 1] ICP22 created new session: [5:7574435955509214762:2048]
2025-11-19T13:59:58.130708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435955509214762:2048] [node 1] ICS09 handshake done sender: [5:7574435955509214748:2116] self: [5:7574435955509214753:0] peer: [1:7574435953278241921:0] socket: 121 qp: -1
2025-11-19T13:59:58.130714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435955509214762:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:58.130721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:58.130945Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435956169214927:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.130952Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435956169214927:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.131492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435956169214918:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.131501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:58.131506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435956169214918:2107] [node 5] ICP22 created new session: [2:7574435956169214943:2107]
2025-11-19T13:59:58.131511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435956169214943:2107] [node 5] ICS09 handshake done sender: [2:7574435956169214929:2112] self: [2:7574435956169214928:0] peer: [5:7574435955509214760:0] socket: 107 qp: -1
2025-11-19T13:59:58.131518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435956169214943:2107] [node 5] ICS10 traffic start
2025-11-19T13:59:58.131525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:58.131810Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435956169214934:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.132829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435955720926266:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17105178ns) {Configure:104(17103821ns) {ProcessConfigured:135(16765053ns) {StartInitialHandshake:157(16764221ns) {DropHandshakes:469(1757ns) {DropIncomingHandshake:441(275ns) DropOutgoingHandshake:456(174ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:476(999ns) {SwitchToState:178(449ns)}}}}}
2025-11-19T13:59:58.135879Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955720926276:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.135913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435951425958901:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.136186Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435953453562563:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.136235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435953453562524:2079] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.133111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435953278241911:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(33025558ns) {HandleHandshakeStatus:329(33024357ns) {DropIncomingHandshake:441(676ns) DropOutgoingHandshake:456(426ns) SwitchToState:178(251ns) ProcessPendingSessionEvents:495(16832659ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16800511ns max# 16727067ns}}}
2025-11-19T13:59:58.133165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435953278241911:2108] [node 5] ICP18 handshake (actor [1:7574435953278241933:2118]) is too old
2025-11-19T13:59:58.136037Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435953278241924:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.136042Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435953278241924:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.136742Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435953278241924:2114] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:58.136834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435953278241912:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.136850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:58.136857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435953278241912:2109] [node 4] ICP22 created new session: [1:7574435953278241944:2109]
2025-11-19T13:59:58.136864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435953278241944:2109] [node 4] ICS09 handshake done sender: [1:7574435953278241924:2114] self: [1:7574435953278241922:0] peer: [4:7574435955720926282:0] socket: 139 qp: -1
2025-11-19T13:59:58.136870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435953278241944:2109] [node 4] ICS10 traffic start
2025-11-19T13:59:58.136887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:58.133704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955509214764:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18991
2025-11-19T13:59:58.133830Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435955509214747:2115] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:58.133899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435951214247376:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.133911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:58.133916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435951214247376:2049] [node 2] ICP22 created new session: [5:7574435955509214765:2049]
2025-11-19T13:59:58.133922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435955509214765:2049] [node 2] ICS09 handshake done sender: [5:7574435955509214747:2115] self: [5:7574435955509214760:0] peer: [2:7574435956169214928:0] socket: 120 qp: -1
2025-11-19T13:59:58.133928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435955509214765:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:58.133944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:58.139158Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435956169214927:2111] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:58.139809Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435955509214720:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.139822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435955509214720:2093] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.144083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435951425958903:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(46893970ns) {Configure:104(46892300ns) {ProcessConfigured:135(46534636ns) {StartInitialHandshake:157(46533316ns) {DropHandshakes:469(2020ns) {DropIncomingHandshake:441(295ns) DropOutgoingHandshake:456(209ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:476(1485ns) {SwitchToState:178(843ns)}}}}}
2025-11-19T13:59:58.144129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435951425958903:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.144552Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435953029390923:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.144564Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435953029390923:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.146807Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435953029390923:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:58.145396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435951874247559:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.145418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:58.145427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435951874247559:2048] [node 1] ICP22 created new session: [2:7574435956169214948:2048]
2025-11-19T13:59:58.145434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435956169214948:2048] [node 1] ICS09 handshake done sender: [2:7574435956169214934:2115] self: [2:7574435956169214937:0] peer: [1:7574435953278241906:0] socket: 108 qp: -1
2025-11-19T13:59:58.145441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435956169214948:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:58.145461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:58.145840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435956169214924:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.145846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:58.145852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435956169214924:2110] [node 4] ICP22 created new session: [2:7574435956169214950:2110]
2025-11-19T13:59:58.145857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435956169214950:2110] [node 4] ICS09 handshake done sender: [2:7574435956169214927:2111] self: [2:7574435956169214925:0] peer: [4:7574435955720926278:0] socket: 106 qp: -1
2025-11-19T13:59:58.145862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435956169214950:2110] [node 4] ICS10 traffic start
2025-11-19T13:59:58.145866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:58.146912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435953029390921:2114] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.146927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:58.146933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435953029390921:2114] [node 4] ICP22 created new session: [3:7574435953029390937:2114]
2025-11-19T13:59:58.146938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435953029390937:2114] [node 4] ICS09 handshake done sender: [3:7574435953029390923:2115] self: [3:7574435953029390922:0] peer: [4:7574435955720926284:0] socket: 128 qp: -1
2025-11-19T13:59:58.146944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435953029390937:2114] [node 4] ICS10 traffic start
2025-11-19T13:59:58.146961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:58.147459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435956169214918:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:58.147254Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435953278241925:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.147454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435948983274550:2048] [node 2] ICP18 handshake (actor [1:7574435953278241925:2115]) is too old
2025-11-19T13:59:58.148535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435951425958901:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6554) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435955720926276:2116] held: no
2025-11-19T13:59:58.153405Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435951214247375:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22624028ns) {HandleHandshakeStatus:329(22623038ns) {DropIncomingHandshake:441(167ns) DropOutgoingHandshake:456(1610ns) SwitchToState:178(80ns) ProcessPendingSessionEvents:495(22540091ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22519788ns max# 22439103ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.155129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435953029390939:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53373
2025-11-19T13:59:58.157530Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435953029390939:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.157579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435948734423550:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.153955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435948983274551:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.153986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:58.153993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435948983274551:2049] [node 3] ICP22 created new session: [1:7574435953278241946:2049]
2025-11-19T13:59:58.154000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435953278241946:2049] [node 3] ICS09 handshake done sender: [1:7574435953278241916:2111] self: [1:7574435953278241913:0] peer: [3:7574435953029390928:0] socket: 95 qp: -1
2025-11-19T13:59:58.154005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435953278241946:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:58.154017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:58.154246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435948983274551:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(23250561ns) {HandleHandshakeStatus:329(23249944ns) {DropIncomingHandshake:441(10772ns) DropOutgoingHandshake:456(172ns) SwitchToState:178(215ns) ProcessPendingSessionEvents:495(102302ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 88752ns max# 15110ns}}}
2025-11-19T13:59:58.157789Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435956169214939:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.157798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435956169214939:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.158398Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435956169214939:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.158519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435953029390941:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53373
2025-11-19T13:59:58.158687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435951874247560:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.158708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:58.158713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435951874247560:2049] [node 3] ICP22 created new session: [2:7574435956169214954:2049]
2025-11-19T13:59:58.158718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435956169214954:2049] [node 3] ICS09 handshake done sender: [2:7574435956169214939:2118] self: [2:7574435956169214920:0] peer: [3:7574435953029390940:0] socket: 98 qp: -1
2025-11-19T13:59:58.158724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435956169214954:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:58.158732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:58.159205Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435953029390939:2118] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:58.159320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435948734423550:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.159334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:58.159340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435948734423550:2049] [node 2] ICP22 created new session: [3:7574435953029390942:2049]
2025-11-19T13:59:58.159344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435953029390942:2049] [node 2] ICS09 handshake done sender: [3:7574435953029390939:2118] self: [3:7574435953029390940:0] peer: [2:7574435956169214920:0] socket: 99 qp: -1
2025-11-19T13:59:58.159349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435953029390942:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:58.159356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.163987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435953453562571:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41461
2025-11-19T13:59:58.164075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435953453562572:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41461
2025-11-19T13:59:58.164087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435953453562573:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41461
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.164901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955509214768:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18991
2025-11-19T13:59:58.165445Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435955509214720:2093] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:58.165358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955720926286:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60278
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.165816Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435953453562573:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.165880Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435953453562571:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.165953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435949158595190:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.165966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435953453562535:2085] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.166256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955720926287:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60278
2025-11-19T13:59:58.166231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435956169214947:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36920
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.166664Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435956169214947:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.166696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435956169214924:2110] [node 4] ICP18 handshake (actor [2:7574435956169214947:2120]) is too old
2025-11-19T13:59:58.166908Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435953453562572:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.166943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435949158595191:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.166498Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955720926270:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.166509Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435955720926270:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.166619Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435955720926276:2116] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:58.167102Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435955720926264:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:58.167259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435955509214700:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.167289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:58.167298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435955509214700:2075] [node 6] ICP22 created new session: [5:7574435955509214769:2075]
2025-11-19T13:59:58.167305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435955509214769:2075] [node 6] ICS09 handshake done sender: [5:7574435955509214720:2093] self: [5:7574435955509214701:0] peer: [6:7574435953453562570:0] socket: 102 qp: -1
2025-11-19T13:59:58.167313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435955509214769:2075] [node 6] ICS10 traffic start
2025-11-19T13:59:58.167326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:58.167117Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435953029390930:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.167124Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435953029390930:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.167867Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435953029390930:2116] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:58.167952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435953029390911:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.167969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:58.167976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435953029390911:2110] [node 6] ICP22 created new session: [3:7574435953029390945:2110]
2025-11-19T13:59:58.167981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435953029390945:2110] [node 6] ICS09 handshake done sender: [3:7574435953029390930:2116] self: [3:7574435953029390929:0] peer: [6:7574435953453562576:0] socket: 146 qp: -1
2025-11-19T13:59:58.167987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435953029390945:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:58.167993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:58.169211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435949158595191:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32208) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435953453562572:2119] held: no
2025-11-19T13:59:58.172207Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435956169214932:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.172231Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435956169214932:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.175792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955720926288:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60278
2025-11-19T13:59:58.176000Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435953453562524:2079] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12106) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435953453562563:2115] held: no
2025-11-19T13:59:58.176630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435951425958901:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.176649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:58.176660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435951425958901:2048] [node 1] ICP22 created new session: [4:7574435955720926290:2048]
2025-11-19T13:59:58.176666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435955720926290:2048] [node 1] ICS09 handshake done sender: [4:7574435955720926276:2116] self: [4:7574435955720926282:0] peer: [1:7574435953278241922:0] socket: 140 qp: -1
2025-11-19T13:59:58.176675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435955720926290:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:58.176700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:58.176942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435951425958902:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.176959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:58.176964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435951425958902:2049] [node 2] ICP22 created new session: [4:7574435955720926292:2049]
2025-11-19T13:59:58.176968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435955720926292:2049] [node 2] ICS09 handshake done sender: [4:7574435955720926264:2111] self: [4:7574435955720926278:0] peer: [2:7574435956169214925:0] socket: 112 qp: -1
2025-11-19T13:59:58.176973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435955720926292:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:58.176977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:58.178866Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435955720926268:2112] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:58.179102Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435955720926270:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:58.179301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435951425958903:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.179315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:58.179321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435951425958903:2050] [node 3] ICP22 created new session: [4:7574435955720926294:2050]
2025-11-19T13:59:58.179328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435955720926294:2050] [node 3] ICS09 handshake done sender: [4:7574435955720926268:2112] self: [4:7574435955720926284:0] peer: [3:7574435953029390922:0] socket: 129 qp: -1
2025-11-19T13:59:58.179334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435955720926294:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:58.179341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:58.179617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435955720926260:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.179628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:58.179633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435955720926260:2105] [node 6] ICP22 created new session: [4:7574435955720926296:2105]
2025-11-19T13:59:58.179637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435955720926296:2105] [node 6] ICS09 handshake done sender: [4:7574435955720926270:2113] self: [4:7574435955720926269:0] peer: [6:7574435953453562575:0] socket: 136 qp: -1
2025-11-19T13:59:58.179640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435955720926296:2105] [node 6] ICS10 traffic start
2025-11-19T13:59:58.179644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.180547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435953453562577:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41461
2025-11-19T13:59:58.184092Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435956169214932:2113] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:58.186240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435956169214917:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.186277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:58.186287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435956169214917:2108] [node 6] ICP22 created new session: [2:7574435956169214957:2108]
2025-11-19T13:59:58.186299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435956169214957:2108] [node 6] ICS09 handshake done sender: [2:7574435956169214932:2113] self: [2:7574435956169214926:0] peer: [6:7574435953453562574:0] socket: 148 qp: -1
2025-11-19T13:59:58.186308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435956169214957:2108] [node 6] ICS10 traffic start
2025-11-19T13:59:58.186321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:58.186624Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435955509214768:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.186694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435955509214718:2088] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.187712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435953453562578:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41461
2025-11-19T13:59:58.187816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435953453562579:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41461
2025-11-19T13:59:58.188080Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955720926280:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.188095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435955720926280:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.189952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955509214772:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18991
2025-11-19T13:59:58.190211Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435953453562563:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:58.190292Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435953453562571:2118] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:58.190369Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435953453562572:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.191918Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435955720926280:2118] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:58.191160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435953453562524:2079] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.191180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:58.191188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435953453562524:2079] [node 5] ICP22 created new session: [6:7574435953453562581:2079]
2025-11-19T13:59:58.191209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435953453562581:2079] [node 5] ICS09 handshake done sender: [6:7574435953453562563:2115] self: [6:7574435953453562570:0] peer: [5:7574435955509214701:0] socket: 111 qp: -1
2025-11-19T13:59:58.191217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435953453562581:2079] [node 5] ICS10 traffic start
2025-11-19T13:59:58.191226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:58.191452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435953453562535:2085] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.191460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:58.191465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435953453562535:2085] [node 4] ICP22 created new session: [6:7574435953453562583:2085]
2025-11-19T13:59:58.191469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435953453562583:2085] [node 4] ICS09 handshake done sender: [6:7574435953453562571:2118] self: [6:7574435953453562575:0] peer: [4:7574435955720926269:0] socket: 137 qp: -1
2025-11-19T13:59:58.191474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435953453562583:2085] [node 4] ICS10 traffic start
2025-11-19T13:59:58.191481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:58.191816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435953453562580:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41461
2025-11-19T13:59:58.192244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435955720926266:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.192272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:58.192283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435955720926266:2109] [node 5] ICP22 created new session: [4:7574435955720926300:2109]
2025-11-19T13:59:58.192290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435955720926300:2109] [node 5] ICS09 handshake done sender: [4:7574435955720926280:2118] self: [4:7574435955720926273:0] peer: [5:7574435955509214771:0] socket: 114 qp: -1
2025-11-19T13:59:58.192297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435955720926300:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:58.192304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:58.192328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435949158595190:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14480) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435953453562573:2120] held: no
2025-11-19T13:59:58.192620Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435955509214768:2122] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:58.192766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435955509214718:2088] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.192777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:58.192782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435955509214718:2088] [node 4] ICP22 created new session: [5:7574435955509214773:2088]
2025-11-19T13:59:58.192787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435955509214773:2088] [node 4] ICS09 handshake done sender: [5:7574435955509214768:2122] self: [5:7574435955509214771:0] peer: [4:7574435955720926273:0] socket: 150 qp: -1
2025-11-19T13:59:58.192792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435955509214773:2088] [node 4] ICS10 traffic start
2025-11-19T13:59:58.192817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:58.200695Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435955509214718:2088] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:58.206865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435949158595191:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.206895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:58.206906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435949158595191:2050] [node 3] ICP22 created new session: [6:7574435953453562585:2050]
2025-11-19T13:59:58.206912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435953453562585:2050] [node 3] ICS09 handshake done sender: [6:7574435953453562572:2119] self: [6:7574435953453562576:0] peer: [3:7574435953029390929:0] socket: 103 qp: -1
2025-11-19T13:59:58.206923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435953453562585:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:58.206931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:58.207236Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435953453562573:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:58.211909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435949158595190:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.211936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:58.211946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435949158595190:2049] [node 2] ICP22 created new session: [6:7574435953453562587:2049]
2025-11-19T13:59:58.211953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435953453562587:2049] [node 2] ICS09 handshake done sender: [6:7574435953453562573:2120] self: [6:7574435953453562574:0] peer: [2:7574435956169214926:0] socket: 109 qp: -1
2025-11-19T13:59:58.211963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435953453562587:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:58.211978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:58.223774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:58.223890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435956169214948:2048] [node 1] ICS01 socket: 108 reason# EndOfStream
2025-11-19T13:59:58.223902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435951874247559:2048] [node 1] ICP30 unregister session Session# [2:7574435956169214948:2048] VirtualId# [2:7574435956169214937:0]
2025-11-19T13:59:58.223911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435956169214948:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:58.223964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:58.223988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:58.224456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:58.224539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435953029390933:2048] [node 1] ICS01 socket: 101 reason# EndOfStream
2025-11-19T13:59:58.224547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435948734423549:2048] [node 1] ICP30 unregister session Session# [3:7574435953029390933:2048] VirtualId# [3:7574435953029390928:0]
2025-11-19T13:59:58.224552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435953029390933:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:58.224584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:58.224607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:58.236089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:58.236198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435955509214762:2048] [node 1] ICS07 socket disconnect 121 reason# EndOfStream
2025-11-19T13:59:58.236207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435955509214762:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:58.236264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:58.236287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435955509214762:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:58.236739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:58.236808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435953453562566:2048] [node 1] ICS07 socket disconnect 97 reason# EndOfStream
2025-11-19T13:59:58.236813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435953453562566:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:58.236847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:58.236868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435953453562566:2048] [node 1] ICS15 start handshake
2025-11-19T13:59:58.236550Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:58.236624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435955720926290:2048] [node 1] ICS01 socket: 140 reason# EndOfStream
2025-11-19T13:59:58.236631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435951425958901:2048] [node 1] ICP30 unregister session Session# [4:7574435955720926290:2048] VirtualId# [4:7574435955720926282:0]
2025-11-19T13:59:58.236638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435955720926290:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:58.236675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:58.236700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:58.237609Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435949158595189:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6554) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:58.237621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435953453562566:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:58.237625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435949158595189:2048] [node 1] ICP30 unregister session Session# [6:7574435953453562566:2048] VirtualId# [6:7574435953453562564:0]
2025-11-19T13:59:58.237668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:58.237674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435949158595189:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6554) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:58.237678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6554) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:58.238090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435951214247375:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6554) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:58.238107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435955509214762:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:58.238112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435951214247375:2048] [node 1] ICP30 unregister session Session# [5:7574435955509214762:2048] VirtualId# [5:7574435955509214753:0]
2025-11-19T13:59:58.238176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:58.238182Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435951214247375:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6554) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:58.238184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6554) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:58.243924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435949158595189:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:58.244534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:58.244644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435953029390942:2049] [node 2] ICS01 socket: 99 reason# EndOfStream
2025-11-19T13:59:58.244652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435948734423550:2049] [node 2] ICP30 unregister session Session# [3:7574435953029390942:2049] VirtualId# [3:7574435953029390940:0]
2025-11-19T13:59:58.244660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435953029390942:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:58.244697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:58.244718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:58.252283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435951214247375:2048] [node 1] ICP33 wake up from error state
2025-11-19T13:59:58.252324Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:58.252420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435955509214765:2049] [node 2] ICS01 socket: 120 reason# EndOfStream
2025-11-19T13:59:58.252430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435951214247376:2049] [node 2] ICP30 unregister session Session# [5:7574435955509214765:2049] VirtualId# [5:7574435955509214760:0]
2025-11-19T13:59:58.252437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435955509214765:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:58.252500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:58.252526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:58.255814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:58.255915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435953453562587:2049] [node 2] ICS01 socket: 109 reason# EndOfStream
2025-11-19T13:59:58.255924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435949158595190:2049] [node 2] ICP30 unregister session Session# [6:7574435953453562587:2049] VirtualId# [6:7574435953453562574:0]
2025-11-19T13:59:58.255933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435953453562587:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:58.255995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:58.256026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:58.256070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:58.256137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435955720926292:2049] [node 2] ICS01 socket: 112 reason# EndOfStream
2025-11-19T13:59:58.256142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435951425958902:2049] [node 2] ICP30 unregister session Session# [4:7574435955720926292:2049] VirtualId# [4:7574435955720926278:0]
2025-11-19T13:59:58.256147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435955720926292:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:58.256177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:58.256198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:58.279665Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:58.279797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435953453562585:2050] [node 3] ICS01 socket: 103 reason# EndOfStream
2025-11-19T13:59:58.279809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435949158595191:2050] [node 3] ICP30 unregister session Session# [6:7574435953453562585:2050] VirtualId# [6:7574435953453562576:0]
2025-11-19T13:59:58.279822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435953453562585:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:58.279890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:58.279917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:58.282924Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:58.283040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435955509214755:2050] [node 3] ICS07 socket disconnect 119 reason# EndOfStream
2025-11-19T13:59:58.283049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435955509214755:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:58.283106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:58.283129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435955509214755:2050] [node 3] ICS15 start handshake
2025-11-19T13:59:58.284894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435951214247377:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32208) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T13:59:58.284914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435955509214755:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T13:59:58.284922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435951214247377:2050] [node 3] ICP30 unregister session Session# [5:7574435955509214755:2050] VirtualId# [5:7574435955509214752:0]
2025-11-19T13:59:58.284953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:58.284961Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435951214247377:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32208) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:58.284965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32208) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T13:59:58.296009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435951214247377:2050] [node 3] ICP33 wake up from error state
2025-11-19T13:59:58.313017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:58.313137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435953453562583:2085] [node 4] ICS01 socket: 137 reason# EndOfStream
2025-11-19T13:59:58.313148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435953453562535:2085] [node 4] ICP30 unregister session Session# [6:7574435953453562583:2085] VirtualId# [6:7574435953453562575:0]
2025-11-19T13:59:58.313158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435953453562583:2085] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:58.313218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:58.313245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T13:59:58.582642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435952264844529:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:58.582660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435952264844530:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:58.615884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435952571576198:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:58.615910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435952571576199:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:58.623837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435956013778640:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:58.623850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435956013778641:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:58.671934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435955945006612:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:58.671951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435955945006613:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:58.671955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435955945006614:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:58.719888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435955404696304:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:58.719914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435955404696305:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:58.719920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435955404696306:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:58.723863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435954245336910:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:58.723876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435954245336911:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:58.723880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435954245336912:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.809687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955945006673:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8435
2025-11-19T13:59:58.809710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955945006674:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8435
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.810129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435956013778700:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21558
2025-11-19T13:59:58.808058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435952264844589:2106] [node 4] ICP01 ready to work
2025-11-19T13:59:58.808515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435954245336961:2097] [node 4] ICP01 ready to work
2025-11-19T13:59:58.812767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435952571576264:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:58.812894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435952571576265:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:58.812992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435952571576266:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:58.825149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435952264844599:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:58.825399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435952264844600:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:58.824043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435955404696367:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:58.824151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435955404696368:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:58.828763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435955945006679:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:58.828945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435955945006680:2111] [node 6] ICP01 ready to work
2025-11-19T13:59:58.836461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435954245336977:2100] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.852181Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435952571576198:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(39098081ns) {Configure:104(39096946ns) {ProcessConfigured:135(38834221ns) {StartInitialHandshake:157(38833588ns) {DropHandshakes:469(436ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(46ns)} GenerateSessionVirtualId:988(70ns) PrepareNewSessionHandshake:476(2135ns) {SwitchToState:178(1170ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.854705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435954245336988:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45355
2025-11-19T13:59:58.861840Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955945006674:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.856373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435956013778707:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:58.856552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435956013778708:2109] [node 5] ICP01 ready to work
2025-11-19T13:59:58.856697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435956013778709:2110] [node 6] ICP01 ready to work
2025-11-19T13:59:58.858217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435956013778704:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21558
2025-11-19T13:59:58.858234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435956013778705:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21558
2025-11-19T13:59:58.858245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435956013778706:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21558
2025-11-19T13:59:58.859863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435952264844599:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(34339576ns) {Configure:104(34339048ns) {ProcessConfigured:135(34048440ns) {StartInitialHandshake:157(34047669ns) {DropHandshakes:469(621ns) {DropIncomingHandshake:441(106ns) DropOutgoingHandshake:456(127ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(1229ns) {SwitchToState:178(1031ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.864090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955404696382:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36207
2025-11-19T13:59:58.864172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955404696383:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36207
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.872001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955945006692:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8435
2025-11-19T13:59:58.872073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955945006693:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8435
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.877062Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435956013778705:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.882516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435952571576275:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6183
2025-11-19T13:59:58.882563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435952571576276:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6183
2025-11-19T13:59:58.882595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435952571576277:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6183
2025-11-19T13:59:58.883888Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955945006693:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-11-19T13:59:58.884080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435955945006614:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.884359Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435956013778713:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.884370Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435956013778713:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.886663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435952264844607:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21055
2025-11-19T13:59:58.886718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435952264844608:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21055
2025-11-19T13:59:58.884205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955945006695:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8435
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.887274Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435955404696382:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.891010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955404696384:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36207
2025-11-19T13:59:58.891425Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435954245336988:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.891462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435954245336977:2100] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.891586Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435955404696372:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.891594Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435955404696372:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.895868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435954245336990:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45355
2025-11-19T13:59:58.896651Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435955404696372:2111] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:58.897842Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435954245336988:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:58.895911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955945006697:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8435
2025-11-19T13:59:58.897962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435954245336977:2100] [node 5] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.897983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:58.897991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435954245336977:2100] [node 5] ICP22 created new session: [6:7574435954245336991:2100]
2025-11-19T13:59:58.897998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435954245336991:2100] [node 5] ICS09 handshake done sender: [6:7574435954245336988:2115] self: [6:7574435954245336989:0] peer: [5:7574435955404696371:0] socket: 115 qp: -1
2025-11-19T13:59:58.898007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435954245336991:2100] [node 5] ICS10 traffic start
2025-11-19T13:59:58.898020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:58.898729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435952264844607:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.899052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955945006673:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.899085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435955945006612:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.899326Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435952264844593:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.899350Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435952264844593:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.900012Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435956013778713:2115] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:58.901485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435952571576275:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.901515Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435952571576276:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.901592Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435952571576277:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.902632Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435956013778704:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.902645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435956013778707:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.902656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:58.902661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435956013778707:2108] [node 4] ICP22 created new session: [3:7574435956013778726:2108]
2025-11-19T13:59:58.902666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435956013778726:2108] [node 4] ICS09 handshake done sender: [3:7574435956013778713:2115] self: [3:7574435956013778712:0] peer: [4:7574435955945006696:0] socket: 118 qp: -1
2025-11-19T13:59:58.902670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435956013778726:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:58.902676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:58.904296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955945006692:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.904338Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435955945006693:2118] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:58.904696Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435952264844593:2108] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:58.904878Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435952264844608:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.904995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435952264844589:2106] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.905014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:58.905021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435952264844589:2106] [node 4] ICP22 created new session: [1:7574435952264844612:2106]
2025-11-19T13:59:58.905027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435952264844612:2106] [node 4] ICS09 handshake done sender: [1:7574435952264844593:2108] self: [1:7574435952264844592:0] peer: [4:7574435955945006698:0] socket: 98 qp: -1
2025-11-19T13:59:58.905033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435952264844612:2106] [node 4] ICS10 traffic start
2025-11-19T13:59:58.905043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.905524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435955945006614:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.905540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:58.905546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435955945006614:2050] [node 3] ICP22 created new session: [4:7574435955945006700:2050]
2025-11-19T13:59:58.905551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435955945006700:2050] [node 3] ICS09 handshake done sender: [4:7574435955945006693:2118] self: [4:7574435955945006696:0] peer: [3:7574435956013778712:0] socket: 120 qp: -1
2025-11-19T13:59:58.905556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435955945006700:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:58.905563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:58.905882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955945006699:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8435
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.901868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435955404696368:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.901902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:58.901907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435955404696368:2109] [node 6] ICP22 created new session: [5:7574435955404696387:2109]
2025-11-19T13:59:58.901913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435955404696387:2109] [node 6] ICS09 handshake done sender: [5:7574435955404696372:2111] self: [5:7574435955404696371:0] peer: [6:7574435954245336989:0] socket: 111 qp: -1
2025-11-19T13:59:58.906374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435952571576283:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6183
2025-11-19T13:59:58.901918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435955404696387:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:58.901927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.909024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435952571576259:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.909039Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435952571576259:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.908231Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435956013778700:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.908264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435956013778725:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21558
2025-11-19T13:59:58.908311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435956013778641:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.908470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435956013778640:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.912026Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435955404696383:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.912105Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435955404696384:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.912201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435955404696306:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.912211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435955404696304:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.912720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435954245336994:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45355
2025-11-19T13:59:58.914482Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435952571576259:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:58.912823Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435956013778706:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.912860Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435956013778715:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.912863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435956013778715:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.914486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435956013778731:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21558
2025-11-19T13:59:58.914654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435952571576199:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.914680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:58.914688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435952571576199:2049] [node 3] ICP22 created new session: [2:7574435952571576285:2049]
2025-11-19T13:59:58.914694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435952571576285:2049] [node 3] ICS09 handshake done sender: [2:7574435952571576259:2107] self: [2:7574435952571576258:0] peer: [3:7574435956013778729:0] socket: 102 qp: -1
2025-11-19T13:59:58.914701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435952571576285:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:58.914711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.915228Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435952264844591:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.915242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435952264844591:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.916363Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435952264844602:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.916368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435952264844602:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.917112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435952264844616:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21055
2025-11-19T13:59:58.917473Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435952571576283:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.917584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435952571576198:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.919246Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435952264844591:2107] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:58.920949Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435956013778725:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.921786Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435954245336994:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.921825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435954245336912:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.920108Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955945006695:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.922424Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435952264844602:2112] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:58.922289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955404696392:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36207
2025-11-19T13:59:58.922353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955404696393:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36207
2025-11-19T13:59:58.922379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955404696394:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36207
2025-11-19T13:59:58.922664Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435955404696394:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.922676Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435955404696384:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:58.922755Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435952264844604:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.922765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435952264844604:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.923298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435952264844530:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.923326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:58.923334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435952264844530:2049] [node 3] ICP22 created new session: [1:7574435952264844619:2049]
2025-11-19T13:59:58.923341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435952264844619:2049] [node 3] ICS09 handshake done sender: [1:7574435952264844591:2107] self: [1:7574435952264844590:0] peer: [3:7574435956013778730:0] socket: 96 qp: -1
2025-11-19T13:59:58.923351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435952264844619:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:58.923360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:58.923597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435952264844599:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.923603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:58.923607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435952264844599:2109] [node 5] ICP22 created new session: [1:7574435952264844621:2109]
2025-11-19T13:59:58.923611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435952264844621:2109] [node 5] ICS09 handshake done sender: [1:7574435952264844602:2112] self: [1:7574435952264844601:0] peer: [5:7574435955404696391:0] socket: 132 qp: -1
2025-11-19T13:59:58.923615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435952264844621:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:58.923621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:58.922797Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435955945006612:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16210) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435955945006673:2108] held: no
2025-11-19T13:59:58.923017Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435955945006673:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.924590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435952571576289:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6183
2025-11-19T13:59:58.924407Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435956013778715:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.927049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435955945006612:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.927077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:58.927084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435955945006612:2048] [node 1] ICP22 created new session: [4:7574435955945006703:2048]
2025-11-19T13:59:58.927092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435955945006703:2048] [node 1] ICS09 handshake done sender: [4:7574435955945006673:2108] self: [4:7574435955945006698:0] peer: [1:7574435952264844592:0] socket: 99 qp: -1
2025-11-19T13:59:58.927101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435955945006703:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:58.927111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:58.927393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435955945006613:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.927991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435954245336996:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45355
2025-11-19T13:59:58.928011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435954245336997:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45355
2025-11-19T13:59:58.928341Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435956013778704:2111] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:58.928441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435956013778641:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10008) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435956013778704:2111] held: no
2025-11-19T13:59:58.928541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435956013778641:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.928551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:58.928556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435956013778641:2049] [node 2] ICP22 created new session: [3:7574435956013778735:2049]
2025-11-19T13:59:58.928562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435956013778735:2049] [node 2] ICS09 handshake done sender: [3:7574435956013778704:2111] self: [3:7574435956013778729:0] peer: [2:7574435952571576258:0] socket: 103 qp: -1
2025-11-19T13:59:58.928568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435956013778735:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:58.928574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:58.928811Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435956013778717:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.928816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435956013778717:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.928702Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435952571576270:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.928715Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435952571576270:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.930162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435955404696304:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.930189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:58.930196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435955404696304:2048] [node 1] ICP22 created new session: [5:7574435955404696395:2048]
2025-11-19T13:59:58.930202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435955404696395:2048] [node 1] ICS09 handshake done sender: [5:7574435955404696384:2117] self: [5:7574435955404696391:0] peer: [1:7574435952264844601:0] socket: 133 qp: -1
2025-11-19T13:59:58.930211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435955404696395:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:58.930219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:58.932488Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435952571576270:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:58.932623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435952571576264:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.932643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:58.932648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435952571576264:2108] [node 4] ICP22 created new session: [2:7574435952571576291:2108]
2025-11-19T13:59:58.932653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435952571576291:2108] [node 4] ICS09 handshake done sender: [2:7574435952571576270:2112] self: [2:7574435952571576269:0] peer: [4:7574435955945006707:0] socket: 126 qp: -1
2025-11-19T13:59:58.932658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435952571576291:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:58.932666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:58.933113Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435952264844604:2113] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:58.934808Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435952571576283:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:58.936441Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435952264844616:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.936720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435952264844529:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.936734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:58.936739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435952264844529:2048] [node 2] ICP22 created new session: [1:7574435952264844625:2048]
2025-11-19T13:59:58.936745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435952264844625:2048] [node 2] ICS09 handshake done sender: [1:7574435952264844604:2113] self: [1:7574435952264844603:0] peer: [2:7574435952571576288:0] socket: 127 qp: -1
2025-11-19T13:59:58.936749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435952264844625:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:58.936755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:58.932095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435956013778708:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.932122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:58.932128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435956013778708:2109] [node 5] ICP22 created new session: [3:7574435956013778737:2109]
2025-11-19T13:59:58.932134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435956013778737:2109] [node 5] ICS09 handshake done sender: [3:7574435956013778715:2116] self: [3:7574435956013778714:0] peer: [5:7574435955404696390:0] socket: 119 qp: -1
2025-11-19T13:59:58.932145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435956013778737:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:58.932156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:58.934894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435952571576198:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.934906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:58.934911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435952571576198:2048] [node 1] ICP22 created new session: [2:7574435952571576294:2048]
2025-11-19T13:59:58.934917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435952571576294:2048] [node 1] ICS09 handshake done sender: [2:7574435952571576283:2118] self: [2:7574435952571576288:0] peer: [1:7574435952264844603:0] socket: 139 qp: -1
2025-11-19T13:59:58.934921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435952571576294:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:58.934928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.947003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435954245336998:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45355
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.959171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435955945006708:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8435
2025-11-19T13:59:58.959381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435955945006613:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10008) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435955945006695:2119] held: no
2025-11-19T13:59:58.959629Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435954245336997:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.959679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435954245336961:2097] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.976570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435952264844599:2109] [node 5] ICP18 handshake (actor [1:7574435952264844616:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.978120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435955404696305:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.978175Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435955404696383:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:58.978357Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435955945006695:2119] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:58.978500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435956013778734:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21558
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.982407Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435954245336998:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.982466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435954245336911:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:58.983317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435954245337001:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45355
2025-11-19T13:59:58.982561Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435952571576274:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.982575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435952571576274:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.983523Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435952571576274:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:58.985098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435952571576266:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.985120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:58.985133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435952571576266:2110] [node 6] ICP22 created new session: [2:7574435952571576297:2110]
2025-11-19T13:59:58.985139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435952571576297:2110] [node 6] ICS09 handshake done sender: [2:7574435952571576274:2114] self: [2:7574435952571576273:0] peer: [6:7574435954245337000:0] socket: 122 qp: -1
2025-11-19T13:59:58.985145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435952571576297:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:58.985153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:58.993213Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955945006688:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.993229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435955945006688:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:58.993654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435955945006613:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.993673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:58.993684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435955945006613:2049] [node 2] ICP22 created new session: [4:7574435955945006709:2049]
2025-11-19T13:59:58.993693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435955945006709:2049] [node 2] ICS09 handshake done sender: [4:7574435955945006695:2119] self: [4:7574435955945006707:0] peer: [2:7574435952571576269:0] socket: 129 qp: -1
2025-11-19T13:59:58.993704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435955945006709:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:58.993715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:58.983702Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435954245336998:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:58.983775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435954245336911:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:58.983791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:58.983798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435954245336911:2049] [node 2] ICP22 created new session: [6:7574435954245337002:2049]
2025-11-19T13:59:58.983804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435954245337002:2049] [node 2] ICS09 handshake done sender: [6:7574435954245336998:2120] self: [6:7574435954245337000:0] peer: [2:7574435952571576273:0] socket: 125 qp: -1
2025-11-19T13:59:58.983812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435954245337002:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:58.983820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:58.994886Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435956013778700:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:58.994890Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435952571576272:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:58.994906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435952571576272:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:58.996501Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435952571576272:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:58.996609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435952571576265:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:58.996624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:58.996631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435952571576265:2109] [node 5] ICP22 created new session: [2:7574435952571576300:2109]
2025-11-19T13:59:58.996638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435952571576300:2109] [node 5] ICS09 handshake done sender: [2:7574435952571576272:2113] self: [2:7574435952571576271:0] peer: [5:7574435955404696398:0] socket: 124 qp: -1
2025-11-19T13:59:58.996643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435952571576300:2109] [node 5] ICS10 traffic start
2025-11-19T13:59:58.996655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:58.996387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435954245337005:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45355
2025-11-19T13:59:58.996411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435954245337006:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45355
2025-11-19T13:59:59.003792Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435956013778717:2117] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:59.004106Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435954245336994:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:59.004429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435956013778640:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.004452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:59.004460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435956013778640:2048] [node 1] ICP22 created new session: [3:7574435960308746038:2048]
2025-11-19T13:59:59.004468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435960308746038:2048] [node 1] ICS09 handshake done sender: [3:7574435956013778700:2107] self: [3:7574435956013778730:0] peer: [1:7574435952264844590:0] socket: 101 qp: -1
2025-11-19T13:59:59.004475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435960308746038:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:59.004486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:59.004858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435956013778709:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.004869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:59.004873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435956013778709:2110] [node 6] ICP22 created new session: [3:7574435960308746040:2110]
2025-11-19T13:59:59.004876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435960308746040:2110] [node 6] ICS09 handshake done sender: [3:7574435956013778717:2117] self: [3:7574435956013778716:0] peer: [6:7574435954245336995:0] socket: 109 qp: -1
2025-11-19T13:59:59.004880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435960308746040:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:59.004886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:59.004224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435954245336912:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.004245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:59.004251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435954245336912:2050] [node 3] ICP22 created new session: [6:7574435958540304303:2050]
2025-11-19T13:59:59.004257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435958540304303:2050] [node 3] ICS09 handshake done sender: [6:7574435954245336994:2117] self: [6:7574435954245336995:0] peer: [3:7574435956013778716:0] socket: 141 qp: -1
2025-11-19T13:59:59.004264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435958540304303:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:59.004274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:59.005880Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435955945006688:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.006389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435955404696306:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.006410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:59.006415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435955404696306:2050] [node 3] ICP22 created new session: [5:7574435959699663695:2050]
2025-11-19T13:59:59.006420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435959699663695:2050] [node 3] ICS09 handshake done sender: [5:7574435955404696383:2116] self: [5:7574435955404696390:0] peer: [3:7574435956013778714:0] socket: 121 qp: -1
2025-11-19T13:59:59.006425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435959699663695:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:59.006431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:59.006567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435955404696306:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:59.006760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435955404696397:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36207
2025-11-19T13:59:59.007031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435959699663697:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36207
2025-11-19T13:59:59.007289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435955945006680:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.007305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:59.007309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435955945006680:2111] [node 6] ICP22 created new session: [4:7574435960239974008:2111]
2025-11-19T13:59:59.007314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435960239974008:2111] [node 6] ICS09 handshake done sender: [4:7574435955945006688:2116] self: [4:7574435955945006687:0] peer: [6:7574435954245336999:0] socket: 152 qp: -1
2025-11-19T13:59:59.007318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435960239974008:2111] [node 6] ICS10 traffic start
2025-11-19T13:59:59.007324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.008047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435956559811923:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21055
2025-11-19T13:59:59.008089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435956559811924:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21055
2025-11-19T13:59:59.008749Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435955404696397:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.008774Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435955404696394:2120] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:59.010334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435955404696305:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.010351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:59.010357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435955404696305:2049] [node 2] ICP22 created new session: [5:7574435959699663699:2049]
2025-11-19T13:59:59.010362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435959699663699:2049] [node 2] ICS09 handshake done sender: [5:7574435955404696394:2120] self: [5:7574435955404696398:0] peer: [2:7574435952571576271:0] socket: 149 qp: -1
2025-11-19T13:59:59.010368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435959699663699:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:59.010373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:59.010666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435955404696367:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.012225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435956866543598:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6183
2025-11-19T13:59:59.015066Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435955945006686:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.015077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435955945006686:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:59.015967Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435952571576265:2109] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T13:59:59.016081Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435955945006686:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:59.017460Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435956559811923:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.016193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435955945006679:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.016209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:59.016216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435955945006679:2110] [node 5] ICP22 created new session: [4:7574435960239974011:2110]
2025-11-19T13:59:59.016222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435960239974011:2110] [node 5] ICS09 handshake done sender: [4:7574435955945006686:2115] self: [4:7574435955945006685:0] peer: [5:7574435959699663701:0] socket: 151 qp: -1
2025-11-19T13:59:59.016227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435960239974011:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:59.016236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:59.018770Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435952264844606:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.018780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435952264844606:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:59.016691Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435954245336996:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.017921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435954245336961:2097] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62240) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435954245336997:2119] held: no
2025-11-19T13:59:59.018083Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435954245336997:2119] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:59.019826Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435952264844606:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:59.018436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435954245336961:2097] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.018445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:59.018450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435954245336961:2097] [node 4] ICP22 created new session: [6:7574435958540304305:2097]
2025-11-19T13:59:59.018454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435958540304305:2097] [node 4] ICS09 handshake done sender: [6:7574435954245336997:2119] self: [6:7574435954245336999:0] peer: [4:7574435955945006687:0] socket: 154 qp: -1
2025-11-19T13:59:59.018459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435958540304305:2097] [node 4] ICS10 traffic start
2025-11-19T13:59:59.018465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:59.018606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435954245336910:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:59.020002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435952264844600:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.020015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:59.020020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435952264844600:2110] [node 6] ICP22 created new session: [1:7574435956559811927:2110]
2025-11-19T13:59:59.020026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435956559811927:2110] [node 6] ICS09 handshake done sender: [1:7574435952264844606:2114] self: [1:7574435952264844605:0] peer: [6:7574435958540304307:0] socket: 117 qp: -1
2025-11-19T13:59:59.020030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435956559811927:2110] [node 6] ICS10 traffic start
2025-11-19T13:59:59.020038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.022049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435955404696367:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62240) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435955404696397:2121] held: no
2025-11-19T13:59:59.022220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435959699663702:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36207
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.022679Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435955404696397:2121] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:59.022740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435955404696367:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.022751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:59.022756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435955404696367:2108] [node 4] ICP22 created new session: [5:7574435959699663703:2108]
2025-11-19T13:59:59.022761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435959699663703:2108] [node 4] ICS09 handshake done sender: [5:7574435955404696397:2121] self: [5:7574435959699663701:0] peer: [4:7574435955945006685:0] socket: 157 qp: -1
2025-11-19T13:59:59.022765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435959699663703:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:59.022771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:59.022420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435954245336910:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16210) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435954245336996:2118] held: no
2025-11-19T13:59:59.022568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435958540304308:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45355
2025-11-19T13:59:59.023248Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435954245336996:2118] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:59.023293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435954245336910:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.023299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:59.023304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435954245336910:2048] [node 1] ICP22 created new session: [6:7574435958540304309:2048]
2025-11-19T13:59:59.023308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435958540304309:2048] [node 1] ICS09 handshake done sender: [6:7574435954245336996:2118] self: [6:7574435958540304307:0] peer: [1:7574435952264844605:0] socket: 153 qp: -1
2025-11-19T13:59:59.023313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435958540304309:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:59.023318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:59.027363Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T13:59:59.027440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435952571576294:2048] [node 1] ICS01 socket: 139 reason# EndOfStream
2025-11-19T13:59:59.027446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435952571576198:2048] [node 1] ICP30 unregister session Session# [2:7574435952571576294:2048] VirtualId# [2:7574435952571576288:0]
2025-11-19T13:59:59.027451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435952571576294:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:59.027479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:59.027499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:59.027885Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T13:59:59.027950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435960308746038:2048] [node 1] ICS01 socket: 101 reason# EndOfStream
2025-11-19T13:59:59.027956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435956013778640:2048] [node 1] ICP30 unregister session Session# [3:7574435960308746038:2048] VirtualId# [3:7574435956013778730:0]
2025-11-19T13:59:59.027960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435960308746038:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:59.027994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:59.028010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:59.036445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T13:59:59.036545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435955945006703:2048] [node 1] ICS01 socket: 99 reason# EndOfStream
2025-11-19T13:59:59.036555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435955945006612:2048] [node 1] ICP30 unregister session Session# [4:7574435955945006703:2048] VirtualId# [4:7574435955945006698:0]
2025-11-19T13:59:59.036564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435955945006703:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:59.036607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:59.036634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:59.036673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T13:59:59.036737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435955404696395:2048] [node 1] ICS01 socket: 133 reason# EndOfStream
2025-11-19T13:59:59.036741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435955404696304:2048] [node 1] ICP30 unregister session Session# [5:7574435955404696395:2048] VirtualId# [5:7574435955404696391:0]
2025-11-19T13:59:59.036745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435955404696395:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:59.036771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:59.036793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:59.036820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T13:59:59.036882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435958540304309:2048] [node 1] ICS01 socket: 153 reason# EndOfStream
2025-11-19T13:59:59.036887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435954245336910:2048] [node 1] ICP30 unregister session Session# [6:7574435958540304309:2048] VirtualId# [6:7574435958540304307:0]
2025-11-19T13:59:59.036891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435958540304309:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:59.036923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T13:59:59.036942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T13:59:59.040399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T13:59:59.040485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435956013778735:2049] [node 2] ICS01 socket: 103 reason# EndOfStream
2025-11-19T13:59:59.040492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435956013778641:2049] [node 2] ICP30 unregister session Session# [3:7574435956013778735:2049] VirtualId# [3:7574435956013778729:0]
2025-11-19T13:59:59.040498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435956013778735:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:59.040533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:59.040553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:59.055874Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T13:59:59.056010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435954245337002:2049] [node 2] ICS01 socket: 125 reason# EndOfStream
2025-11-19T13:59:59.056023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435954245336911:2049] [node 2] ICP30 unregister session Session# [6:7574435954245337002:2049] VirtualId# [6:7574435954245337000:0]
2025-11-19T13:59:59.056034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435954245337002:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:59.056090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:59.056118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:59.055869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T13:59:59.056028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435955945006709:2049] [node 2] ICS01 socket: 129 reason# EndOfStream
2025-11-19T13:59:59.056037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435955945006613:2049] [node 2] ICP30 unregister session Session# [4:7574435955945006709:2049] VirtualId# [4:7574435955945006707:0]
2025-11-19T13:59:59.056044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435955945006709:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:59.056092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:59.056118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:59.056157Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T13:59:59.056220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435959699663699:2049] [node 2] ICS01 socket: 149 reason# EndOfStream
2025-11-19T13:59:59.056224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435955404696305:2049] [node 2] ICP30 unregister session Session# [5:7574435959699663699:2049] VirtualId# [5:7574435955404696398:0]
2025-11-19T13:59:59.056228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435959699663699:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:59.056260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T13:59:59.056281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T13:59:59.063813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T13:59:59.063989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435958540304303:2050] [node 3] ICS01 socket: 141 reason# EndOfStream
2025-11-19T13:59:59.064000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435954245336912:2050] [node 3] ICP30 unregister session Session# [6:7574435958540304303:2050] VirtualId# [6:7574435954245336995:0]
2025-11-19T13:59:59.064011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435958540304303:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:59.064075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:59.064101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:59.063880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T13:59:59.064033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435959699663695:2050] [node 3] ICS01 socket: 121 reason# EndOfStream
2025-11-19T13:59:59.064040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435955404696306:2050] [node 3] ICP30 unregister session Session# [5:7574435959699663695:2050] VirtualId# [5:7574435955404696390:0]
2025-11-19T13:59:59.064046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435959699663695:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:59.064088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T13:59:59.064112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T13:59:59.078631Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T13:59:59.078740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435958540304305:2097] [node 4] ICS01 socket: 154 reason# EndOfStream
2025-11-19T13:59:59.078751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435954245336961:2097] [node 4] ICP30 unregister session Session# [6:7574435958540304305:2097] VirtualId# [6:7574435954245336999:0]
2025-11-19T13:59:59.078763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435958540304305:2097] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:59.078806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T13:59:59.078833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T13:59:59.091987Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T13:59:59.092448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435954245336991:2100] [node 5] ICS01 socket: 115 reason# EndOfStream
2025-11-19T13:59:59.092468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435954245336977:2100] [node 5] ICP30 unregister session Session# [6:7574435954245336991:2100] VirtualId# [6:7574435954245336989:0]
2025-11-19T13:59:59.092489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435954245336991:2100] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T13:59:59.092545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T13:59:59.092573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T13:59:59.536175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435958353175907:2048] [node 2] ICP01 ready to work
2025-11-19T13:59:59.536192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435958353175908:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:59.608313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435958151726137:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:59.608331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435958151726138:2049] [node 3] ICP01 ready to work
2025-11-19T13:59:59.684064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435958643143174:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:59.684083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435958643143175:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:59.771789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435957556777284:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:59.771820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435957556777285:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:59.771825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435957556777286:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:59.785166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435958228014296:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:59.785189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435958228014297:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:59.785194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435958228014298:2050] [node 3] ICP01 ready to work
2025-11-19T13:59:59.835965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435957342607020:2048] [node 1] ICP01 ready to work
2025-11-19T13:59:59.835982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435957342607021:2049] [node 2] ICP01 ready to work
2025-11-19T13:59:59.835987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435957342607022:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.881916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435958151726197:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52240
2025-11-19T13:59:59.884276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435958353175970:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.884823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435958151726201:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:59.884921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435958151726202:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:59.885033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435958151726203:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:59.886084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435958151726204:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52240
2025-11-19T13:59:59.886699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435957342607064:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44337
2025-11-19T13:59:59.885117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435958643143238:2108] [node 4] ICP01 ready to work
2025-11-19T13:59:59.888246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435958643143246:2110] [node 5] ICP01 ready to work
2025-11-19T13:59:59.888323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435958643143247:2112] [node 6] ICP01 ready to work
2025-11-19T13:59:59.886584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435957556777345:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36187
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.891467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435957342607067:2093] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.893408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435957342607073:2096] [node 5] ICP01 ready to work
2025-11-19T13:59:59.890269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435958228014361:2109] [node 6] ICP01 ready to work
2025-11-19T13:59:59.892100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435957556777346:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36187
2025-11-19T13:59:59.892800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435957556777347:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36187
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.896239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435958643143254:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16953
2025-11-19T13:59:59.898428Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435958643143254:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.898560Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435958151726204:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.897892Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435957556777346:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.898655Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435957556777345:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.898673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435957556777286:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:59.899043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435957556777285:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:59.899394Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435957342607064:2091] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.899809Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435957556777347:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.899703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435957342607021:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:59.903887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435958643143253:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16953
2025-11-19T13:59:59.904353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435958228014366:2110] [node 4] ICP01 ready to work
2025-11-19T13:59:59.908048Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435958643143240:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.908067Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435958643143240:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T13:59:59.908123Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435958151726206:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.908138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435958151726206:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.920281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435958353175977:2108] [node 5] ICP01 ready to work
2025-11-19T13:59:59.920518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435958353175981:2109] [node 4] ICP01 ready to work
2025-11-19T13:59:59.922426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435958353175978:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36594
2025-11-19T13:59:59.922440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435958353175979:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36594
2025-11-19T13:59:59.922491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435958353175980:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36594
2025-11-19T13:59:59.923086Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435958151726208:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.923102Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435958151726208:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:59.923727Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435958151726208:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.924146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435958643143256:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16953
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.924378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435957342607073:2096] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(29896112ns) {Configure:104(29893579ns) {ProcessConfigured:135(29585137ns) {StartInitialHandshake:157(29584753ns) {DropHandshakes:469(476ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(40ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:476(1770ns) {SwitchToState:178(1023ns)}}}}}
2025-11-19T13:59:59.924938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435957342607090:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44337
2025-11-19T13:59:59.927784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435957556777352:2112] [node 6] ICP01 ready to work
2025-11-19T13:59:59.925072Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435958228014361:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25176264ns) {Configure:104(25174818ns) {ProcessConfigured:135(24827100ns) {StartInitialHandshake:157(24826665ns) {DropHandshakes:469(586ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(283ns) PrepareNewSessionHandshake:476(894ns) {SwitchToState:178(789ns)}}}}}
2025-11-19T13:59:59.926499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435958228014374:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53514
2025-11-19T13:59:59.932194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435958151726202:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.932225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:59.932235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435958151726202:2109] [node 6] ICP22 created new session: [2:7574435958151726220:2109]
2025-11-19T13:59:59.932245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435958151726220:2109] [node 6] ICS09 handshake done sender: [2:7574435958151726208:2113] self: [2:7574435958151726207:0] peer: [6:7574435957342607081:0] socket: 108 qp: -1
2025-11-19T13:59:59.932253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435958151726220:2109] [node 6] ICS10 traffic start
2025-11-19T13:59:59.932278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:59.932899Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435958643143253:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.931978Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435958353175979:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.934896Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435958643143240:2109] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:59.934933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435958228014375:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53514
2025-11-19T13:59:59.935013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435958643143238:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.935033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:59.935039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435958643143238:2108] [node 4] ICP22 created new session: [3:7574435958643143259:2108]
2025-11-19T13:59:59.935044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435958643143259:2108] [node 4] ICS09 handshake done sender: [3:7574435958643143240:2109] self: [3:7574435958643143239:0] peer: [4:7574435957556777353:0] socket: 112 qp: -1
2025-11-19T13:59:59.935052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435958643143259:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:59.935060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.935699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435958228014376:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53514
2025-11-19T13:59:59.934478Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435958151726197:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.934567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435958151726137:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:59.936195Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435958151726206:2112] [node 4] ICH04 handshake succeeded
2025-11-19T13:59:59.936373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435958151726201:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.936391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T13:59:59.936397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435958151726201:2108] [node 4] ICP22 created new session: [2:7574435958151726225:2108]
2025-11-19T13:59:59.936403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435958151726225:2108] [node 4] ICS09 handshake done sender: [2:7574435958151726206:2112] self: [2:7574435958151726205:0] peer: [4:7574435957556777354:0] socket: 107 qp: -1
2025-11-19T13:59:59.936409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435958151726225:2108] [node 4] ICS10 traffic start
2025-11-19T13:59:59.936417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T13:59:59.936159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435957342607020:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(26994274ns) {Configure:104(26993387ns) {ProcessConfigured:135(26700357ns) {StartInitialHandshake:157(26699793ns) {DropHandshakes:469(624ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(38ns)} GenerateSessionVirtualId:988(123ns) PrepareNewSessionHandshake:476(1141ns) {SwitchToState:178(733ns)}}}}}
2025-11-19T13:59:59.936401Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435957342607064:2091] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:59.936541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435958353175970:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(40105148ns) {Configure:104(40104125ns) {ProcessConfigured:135(39789836ns) {StartInitialHandshake:157(39789567ns) {DropHandshakes:469(372ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:476(651ns) {SwitchToState:178(254ns)}}}}}
2025-11-19T13:59:59.940058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435957342607021:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.940075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:59.940084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435957342607021:2049] [node 2] ICP22 created new session: [6:7574435957342607099:2049]
2025-11-19T13:59:59.940090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435957342607099:2049] [node 2] ICS09 handshake done sender: [6:7574435957342607064:2091] self: [6:7574435957342607081:0] peer: [2:7574435958151726207:0] socket: 111 qp: -1
2025-11-19T13:59:59.940097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435957342607099:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:59.940116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:59.940255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435957556777360:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.945512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435957556777284:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(43238991ns) {Configure:104(43238496ns) {ProcessConfigured:135(42971470ns) {StartInitialHandshake:157(42970883ns) {DropHandshakes:469(514ns) {DropIncomingHandshake:441(117ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:476(880ns) {SwitchToState:178(449ns)}}}}}
2025-11-19T13:59:59.946114Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435957556777286:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435957556777346:2109] held: no
2025-11-19T13:59:59.946335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435957556777357:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36187
2025-11-19T13:59:59.946378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435957556777358:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36187
2025-11-19T13:59:59.946504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435958228014377:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53514
2025-11-19T13:59:59.946763Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435958353175978:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.946798Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435958353175967:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.946802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435958353175967:2106] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T13:59:59.947628Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435958228014376:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.947651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435958228014298:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:59.947825Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435958643143249:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.947837Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435958643143249:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.948492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435958228014379:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53514
2025-11-19T13:59:59.948632Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435958643143249:2113] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:59.948918Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435958228014376:2117] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:59.949030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435958228014298:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.949046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:59.949051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435958228014298:2050] [node 3] ICP22 created new session: [5:7574435958228014380:2050]
2025-11-19T13:59:59.949057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435958228014380:2050] [node 3] ICS09 handshake done sender: [5:7574435958228014376:2117] self: [5:7574435958228014378:0] peer: [3:7574435958643143248:0] socket: 137 qp: -1
2025-11-19T13:59:59.949063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435958228014380:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:59.949069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.949137Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435958151726202:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(24705261ns) {HandleHandshakeStatus:329(24704101ns) {DropIncomingHandshake:441(767ns) DropOutgoingHandshake:456(554ns) SwitchToState:178(336ns) ProcessPendingSessionEvents:495(9212053ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9194697ns max# 9119509ns}}}
2025-11-19T13:59:59.950751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435958151726227:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52240
2025-11-19T13:59:59.950776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435958151726229:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52240
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.950897Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435958353175967:2106] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:59.951140Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435958353175980:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.952208Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435958228014374:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.952264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435958228014296:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:59.952489Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435958228014377:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.955780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435957556777359:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36187
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.959236Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435958643143256:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.959392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435958643143263:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16953
2025-11-19T13:59:59.959404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435958643143264:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16953
2025-11-19T13:59:59.959419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435958643143174:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.959646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435957342607101:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44337
2025-11-19T13:59:59.964369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435958151726201:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(27893332ns) {HandleHandshakeStatus:329(27892741ns) {DropIncomingHandshake:441(89ns) DropOutgoingHandshake:456(213ns) SwitchToState:178(68ns) ProcessPendingSessionEvents:495(22704208ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22686012ns max# 22606175ns}}}
2025-11-19T13:59:59.964726Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435958151726197:2107] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:59.964988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435958643143246:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.965023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:59.965031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435958643143246:2110] [node 5] ICP22 created new session: [3:7574435958643143267:2110]
2025-11-19T13:59:59.965039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435958643143267:2110] [node 5] ICS09 handshake done sender: [3:7574435958643143249:2113] self: [3:7574435958643143248:0] peer: [5:7574435958228014378:0] socket: 136 qp: -1
2025-11-19T13:59:59.965046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435958643143267:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:59.965055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:59.964995Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435958228014375:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.965025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435958228014297:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:59.967888Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435958353175983:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.967912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435958353175983:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:59.968282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435958353175907:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.968301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:59.968309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435958353175907:2048] [node 2] ICP22 created new session: [1:7574435958353175993:2048]
2025-11-19T13:59:59.968315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435958353175993:2048] [node 2] ICS09 handshake done sender: [1:7574435958353175967:2106] self: [1:7574435958353175966:0] peer: [2:7574435958151726224:0] socket: 102 qp: -1
2025-11-19T13:59:59.968323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435958353175993:2048] [node 2] ICS10 traffic start
2025-11-19T13:59:59.968332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:59.968609Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435958353175974:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.968613Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435958353175974:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.969803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435958228014386:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53514
2025-11-19T13:59:59.970114Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435958228014374:2115] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:59.970194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435958228014296:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.970206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:59.970213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435958228014296:2048] [node 1] ICP22 created new session: [5:7574435958228014387:2048]
2025-11-19T13:59:59.970219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435958228014387:2048] [node 1] ICS09 handshake done sender: [5:7574435958228014374:2115] self: [5:7574435958228014383:0] peer: [1:7574435958353175982:0] socket: 130 qp: -1
2025-11-19T13:59:59.970224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435958228014387:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:59.970232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.971170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435957342607102:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44337
2025-11-19T13:59:59.971986Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435957556777357:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.972032Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435957556777346:2109] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:59.973606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435958643143246:2110] [node 5] ICP35 event processing took too much time PendingConnection:278(22140892ns) {HandleHandshakeStatus:329(22140037ns) {DropIncomingHandshake:441(7833ns) DropOutgoingHandshake:456(203ns) SwitchToState:178(191ns) ProcessPendingSessionEvents:495(124799ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 101591ns max# 24071ns}}}
2025-11-19T13:59:59.969936Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435958353175983:2115] [node 5] ICH04 handshake succeeded
2025-11-19T13:59:59.969966Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435958353175974:2110] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:59.970065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435958353175908:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.970078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:59.970083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435958353175908:2049] [node 3] ICP22 created new session: [1:7574435958353175998:2049]
2025-11-19T13:59:59.970089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435958353175998:2049] [node 3] ICS09 handshake done sender: [1:7574435958353175974:2110] self: [1:7574435958353175972:0] peer: [3:7574435958643143266:0] socket: 125 qp: -1
2025-11-19T13:59:59.970099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435958353175998:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:59.970106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:59.978460Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435957556777345:2108] [node 2] ICH04 handshake succeeded
2025-11-19T13:59:59.979810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435958151726137:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.979835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:59.979843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435958151726137:2048] [node 1] ICP22 created new session: [2:7574435958151726230:2048]
2025-11-19T13:59:59.979850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435958151726230:2048] [node 1] ICS09 handshake done sender: [2:7574435958151726197:2107] self: [2:7574435958151726224:0] peer: [1:7574435958353175966:0] socket: 103 qp: -1
2025-11-19T13:59:59.979858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435958151726230:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:59.979867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:59.980263Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435958151726229:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.980290Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435958151726212:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.980293Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435958151726212:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T13:59:59.981405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435958151726137:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.982316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435957556777286:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.982330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:59.982336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435957556777286:2050] [node 3] ICP22 created new session: [4:7574435957556777371:2050]
2025-11-19T13:59:59.982341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435957556777371:2050] [node 3] ICS09 handshake done sender: [4:7574435957556777346:2109] self: [4:7574435957556777353:0] peer: [3:7574435958643143239:0] socket: 115 qp: -1
2025-11-19T13:59:59.982346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435957556777371:2050] [node 3] ICS10 traffic start
2025-11-19T13:59:59.982364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:59.982664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435957556777370:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36187
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.982842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435957342607103:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44337
2025-11-19T13:59:59.981855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435958353175977:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.981877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:59.981882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435958353175977:2108] [node 5] ICP22 created new session: [1:7574435958353176000:2108]
2025-11-19T13:59:59.981887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435958353176000:2108] [node 5] ICS09 handshake done sender: [1:7574435958353175983:2115] self: [1:7574435958353175982:0] peer: [5:7574435958228014383:0] socket: 126 qp: -1
2025-11-19T13:59:59.981892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435958353176000:2108] [node 5] ICS10 traffic start
2025-11-19T13:59:59.981899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:59.982982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435957556777285:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.982994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T13:59:59.982998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435957556777285:2049] [node 2] ICP22 created new session: [4:7574435957556777374:2049]
2025-11-19T13:59:59.983003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435957556777374:2049] [node 2] ICS09 handshake done sender: [4:7574435957556777345:2108] self: [4:7574435957556777354:0] peer: [2:7574435958151726205:0] socket: 110 qp: -1
2025-11-19T13:59:59.983007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435957556777374:2049] [node 2] ICS10 traffic start
2025-11-19T13:59:59.983012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T13:59:59.985732Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435958151726212:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.986070Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435958643143263:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.986215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435958643143174:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62094) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435958643143256:2117] held: no
2025-11-19T13:59:59.986386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435958643143269:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16953
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T13:59:59.987544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435958353176003:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36594
2025-11-19T13:59:59.987680Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435957342607101:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.987778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435957342607022:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:59.987797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435957342607104:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44337
2025-11-19T13:59:59.987111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435958151726203:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.987127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T13:59:59.987132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435958151726203:2110] [node 5] ICP22 created new session: [2:7574435958151726234:2110]
2025-11-19T13:59:59.987137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435958151726234:2110] [node 5] ICS09 handshake done sender: [2:7574435958151726212:2115] self: [2:7574435958151726211:0] peer: [5:7574435958228014385:0] socket: 134 qp: -1
2025-11-19T13:59:59.987141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435958151726234:2110] [node 5] ICS10 traffic start
2025-11-19T13:59:59.987148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T13:59:59.988246Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435958643143264:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.988279Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435958643143251:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.988282Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435958643143251:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T13:59:59.988748Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435958643143256:2117] [node 1] ICH04 handshake succeeded
2025-11-19T13:59:59.989296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435958151726217:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.989303Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435958151726217:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T13:59:59.988883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435958643143174:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T13:59:59.988897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T13:59:59.988902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435958643143174:2048] [node 1] ICP22 created new session: [3:7574435958643143272:2048]
2025-11-19T13:59:59.988907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435958643143272:2048] [node 1] ICS09 handshake done sender: [3:7574435958643143256:2117] self: [3:7574435958643143266:0] peer: [1:7574435958353175972:0] socket: 128 qp: -1
2025-11-19T13:59:59.988912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435958643143272:2048] [node 1] ICS10 traffic start
2025-11-19T13:59:59.988918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T13:59:59.989196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435958643143175:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:59.990710Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435958643143251:2114] [node 6] ICH04 handshake succeeded
2025-11-19T13:59:59.990824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435958643143247:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.990837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T13:59:59.990842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435958643143247:2112] [node 6] ICP22 created new session: [3:7574435958643143276:2112]
2025-11-19T13:59:59.990847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435958643143276:2112] [node 6] ICS09 handshake done sender: [3:7574435958643143251:2114] self: [3:7574435958643143250:0] peer: [6:7574435957342607105:0] socket: 149 qp: -1
2025-11-19T13:59:59.990850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435958643143276:2112] [node 6] ICS10 traffic start
2025-11-19T13:59:59.990857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T13:59:59.990252Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435958151726217:2116] [node 3] ICH04 handshake succeeded
2025-11-19T13:59:59.990326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435958151726138:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T13:59:59.990341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T13:59:59.990348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435958151726138:2049] [node 3] ICP22 created new session: [2:7574435958151726237:2049]
2025-11-19T13:59:59.990353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435958151726237:2049] [node 3] ICS09 handshake done sender: [2:7574435958151726217:2116] self: [2:7574435958151726216:0] peer: [3:7574435958643143274:0] socket: 148 qp: -1
2025-11-19T13:59:59.990357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435958151726237:2049] [node 3] ICS10 traffic start
2025-11-19T13:59:59.990362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T13:59:59.998951Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435958353176003:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.998895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435957342607022:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14658) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435957342607101:2116] held: no
2025-11-19T13:59:59.999028Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435957342607102:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.999149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435957342607073:2096] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T13:59:59.999338Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435957556777370:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T13:59:59.999400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435957556777284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.000144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435958643143275:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16953
2025-11-19T13:59:59.999523Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435957342607103:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.000482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435958228014297:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4300) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435958228014375:2116] held: no
2025-11-19T14:00:00.000616Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435958228014373:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.000620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435958228014373:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.001364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435958228014390:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53514
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.001551Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435958353175985:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.001563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435958353175985:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.002264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435958353176005:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36594
2025-11-19T14:00:00.002439Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435958353175985:2116] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:00.002532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435958353175981:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.002552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:00.002559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435958353175981:2109] [node 4] ICP22 created new session: [1:7574435958353176007:2109]
2025-11-19T14:00:00.002566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435958353176007:2109] [node 4] ICS09 handshake done sender: [1:7574435958353175985:2116] self: [1:7574435958353175984:0] peer: [4:7574435957556777376:0] socket: 153 qp: -1
2025-11-19T14:00:00.002572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435958353176007:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:00.002581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.003069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435958228014391:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53514
2025-11-19T14:00:00.003440Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435958228014373:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:00.003517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435958228014361:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.003557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:00.003562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435958228014361:2109] [node 6] ICP22 created new session: [5:7574435962522981689:2109]
2025-11-19T14:00:00.003568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435962522981689:2109] [node 6] ICS09 handshake done sender: [5:7574435958228014373:2114] self: [5:7574435958228014364:0] peer: [6:7574435957342607107:0] socket: 122 qp: -1
2025-11-19T14:00:00.003572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435962522981689:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:00.003579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.004191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435957342607067:2093] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.004263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435961637574404:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44337
2025-11-19T14:00:00.004180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435958643143175:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4300) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435958643143264:2119] held: no
2025-11-19T14:00:00.004353Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435958643143264:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:00.004437Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435958228014375:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.004567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435961637574405:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44337
2025-11-19T14:00:00.004637Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435957556777365:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.004650Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435957556777365:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.004612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435958643143175:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.004628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:00.004633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435958643143175:2049] [node 2] ICP22 created new session: [3:7574435962938110574:2049]
2025-11-19T14:00:00.004638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435962938110574:2049] [node 2] ICS09 handshake done sender: [3:7574435958643143264:2119] self: [3:7574435958643143274:0] peer: [2:7574435958151726216:0] socket: 105 qp: -1
2025-11-19T14:00:00.004643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435962938110574:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:00.004649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:00.005375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435957556777284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62094) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435957556777370:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.005921Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435958228014391:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.005934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435958228014297:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.005942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:00.005947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435958228014297:2049] [node 2] ICP22 created new session: [5:7574435962522981691:2049]
2025-11-19T14:00:00.005952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435962522981691:2049] [node 2] ICS09 handshake done sender: [5:7574435958228014375:2116] self: [5:7574435958228014385:0] peer: [2:7574435958151726211:0] socket: 135 qp: -1
2025-11-19T14:00:00.005957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435962522981691:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:00.005963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:00.006176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435958228014366:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.008857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435962522981694:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53514
2025-11-19T14:00:00.010051Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435958228014391:2122] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:00.010316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435958228014366:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.010329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:00.010334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435958228014366:2110] [node 4] ICP22 created new session: [5:7574435962522981695:2110]
2025-11-19T14:00:00.010339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435962522981695:2110] [node 4] ICS09 handshake done sender: [5:7574435958228014391:2122] self: [5:7574435962522981693:0] peer: [4:7574435957556777362:0] socket: 118 qp: -1
2025-11-19T14:00:00.010345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435962522981695:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:00.010351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:00.005609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435961851744673:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36187
2025-11-19T14:00:00.008117Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435957556777363:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.008123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435957556777363:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.008551Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435957556777370:2120] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:00.008747Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435957556777365:2119] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:00.009052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435957556777284:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.009065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:00.009070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435957556777284:2048] [node 1] ICP22 created new session: [4:7574435961851744676:2048]
2025-11-19T14:00:00.009075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435961851744676:2048] [node 1] ICS09 handshake done sender: [4:7574435957556777370:2120] self: [4:7574435957556777376:0] peer: [1:7574435958353175984:0] socket: 160 qp: -1
2025-11-19T14:00:00.009081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435961851744676:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:00.009087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:00.009286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435957556777352:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.009296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:00.009301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435957556777352:2112] [node 6] ICP22 created new session: [4:7574435961851744678:2112]
2025-11-19T14:00:00.009304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435961851744678:2112] [node 6] ICS09 handshake done sender: [4:7574435957556777365:2119] self: [4:7574435957556777364:0] peer: [6:7574435961637574406:0] socket: 159 qp: -1
2025-11-19T14:00:00.009307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435961851744678:2112] [node 6] ICS10 traffic start
2025-11-19T14:00:00.009313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:00.009880Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435957556777363:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:00.011503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435957556777360:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.011515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:00.011520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435957556777360:2113] [node 5] ICP22 created new session: [4:7574435961851744680:2113]
2025-11-19T14:00:00.011524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435961851744680:2113] [node 5] ICS09 handshake done sender: [4:7574435957556777363:2118] self: [4:7574435957556777362:0] peer: [5:7574435962522981693:0] socket: 158 qp: -1
2025-11-19T14:00:00.011529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435961851744680:2113] [node 5] ICS10 traffic start
2025-11-19T14:00:00.011533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:00.012858Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435957342607104:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.024480Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435957342607101:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:00.024586Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435957342607102:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.024681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435957342607073:2096] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2769) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435957342607102:2117] held: no
2025-11-19T14:00:00.024777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435957342607073:2096] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.024792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:00.024798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435957342607073:2096] [node 5] ICP22 created new session: [6:7574435961637574408:2096]
2025-11-19T14:00:00.024804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435961637574408:2096] [node 5] ICS09 handshake done sender: [6:7574435957342607102:2117] self: [6:7574435957342607107:0] peer: [5:7574435958228014364:0] socket: 123 qp: -1
2025-11-19T14:00:00.024813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435961637574408:2096] [node 5] ICS10 traffic start
2025-11-19T14:00:00.024823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:00.025133Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435957342607067:2093] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435957342607103:2118] held: no
2025-11-19T14:00:00.025225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435957342607022:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.025231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:00.025234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435957342607022:2050] [node 3] ICP22 created new session: [6:7574435961637574410:2050]
2025-11-19T14:00:00.025237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435961637574410:2050] [node 3] ICS09 handshake done sender: [6:7574435957342607101:2116] self: [6:7574435957342607105:0] peer: [3:7574435958643143250:0] socket: 150 qp: -1
2025-11-19T14:00:00.025243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435961637574410:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:00.025248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:00.025560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435957342607020:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.025588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435961637574407:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44337
2025-11-19T14:00:00.026100Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435958353176005:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.026136Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435958353175988:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.026140Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435958353175988:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.027204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435961637574413:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44337
2025-11-19T14:00:00.027299Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435958353175988:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:00.027487Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435957342607104:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:00.027566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435958353175970:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.027583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:00.027588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435958353175970:2107] [node 6] ICP22 created new session: [1:7574435962648143307:2107]
2025-11-19T14:00:00.027593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435962648143307:2107] [node 6] ICS09 handshake done sender: [1:7574435958353175988:2117] self: [1:7574435958353175975:0] peer: [6:7574435961637574412:0] socket: 114 qp: -1
2025-11-19T14:00:00.027598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435962648143307:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:00.027605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:00.027964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435957342607020:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.027979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:00.027983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435957342607020:2048] [node 1] ICP22 created new session: [6:7574435961637574414:2048]
2025-11-19T14:00:00.027988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435961637574414:2048] [node 1] ICS09 handshake done sender: [6:7574435957342607104:2119] self: [6:7574435961637574412:0] peer: [1:7574435958353175975:0] socket: 147 qp: -1
2025-11-19T14:00:00.027993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435961637574414:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:00.027997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:00.029267Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435957342607103:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:00.035816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435957342607067:2093] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.035830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:00.035837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435957342607067:2093] [node 4] ICP22 created new session: [6:7574435961637574416:2093]
2025-11-19T14:00:00.035843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435961637574416:2093] [node 4] ICS09 handshake done sender: [6:7574435957342607103:2118] self: [6:7574435961637574406:0] peer: [4:7574435957556777364:0] socket: 161 qp: -1
2025-11-19T14:00:00.035850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435961637574416:2093] [node 4] ICS10 traffic start
2025-11-19T14:00:00.035857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:00.053878Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:00.053980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435958151726230:2048] [node 1] ICS01 socket: 103 reason# EndOfStream
2025-11-19T14:00:00.053991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435958151726137:2048] [node 1] ICP30 unregister session Session# [2:7574435958151726230:2048] VirtualId# [2:7574435958151726224:0]
2025-11-19T14:00:00.053999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435958151726230:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.054032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:00.054052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:00.054460Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:00.054522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435958643143272:2048] [node 1] ICS01 socket: 128 reason# EndOfStream
2025-11-19T14:00:00.054528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435958643143174:2048] [node 1] ICP30 unregister session Session# [3:7574435958643143272:2048] VirtualId# [3:7574435958643143266:0]
2025-11-19T14:00:00.054532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435958643143272:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.054554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:00.054569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:00.066669Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:00.066767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435961637574414:2048] [node 1] ICS07 socket disconnect 147 reason# EndOfStream
2025-11-19T14:00:00.066774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435961637574414:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.066820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:00.066838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435961637574414:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:00.067039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:00.067099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435958228014387:2048] [node 1] ICS01 socket: 130 reason# EndOfStream
2025-11-19T14:00:00.067105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435958228014296:2048] [node 1] ICP30 unregister session Session# [5:7574435958228014387:2048] VirtualId# [5:7574435958228014383:0]
2025-11-19T14:00:00.067110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435958228014387:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.067135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:00.067156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:00.067188Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:00.067260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435961851744676:2048] [node 1] ICS01 socket: 160 reason# EndOfStream
2025-11-19T14:00:00.067264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435957556777284:2048] [node 1] ICP30 unregister session Session# [4:7574435961851744676:2048] VirtualId# [4:7574435957556777376:0]
2025-11-19T14:00:00.067268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435961851744676:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.067291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:00.067309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:00.067910Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435957342607020:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62094) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:00.067926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435961637574414:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:00.067931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435957342607020:2048] [node 1] ICP30 unregister session Session# [6:7574435961637574414:2048] VirtualId# [6:7574435961637574412:0]
2025-11-19T14:00:00.067983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:00.067990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435957342607020:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62094) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:00.067993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62094) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:00.076351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:00.076447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435962938110574:2049] [node 2] ICS01 socket: 105 reason# EndOfStream
2025-11-19T14:00:00.076455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435958643143175:2049] [node 2] ICP30 unregister session Session# [3:7574435962938110574:2049] VirtualId# [3:7574435958643143274:0]
2025-11-19T14:00:00.076463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435962938110574:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.076494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:00.076513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:00.081422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435957342607020:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:00.086863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:00.086969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435957556777374:2049] [node 2] ICS01 socket: 110 reason# EndOfStream
2025-11-19T14:00:00.086978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435957556777285:2049] [node 2] ICP30 unregister session Session# [4:7574435957556777374:2049] VirtualId# [4:7574435957556777354:0]
2025-11-19T14:00:00.086986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435957556777374:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.087044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:00.087069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:00.087109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:00.087169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435962522981691:2049] [node 2] ICS01 socket: 135 reason# EndOfStream
2025-11-19T14:00:00.087174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435958228014297:2049] [node 2] ICP30 unregister session Session# [5:7574435962522981691:2049] VirtualId# [5:7574435958228014385:0]
2025-11-19T14:00:00.087178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435962522981691:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.087202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:00.087232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:00.087449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:00.087506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435957342607099:2049] [node 2] ICS07 socket disconnect 111 reason# EndOfStream
2025-11-19T14:00:00.087509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435957342607099:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.087535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:00.087554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435957342607099:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:00.088107Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435957342607021:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4300) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:00.088118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435957342607099:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:00.088123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435957342607021:2049] [node 2] ICP30 unregister session Session# [6:7574435957342607099:2049] VirtualId# [6:7574435957342607081:0]
2025-11-19T14:00:00.088164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:00.088170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435957342607021:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4300) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:00.088172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4300) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:00.097960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435957342607021:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:00.104082Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:00.104192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435958228014380:2050] [node 3] ICS01 socket: 137 reason# EndOfStream
2025-11-19T14:00:00.104200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435958228014298:2050] [node 3] ICP30 unregister session Session# [5:7574435958228014380:2050] VirtualId# [5:7574435958228014378:0]
2025-11-19T14:00:00.104209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435958228014380:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.104262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:00.104284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:00.104319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:00.104374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435961637574410:2050] [node 3] ICS01 socket: 150 reason# EndOfStream
2025-11-19T14:00:00.104378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435957342607022:2050] [node 3] ICP30 unregister session Session# [6:7574435961637574410:2050] VirtualId# [6:7574435957342607105:0]
2025-11-19T14:00:00.104382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435961637574410:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.104406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:00.104427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:00.124654Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:00.124763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435961637574416:2093] [node 4] ICS01 socket: 161 reason# EndOfStream
2025-11-19T14:00:00.124774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435957342607067:2093] [node 4] ICP30 unregister session Session# [6:7574435961637574416:2093] VirtualId# [6:7574435961637574406:0]
2025-11-19T14:00:00.124781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435961637574416:2093] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.124830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:00.124854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:00.208209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435960788655480:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:00.208225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435960788655481:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:00.256141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435963269256244:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:00.256157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435963269256245:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:00.295848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435962902658080:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:00.295862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435962902658081:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:00.356044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435963432975197:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:00.356065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435963432975198:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:00.356071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435963432975199:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:00.388005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435964479930901:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:00.388021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435964479930902:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:00.388026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435964479930903:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:00.428008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435963829102049:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:00.428027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435963829102050:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:00.428032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435963829102051:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:00.457458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435960788655530:2096] [node 5] ICP01 ready to work
2025-11-19T14:00:00.460171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435963269256280:2080] [node 5] ICP01 ready to work
2025-11-19T14:00:00.460257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435963269256281:2082] [node 6] ICP01 ready to work
2025-11-19T14:00:00.463973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435962902658116:2079] [node 5] ICP01 ready to work
2025-11-19T14:00:00.466040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435960788655550:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.472783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435963269256305:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
2025-11-19T14:00:00.473971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435960788655538:2100] [node 6] ICP01 ready to work
2025-11-19T14:00:00.480105Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435963269256281:2082] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(18007913ns) {Configure:104(18007120ns) {ProcessConfigured:135(17725537ns) {StartInitialHandshake:157(17724539ns) {DropHandshakes:469(448ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:476(1923ns) {SwitchToState:178(772ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.481927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435963432975198:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19512220ns) {Configure:104(19511614ns) {ProcessConfigured:135(19261088ns) {StartInitialHandshake:157(19260956ns) {DropHandshakes:469(144ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:476(840ns) {SwitchToState:178(156ns)}}}}}
2025-11-19T14:00:00.482164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435963432975265:2089] [node 5] ICP01 ready to work
2025-11-19T14:00:00.480323Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435964479930901:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18223564ns) {Configure:104(18223046ns) {ProcessConfigured:135(17819106ns) {StartInitialHandshake:157(17818675ns) {DropHandshakes:469(536ns) {DropIncomingHandshake:441(82ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(248ns) PrepareNewSessionHandshake:476(1153ns) {SwitchToState:178(27ns)}}}}}
2025-11-19T14:00:00.480517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435964479930964:2079] [node 4] ICP01 ready to work
2025-11-19T14:00:00.480656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435964479930965:2082] [node 6] ICP01 ready to work
2025-11-19T14:00:00.483421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435964479930975:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21087
2025-11-19T14:00:00.483438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435964479930976:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21087
2025-11-19T14:00:00.485152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435963432975229:2070] [node 6] ICP01 ready to work
2025-11-19T14:00:00.485260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435962902658113:2072] [node 4] ICP01 ready to work
2025-11-19T14:00:00.488103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435963269256288:2077] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.492130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435960788655558:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21111
2025-11-19T14:00:00.492161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435960788655559:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21111
2025-11-19T14:00:00.492173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435960788655560:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21111
2025-11-19T14:00:00.492297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435960788655538:2100] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(18187001ns) {Configure:104(18186357ns) {ProcessConfigured:135(17865888ns) {StartInitialHandshake:157(17865240ns) {DropHandshakes:469(824ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(162ns)} GenerateSessionVirtualId:988(390ns) PrepareNewSessionHandshake:476(1234ns) {SwitchToState:178(372ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.492416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435963829102049:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(32415770ns) {Configure:104(32414319ns) {ProcessConfigured:135(32006303ns) {StartInitialHandshake:157(32005961ns) {DropHandshakes:469(944ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(262ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:476(1012ns) {SwitchToState:178(287ns)}}}}}
2025-11-19T14:00:00.493351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435963829102120:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.493890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435960788655562:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21111
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.496545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435962902658153:2084] [node 6] ICP01 ready to work
2025-11-19T14:00:00.497210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435962902658147:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5232
2025-11-19T14:00:00.497223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435962902658148:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5232
2025-11-19T14:00:00.497235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435962902658149:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5232
2025-11-19T14:00:00.497245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435962902658150:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5232
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.505927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435964479930980:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21087
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.512444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435963269256323:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
2025-11-19T14:00:00.512446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435963269256324:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
2025-11-19T14:00:00.512300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435963432975275:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14425
2025-11-19T14:00:00.513540Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435962902658081:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(28211967ns) {Configure:104(28210334ns) {ProcessConfigured:135(27930564ns) {StartInitialHandshake:157(27929348ns) {DropHandshakes:469(1648ns) {DropIncomingHandshake:441(151ns) DropOutgoingHandshake:456(175ns)} GenerateSessionVirtualId:988(321ns) PrepareNewSessionHandshake:476(2156ns) {SwitchToState:178(1081ns)}}}}}
2025-11-19T14:00:00.513622Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435962902658147:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.513652Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435962902658149:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.513783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435962902658081:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.511069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435963829102102:2068] [node 5] ICP01 ready to work
2025-11-19T14:00:00.512129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435963829102103:2076] [node 4] ICP01 ready to work
2025-11-19T14:00:00.516610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435963432975229:2070] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(30632307ns) {Configure:104(30631786ns) {ProcessConfigured:135(30375066ns) {StartInitialHandshake:157(30374495ns) {DropHandshakes:469(367ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(141ns) PrepareNewSessionHandshake:476(1458ns) {SwitchToState:178(987ns)}}}}}
2025-11-19T14:00:00.519889Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435963269256323:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.519968Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435963269256307:2103] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.519975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435963269256307:2103] [node 3] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.520738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435963432975274:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14425
2025-11-19T14:00:00.524159Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435963269256307:2103] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:00.524321Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435963269256324:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.524578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435963269256245:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.524603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:00.524612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435963269256245:2049] [node 3] ICP22 created new session: [2:7574435963269256327:2049]
2025-11-19T14:00:00.524619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435963269256327:2049] [node 3] ICS09 handshake done sender: [2:7574435963269256307:2103] self: [2:7574435963269256306:0] peer: [3:7574435962902658163:0] socket: 117 qp: -1
2025-11-19T14:00:00.524629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435963269256327:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:00.524668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:00.525744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435962902658080:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(61697839ns) {Configure:104(61696676ns) {ProcessConfigured:135(61330250ns) {StartInitialHandshake:157(61329436ns) {DropHandshakes:469(745ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(245ns)} GenerateSessionVirtualId:988(269ns) PrepareNewSessionHandshake:476(842ns) {SwitchToState:178(660ns)}}}}}
2025-11-19T14:00:00.525799Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435962902658150:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.526413Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435963432975274:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.526693Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435963829102120:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.526732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435963829102102:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.526875Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435964479930969:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.526890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435964479930969:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.527592Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435964479930969:2109] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:00.527687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435964479930965:2082] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.527698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:00.527706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435964479930965:2082] [node 6] ICP22 created new session: [5:7574435964479930982:2082]
2025-11-19T14:00:00.527713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435964479930982:2082] [node 6] ICS09 handshake done sender: [5:7574435964479930969:2109] self: [5:7574435964479930968:0] peer: [6:7574435963829102128:0] socket: 121 qp: -1
2025-11-19T14:00:00.527721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435964479930982:2082] [node 6] ICS10 traffic start
2025-11-19T14:00:00.527730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.528171Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435960788655560:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.528202Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435960788655558:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.528471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435963829102129:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T14:00:00.528491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435963829102130:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T14:00:00.528530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435963829102131:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T14:00:00.529545Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435963829102120:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:00.529685Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435964479930976:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.529642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435963829102102:2068] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.529659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:00.529666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435963829102102:2068] [node 5] ICP22 created new session: [6:7574435963829102132:2068]
2025-11-19T14:00:00.529672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435963829102132:2068] [node 5] ICS09 handshake done sender: [6:7574435963829102120:2113] self: [6:7574435963829102128:0] peer: [5:7574435964479930968:0] socket: 129 qp: -1
2025-11-19T14:00:00.529677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435963829102132:2068] [node 5] ICS10 traffic start
2025-11-19T14:00:00.529695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:00.530051Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435963829102130:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.530236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435963829102050:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.530612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435962902658164:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5232
2025-11-19T14:00:00.530895Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435960788655559:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.532473Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435960788655532:2097] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.532481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435960788655532:2097] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.532997Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435960788655545:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.533000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435960788655545:2106] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.533997Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435960788655532:2097] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.534340Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435960788655545:2106] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:00.534704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435960788655480:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.534725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:00.534731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435960788655480:2048] [node 2] ICP22 created new session: [1:7574435960788655568:2048]
2025-11-19T14:00:00.534738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435960788655568:2048] [node 2] ICS09 handshake done sender: [1:7574435960788655545:2106] self: [1:7574435960788655544:0] peer: [2:7574435963269256332:0] socket: 113 qp: -1
2025-11-19T14:00:00.534744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435960788655568:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:00.534763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.535511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435963432975279:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14425
2025-11-19T14:00:00.530815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435964479930902:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(66841421ns) {Configure:104(54959505ns) {ProcessConfigured:135(54625994ns) {StartInitialHandshake:157(54625218ns) {DropHandshakes:469(1051ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(192ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(566ns) {SwitchToState:178(219ns)}}}}}
2025-11-19T14:00:00.531098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435964479930902:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.531188Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435964479930975:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.531201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435964479930901:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.534503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435964479930988:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21087
2025-11-19T14:00:00.534522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435964479930989:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21087
2025-11-19T14:00:00.535759Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435964479930975:2111] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:00.535782Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435964479930976:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.531313Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435963269256321:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.531325Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435963269256321:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.531664Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435963269256305:2102] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.531686Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435963269256302:2100] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.531689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435963269256302:2100] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.532345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435963269256244:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.534393Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435963269256321:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:00.534540Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435963269256302:2100] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:00.534646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435963269256281:2082] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.534662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:00.534669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435963269256281:2082] [node 6] ICP22 created new session: [2:7574435963269256334:2082]
2025-11-19T14:00:00.534675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435963269256334:2082] [node 6] ICS09 handshake done sender: [2:7574435963269256321:2114] self: [2:7574435963269256289:0] peer: [6:7574435963829102134:0] socket: 146 qp: -1
2025-11-19T14:00:00.534684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435963269256334:2082] [node 6] ICS10 traffic start
2025-11-19T14:00:00.534694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:00.534904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435963269256280:2080] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.534911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:00.534916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435963269256280:2080] [node 5] ICP22 created new session: [2:7574435963269256336:2080]
2025-11-19T14:00:00.534920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435963269256336:2080] [node 5] ICS09 handshake done sender: [2:7574435963269256302:2100] self: [2:7574435963269256287:0] peer: [5:7574435964479930986:0] socket: 115 qp: -1
2025-11-19T14:00:00.534924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435963269256336:2080] [node 5] ICS10 traffic start
2025-11-19T14:00:00.534929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:00.536020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435963269256333:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.537420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435963829102136:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T14:00:00.537526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435964479930901:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.537546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:00.537552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435964479930901:2048] [node 1] ICP22 created new session: [5:7574435964479930990:2048]
2025-11-19T14:00:00.537557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435964479930990:2048] [node 1] ICS09 handshake done sender: [5:7574435964479930975:2111] self: [5:7574435964479930987:0] peer: [1:7574435960788655531:0] socket: 109 qp: -1
2025-11-19T14:00:00.537564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435964479930990:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:00.537571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:00.538485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435964479930902:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.538497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:00.538502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435964479930902:2049] [node 2] ICP22 created new session: [5:7574435964479930992:2049]
2025-11-19T14:00:00.538508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435964479930992:2049] [node 2] ICS09 handshake done sender: [5:7574435964479930976:2112] self: [5:7574435964479930986:0] peer: [2:7574435963269256287:0] socket: 124 qp: -1
2025-11-19T14:00:00.538513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435964479930992:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:00.538518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:00.540773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435963269256244:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30546) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435963269256305:2102] held: no
2025-11-19T14:00:00.540929Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435963269256305:2102] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:00.541182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435963269256244:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.541195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:00.541202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435963269256244:2048] [node 1] ICP22 created new session: [2:7574435963269256338:2048]
2025-11-19T14:00:00.541207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435963269256338:2048] [node 1] ICS09 handshake done sender: [2:7574435963269256305:2102] self: [2:7574435963269256332:0] peer: [1:7574435960788655544:0] socket: 114 qp: -1
2025-11-19T14:00:00.541213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435963269256338:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:00.541219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:00.541614Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435963829102130:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:00.541694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435963829102050:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.541710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:00.541716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435963829102050:2049] [node 2] ICP22 created new session: [6:7574435963829102137:2049]
2025-11-19T14:00:00.541721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435963829102137:2049] [node 2] ICS09 handshake done sender: [6:7574435963829102130:2117] self: [6:7574435963829102134:0] peer: [2:7574435963269256289:0] socket: 149 qp: -1
2025-11-19T14:00:00.541727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435963829102137:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:00.541734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:00.535445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435960788655530:2096] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.535467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:00.535472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435960788655530:2096] [node 5] ICP22 created new session: [1:7574435960788655571:2096]
2025-11-19T14:00:00.535478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435960788655571:2096] [node 5] ICS09 handshake done sender: [1:7574435960788655532:2097] self: [1:7574435960788655531:0] peer: [5:7574435964479930987:0] socket: 108 qp: -1
2025-11-19T14:00:00.535482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435960788655571:2096] [node 5] ICS10 traffic start
2025-11-19T14:00:00.535490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:00.548880Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435960788655562:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.552066Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435962902658148:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.555150Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435962902658149:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:00.555333Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435963432975279:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.555391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435963432975197:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.555569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435963269256288:2077] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(67305754ns) {Configure:104(67304869ns) {ProcessConfigured:135(66956847ns) {StartInitialHandshake:157(66955947ns) {DropHandshakes:469(642ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(95ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:476(1337ns) {SwitchToState:178(800ns)}}}}}
2025-11-19T14:00:00.555651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435962902658167:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5232
2025-11-19T14:00:00.555736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435962902658080:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.555815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435963432975278:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14425
2025-11-19T14:00:00.556220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435963432975281:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14425
2025-11-19T14:00:00.556201Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435962902658167:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.556319Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435964479930980:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.556359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435964479930903:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.555766Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435960788655552:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.555781Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435960788655552:2109] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.556559Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435963432975275:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.556585Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435963432975278:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.556606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435963432975199:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.557146Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435963829102129:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.557763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435963829102051:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.557209Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435962902658152:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.557219Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435962902658152:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.558048Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435962902658155:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.558051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435962902658155:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.560162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435963829102139:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T14:00:00.558436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435962902658081:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.558460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:00.558466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435962902658081:2049] [node 2] ICP22 created new session: [3:7574435962902658170:2049]
2025-11-19T14:00:00.558473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435962902658170:2049] [node 2] ICS09 handshake done sender: [3:7574435962902658149:2112] self: [3:7574435962902658163:0] peer: [2:7574435963269256306:0] socket: 135 qp: -1
2025-11-19T14:00:00.558480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435962902658170:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:00.558499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:00.558912Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435962902658157:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.558915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435962902658157:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.565246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435963432975284:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14425
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.568382Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435962902658155:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:00.568494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435964479930996:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21087
2025-11-19T14:00:00.568941Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435962902658157:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:00.568078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435963432975285:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14425
2025-11-19T14:00:00.568293Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435963432975279:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:00.568801Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435963432975275:2115] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:00.569064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435963432975197:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.569097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:00.569103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435963432975197:2048] [node 1] ICP22 created new session: [4:7574435963432975286:2048]
2025-11-19T14:00:00.569110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435963432975286:2048] [node 1] ICS09 handshake done sender: [4:7574435963432975279:2118] self: [4:7574435963432975280:0] peer: [1:7574435960788655551:0] socket: 162 qp: -1
2025-11-19T14:00:00.569118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435963432975286:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:00.569142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.569676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435963432975199:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.569687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:00.569691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435963432975199:2050] [node 3] ICP22 created new session: [4:7574435963432975288:2050]
2025-11-19T14:00:00.569695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435963432975288:2050] [node 3] ICS09 handshake done sender: [4:7574435963432975275:2115] self: [4:7574435963432975282:0] peer: [3:7574435962902658154:0] socket: 142 qp: -1
2025-11-19T14:00:00.569699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435963432975288:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:00.569712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:00.557272Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435960788655543:2105] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.557284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435960788655543:2105] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.557681Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435960788655552:2109] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.574126Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435962902658152:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:00.574269Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435964479930980:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:00.570423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435963829102141:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T14:00:00.570623Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435963829102129:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:00.570707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435963829102051:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.570718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:00.570722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435963829102051:2050] [node 3] ICP22 created new session: [6:7574435963829102142:2050]
2025-11-19T14:00:00.570727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435963829102142:2050] [node 3] ICS09 handshake done sender: [6:7574435963829102129:2116] self: [6:7574435963829102140:0] peer: [3:7574435962902658156:0] socket: 141 qp: -1
2025-11-19T14:00:00.570741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435963829102142:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:00.570746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:00.574354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435964479930997:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21087
2025-11-19T14:00:00.574363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435964479930903:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.574388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:00.574394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435964479930903:2050] [node 3] ICP22 created new session: [5:7574435964479930998:2050]
2025-11-19T14:00:00.574400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435964479930998:2050] [node 3] ICS09 handshake done sender: [5:7574435964479930980:2116] self: [5:7574435964479930995:0] peer: [3:7574435962902658151:0] socket: 140 qp: -1
2025-11-19T14:00:00.574404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435964479930998:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:00.574411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.575691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435962902658113:2072] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.575710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:00.575715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435962902658113:2072] [node 4] ICP22 created new session: [3:7574435962902658176:2072]
2025-11-19T14:00:00.575721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435962902658176:2072] [node 4] ICS09 handshake done sender: [3:7574435962902658155:2115] self: [3:7574435962902658154:0] peer: [4:7574435963432975282:0] socket: 138 qp: -1
2025-11-19T14:00:00.575725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435962902658176:2072] [node 4] ICS10 traffic start
2025-11-19T14:00:00.575732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:00.575987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435962902658153:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.576001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:00.576007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435962902658153:2084] [node 6] ICP22 created new session: [3:7574435962902658178:2084]
2025-11-19T14:00:00.576014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435962902658178:2084] [node 6] ICS09 handshake done sender: [3:7574435962902658157:2116] self: [3:7574435962902658156:0] peer: [6:7574435963829102140:0] socket: 139 qp: -1
2025-11-19T14:00:00.576019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435962902658178:2084] [node 6] ICS10 traffic start
2025-11-19T14:00:00.576027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:00.576218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435962902658116:2079] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.576231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:00.576236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435962902658116:2079] [node 5] ICP22 created new session: [3:7574435962902658181:2079]
2025-11-19T14:00:00.576240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435962902658181:2079] [node 5] ICS09 handshake done sender: [3:7574435962902658152:2114] self: [3:7574435962902658151:0] peer: [5:7574435964479930995:0] socket: 137 qp: -1
2025-11-19T14:00:00.576243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435962902658181:2079] [node 5] ICS10 traffic start
2025-11-19T14:00:00.576260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:00.572216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435960788655550:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.572235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:00.572242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435960788655550:2108] [node 4] ICP22 created new session: [1:7574435960788655578:2108]
2025-11-19T14:00:00.572249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435960788655578:2108] [node 4] ICS09 handshake done sender: [1:7574435960788655552:2109] self: [1:7574435960788655551:0] peer: [4:7574435963432975280:0] socket: 161 qp: -1
2025-11-19T14:00:00.572255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435960788655578:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:00.572277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:00.577244Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435960788655543:2105] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.577578Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435963432975281:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.577360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435960788655577:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21111
2025-11-19T14:00:00.577418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435960788655481:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.577428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:00.577433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435960788655481:2049] [node 3] ICP22 created new session: [1:7574435960788655580:2049]
2025-11-19T14:00:00.577440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435960788655580:2049] [node 3] ICS09 handshake done sender: [1:7574435960788655543:2105] self: [1:7574435960788655542:0] peer: [3:7574435962902658168:0] socket: 112 qp: -1
2025-11-19T14:00:00.577444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435960788655580:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:00.577451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.579116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435962902658180:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5232
2025-11-19T14:00:00.575568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435963829102144:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T14:00:00.579895Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435963829102139:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.579939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435963829102049:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.580324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435963432975198:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.580341Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435960788655561:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.580364Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435960788655561:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.580738Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435963269256340:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.580751Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435963269256340:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.588115Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435963269256340:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:00.588371Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435960788655577:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.588556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435960788655481:2049] [node 3] ICP18 handshake (actor [1:7574435960788655577:2119]) is too old
2025-11-19T14:00:00.588380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435963269256288:2077] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.588409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:00.588420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435963269256288:2077] [node 4] ICP22 created new session: [2:7574435963269256344:2077]
2025-11-19T14:00:00.588429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435963269256344:2077] [node 4] ICS09 handshake done sender: [2:7574435963269256340:2118] self: [2:7574435963269256322:0] peer: [4:7574435963432975292:0] socket: 116 qp: -1
2025-11-19T14:00:00.588437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435963269256344:2077] [node 4] ICS10 traffic start
2025-11-19T14:00:00.588449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:00.589215Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435960788655561:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.589775Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435963829102144:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.589836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435963829102146:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T14:00:00.589988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435963829102103:2076] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:00.590048Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435963829102139:2120] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:00.590105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435963829102049:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.590120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:00.590125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435963829102049:2048] [node 1] ICP22 created new session: [6:7574435963829102148:2048]
2025-11-19T14:00:00.590131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435963829102148:2048] [node 1] ICS09 handshake done sender: [6:7574435963829102139:2120] self: [6:7574435963829102145:0] peer: [1:7574435960788655555:0] socket: 127 qp: -1
2025-11-19T14:00:00.590138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435963829102148:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:00.590146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:00.591075Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435964479930997:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.591116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435964479930964:2079] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.591421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435963432975293:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14425
2025-11-19T14:00:00.596556Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435963432975198:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7657) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435963432975281:2119] held: no
2025-11-19T14:00:00.596709Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435963432975276:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.596714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435963432975276:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.597198Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435963432975269:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:00.597202Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435963432975269:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:00.597774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435960788655538:2100] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.597805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:00.597814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435960788655538:2100] [node 6] ICP22 created new session: [1:7574435960788655583:2100]
2025-11-19T14:00:00.597822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435960788655583:2100] [node 6] ICS09 handshake done sender: [1:7574435960788655561:2117] self: [1:7574435960788655555:0] peer: [6:7574435963829102145:0] socket: 119 qp: -1
2025-11-19T14:00:00.597830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435960788655583:2100] [node 6] ICS10 traffic start
2025-11-19T14:00:00.597843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:00.598766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435962902658080:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30546) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435962902658148:2111] held: no
2025-11-19T14:00:00.622169Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435962902658148:2111] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:00.628339Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435963432975281:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.628783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435964479931001:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21087
2025-11-19T14:00:00.628103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435962902658080:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.628135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:00.628148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435962902658080:2048] [node 1] ICP22 created new session: [3:7574435962902658183:2048]
2025-11-19T14:00:00.628158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435962902658183:2048] [node 1] ICS09 handshake done sender: [3:7574435962902658148:2111] self: [3:7574435962902658168:0] peer: [1:7574435960788655542:0] socket: 134 qp: -1
2025-11-19T14:00:00.628170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435962902658183:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:00.628180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:00.631980Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435963432975276:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:00.632066Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435963432975269:2112] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:00.632126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435963432975198:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.632144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:00.632153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435963432975198:2049] [node 2] ICP22 created new session: [4:7574435963432975296:2049]
2025-11-19T14:00:00.632160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435963432975296:2049] [node 2] ICS09 handshake done sender: [4:7574435963432975281:2119] self: [4:7574435963432975292:0] peer: [2:7574435963269256322:0] socket: 132 qp: -1
2025-11-19T14:00:00.632168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435963432975296:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:00.632186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:00.632332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435963432975265:2089] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.632352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:00.632357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435963432975265:2089] [node 5] ICP22 created new session: [4:7574435963432975298:2089]
2025-11-19T14:00:00.632364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435963432975298:2089] [node 5] ICS09 handshake done sender: [4:7574435963432975269:2112] self: [4:7574435963432975268:0] peer: [5:7574435964479931000:0] socket: 174 qp: -1
2025-11-19T14:00:00.632370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435963432975298:2089] [node 5] ICS10 traffic start
2025-11-19T14:00:00.632378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:00.632106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435963829102150:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45829
2025-11-19T14:00:00.632942Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435964479930997:2121] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:00.632813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435963432975229:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:00.632832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:00.632837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435963432975229:2070] [node 6] ICP22 created new session: [4:7574435963432975300:2070]
2025-11-19T14:00:00.632842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435963432975300:2070] [node 6] ICS09 handshake done sender: [4:7574435963432975276:2116] self: [4:7574435963432975273:0] peer: [6:7574435963829102147:0] socket: 175 qp: -1
2025-11-19T14:00:00.632847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435963432975300:2070] [node 6] ICS10 traffic start
2025-11-19T14:00:00.632854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:00.633274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435964479930964:2079] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.633290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:00.633296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435964479930964:2079] [node 4] ICP22 created new session: [5:7574435964479931002:2079]
2025-11-19T14:00:00.633301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435964479931002:2079] [node 4] ICS09 handshake done sender: [5:7574435964479930997:2121] self: [5:7574435964479931000:0] peer: [4:7574435963432975268:0] socket: 176 qp: -1
2025-11-19T14:00:00.633307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435964479931002:2079] [node 4] ICS10 traffic start
2025-11-19T14:00:00.633315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:00.634445Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435963829102144:2122] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:00.635255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435963829102103:2076] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:00.635272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:00.635279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435963829102103:2076] [node 4] ICP22 created new session: [6:7574435963829102151:2076]
2025-11-19T14:00:00.635285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435963829102151:2076] [node 4] ICS09 handshake done sender: [6:7574435963829102144:2122] self: [6:7574435963829102147:0] peer: [4:7574435963432975273:0] socket: 177 qp: -1
2025-11-19T14:00:00.635291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435963829102151:2076] [node 4] ICS10 traffic start
2025-11-19T14:00:00.635299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:00.668332Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:00.668439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435963269256338:2048] [node 1] ICS01 socket: 114 reason# EndOfStream
2025-11-19T14:00:00.668452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435963269256244:2048] [node 1] ICP30 unregister session Session# [2:7574435963269256338:2048] VirtualId# [2:7574435963269256332:0]
2025-11-19T14:00:00.668461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435963269256338:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.668497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:00.668520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:00.668908Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:00.668968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435962902658183:2048] [node 1] ICS01 socket: 134 reason# EndOfStream
2025-11-19T14:00:00.668974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435962902658080:2048] [node 1] ICP30 unregister session Session# [3:7574435962902658183:2048] VirtualId# [3:7574435962902658168:0]
2025-11-19T14:00:00.668978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435962902658183:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.669003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:00.669019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:00.679533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:00.679644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435963829102148:2048] [node 1] ICS01 socket: 127 reason# EndOfStream
2025-11-19T14:00:00.679656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435963829102049:2048] [node 1] ICP30 unregister session Session# [6:7574435963829102148:2048] VirtualId# [6:7574435963829102145:0]
2025-11-19T14:00:00.679666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435963829102148:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.679722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:00.679772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:00.679814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:00.679874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435963432975286:2048] [node 1] ICS01 socket: 162 reason# EndOfStream
2025-11-19T14:00:00.679880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435963432975197:2048] [node 1] ICP30 unregister session Session# [4:7574435963432975286:2048] VirtualId# [4:7574435963432975280:0]
2025-11-19T14:00:00.679884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435963432975286:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.679913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:00.679935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:00.680064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:00.680123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435964479930990:2048] [node 1] ICS01 socket: 109 reason# EndOfStream
2025-11-19T14:00:00.680127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435964479930901:2048] [node 1] ICP30 unregister session Session# [5:7574435964479930990:2048] VirtualId# [5:7574435964479930987:0]
2025-11-19T14:00:00.680131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435964479930990:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.680158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:00.680179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:00.701415Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:00.701516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435962902658170:2049] [node 2] ICS01 socket: 135 reason# EndOfStream
2025-11-19T14:00:00.701527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435962902658081:2049] [node 2] ICP30 unregister session Session# [3:7574435962902658170:2049] VirtualId# [3:7574435962902658163:0]
2025-11-19T14:00:00.701535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435962902658170:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.701567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:00.701586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:00.706167Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:00.706281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435963432975296:2049] [node 2] ICS01 socket: 132 reason# EndOfStream
2025-11-19T14:00:00.706291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435963432975198:2049] [node 2] ICP30 unregister session Session# [4:7574435963432975296:2049] VirtualId# [4:7574435963432975292:0]
2025-11-19T14:00:00.706300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435963432975296:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.706347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:00.706373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:00.706422Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:00.706486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435964479930992:2049] [node 2] ICS01 socket: 124 reason# EndOfStream
2025-11-19T14:00:00.706490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435964479930902:2049] [node 2] ICP30 unregister session Session# [5:7574435964479930992:2049] VirtualId# [5:7574435964479930986:0]
2025-11-19T14:00:00.706494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435964479930992:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.706519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:00.706537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:00.706566Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:00.706622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435963829102137:2049] [node 2] ICS01 socket: 149 reason# EndOfStream
2025-11-19T14:00:00.706626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435963829102050:2049] [node 2] ICP30 unregister session Session# [6:7574435963829102137:2049] VirtualId# [6:7574435963829102134:0]
2025-11-19T14:00:00.706630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435963829102137:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.706655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:00.706674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:00.721360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T14:00:00.721496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435963432975288:2050] [node 3] ICS01 socket: 142 reason# EndOfStream
2025-11-19T14:00:00.721508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435963432975199:2050] [node 3] ICP30 unregister session Session# [4:7574435963432975288:2050] VirtualId# [4:7574435963432975282:0]
2025-11-19T14:00:00.721528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435963432975288:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.721568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:00.721601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:00.721715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:00.721776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435964479930998:2050] [node 3] ICS01 socket: 140 reason# EndOfStream
2025-11-19T14:00:00.721782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435964479930903:2050] [node 3] ICP30 unregister session Session# [5:7574435964479930998:2050] VirtualId# [5:7574435964479930995:0]
2025-11-19T14:00:00.721786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435964479930998:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.721815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:00.721833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:00.726404Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:00.726514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435963829102142:2050] [node 3] ICS01 socket: 141 reason# EndOfStream
2025-11-19T14:00:00.726525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435963829102051:2050] [node 3] ICP30 unregister session Session# [6:7574435963829102142:2050] VirtualId# [6:7574435963829102140:0]
2025-11-19T14:00:00.726533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435963829102142:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.726596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:00.726623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:00.755846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:00.755987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435963829102151:2076] [node 4] ICS01 socket: 177 reason# EndOfStream
2025-11-19T14:00:00.756001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435963829102103:2076] [node 4] ICP30 unregister session Session# [6:7574435963829102151:2076] VirtualId# [6:7574435963829102147:0]
2025-11-19T14:00:00.756009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435963829102151:2076] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.756072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:00.756101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:00.767093Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T14:00:00.767200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435963829102132:2068] [node 5] ICS01 socket: 129 reason# EndOfStream
2025-11-19T14:00:00.767213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435963829102102:2068] [node 5] ICP30 unregister session Session# [6:7574435963829102132:2068] VirtualId# [6:7574435963829102128:0]
2025-11-19T14:00:00.767221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435963829102132:2068] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:00.767282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T14:00:00.767308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:01.001303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435961936880308:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:01.001325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435961936880309:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:00.996762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435961545824299:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:00.996779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435961545824300:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:01.056316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435967468770182:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:01.056338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435967468770183:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:01.087978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435964811522323:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:01.088000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435964811522324:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:01.088005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435964811522325:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:01.137397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435968479439177:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:01.137422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435968479439178:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:01.137426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435968479439179:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:01.212397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435966231847663:2102] [node 5] ICP01 ready to work
2025-11-19T14:00:01.212542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435966231847664:2105] [node 4] ICP01 ready to work
2025-11-19T14:00:01.212455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435964811522377:2100] [node 5] ICP01 ready to work
2025-11-19T14:00:01.212754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435966231847669:2107] [node 6] ICP01 ready to work
2025-11-19T14:00:01.212941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435967468770239:2102] [node 4] ICP01 ready to work
2025-11-19T14:00:01.213616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435967468770243:2104] [node 5] ICP01 ready to work
2025-11-19T14:00:01.223617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435964811522388:2104] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.228551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435968479439236:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54133
2025-11-19T14:00:01.227273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435964811522395:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4210
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.232011Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435966231847664:2105] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19191509ns) {Configure:104(19190806ns) {ProcessConfigured:135(18390516ns) {StartInitialHandshake:157(18390069ns) {DropHandshakes:469(124ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(24ns)} GenerateSessionVirtualId:988(45ns) PrepareNewSessionHandshake:476(1866ns) {SwitchToState:178(824ns)}}}}}
2025-11-19T14:00:01.232667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435966231847677:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1611
2025-11-19T14:00:01.232678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435966231847678:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1611
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.233999Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435967468770243:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(20341046ns) {Configure:104(20340608ns) {ProcessConfigured:135(20063051ns) {StartInitialHandshake:157(20062668ns) {DropHandshakes:469(416ns) {DropIncomingHandshake:441(82ns) DropOutgoingHandshake:456(37ns)} GenerateSessionVirtualId:988(92ns) PrepareNewSessionHandshake:476(783ns) {SwitchToState:178(214ns)}}}}}
2025-11-19T14:00:01.234140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435967468770252:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15403
2025-11-19T14:00:01.234152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435967468770253:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15403
2025-11-19T14:00:01.233712Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435961936880309:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21137551ns) {Configure:104(21136984ns) {ProcessConfigured:135(19867928ns) {StartInitialHandshake:157(19867412ns) {DropHandshakes:469(269ns) {DropIncomingHandshake:441(21ns) DropOutgoingHandshake:456(26ns)} GenerateSessionVirtualId:988(65ns) PrepareNewSessionHandshake:476(1011ns) {SwitchToState:178(407ns)}}}}}
2025-11-19T14:00:01.234080Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435968479439236:2101] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.239896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435965840791638:2089] [node 5] ICP01 ready to work
2025-11-19T14:00:01.239999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435965840791640:2090] [node 6] ICP01 ready to work
2025-11-19T14:00:01.240530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435965840791642:2091] [node 4] ICP01 ready to work
2025-11-19T14:00:01.242308Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435961545824300:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(25610181ns) {Configure:104(25608892ns) {ProcessConfigured:135(25161848ns) {StartInitialHandshake:157(25161063ns) {DropHandshakes:469(674ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:476(918ns) {SwitchToState:178(397ns)}}}}}
2025-11-19T14:00:01.243317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435966231847669:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26346297ns) {Configure:104(26345646ns) {ProcessConfigured:135(26048939ns) {StartInitialHandshake:157(26048654ns) {DropHandshakes:469(602ns) {DropIncomingHandshake:441(103ns) DropOutgoingHandshake:456(87ns)} GenerateSessionVirtualId:988(242ns) PrepareNewSessionHandshake:476(710ns) {SwitchToState:178(343ns)}}}}}
2025-11-19T14:00:01.244264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435966231847683:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1611
2025-11-19T14:00:01.245307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435968479439230:2092] [node 4] ICP01 ready to work
2025-11-19T14:00:01.245898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435968479439231:2094] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.249098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435968479439239:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54133
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.251660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435964811522400:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4210
2025-11-19T14:00:01.251681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435964811522401:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4210
2025-11-19T14:00:01.196029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435968173462475:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:01.204549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435968173462473:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:01.204608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435968173462474:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:01.214110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435968173462515:2086] [node 4] ICP01 ready to work
2025-11-19T14:00:01.233027Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435968173462474:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19696317ns) {Configure:104(19695651ns) {ProcessConfigured:135(19303179ns) {StartInitialHandshake:157(19302914ns) {DropHandshakes:469(265ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(26ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:476(1471ns) {SwitchToState:178(441ns)}}}}}
2025-11-19T14:00:01.234284Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435968173462473:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(20537307ns) {Configure:104(20537057ns) {ProcessConfigured:135(20271836ns) {StartInitialHandshake:157(20271538ns) {DropHandshakes:469(135ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(26ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(657ns) {SwitchToState:178(323ns)}}}}}
2025-11-19T14:00:01.246451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435968173462522:2090] [node 5] ICP01 ready to work
2025-11-19T14:00:01.251430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968173462549:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43504
2025-11-19T14:00:01.251511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968173462550:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43504
2025-11-19T14:00:01.251910Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435968173462549:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.257036Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435968173462550:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.256227Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435964811522324:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(18646715ns) {Configure:104(18645842ns) {ProcessConfigured:135(18144109ns) {StartInitialHandshake:157(18143749ns) {DropHandshakes:469(581ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:476(1059ns) {SwitchToState:178(330ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.259866Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435966231847663:2102] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(47345238ns) {Configure:104(47343949ns) {ProcessConfigured:135(46494010ns) {StartInitialHandshake:157(46493287ns) {DropHandshakes:469(593ns) {DropIncomingHandshake:441(94ns) DropOutgoingHandshake:456(109ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:476(1316ns) {SwitchToState:178(715ns)}}}}}
2025-11-19T14:00:01.260321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435965840791672:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15224
2025-11-19T14:00:01.260408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435965840791673:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15224
2025-11-19T14:00:01.260443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435965840791674:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15224
2025-11-19T14:00:01.260454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435965840791675:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15224
2025-11-19T14:00:01.260602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435964811522325:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(20722103ns) {Configure:104(20721440ns) {ProcessConfigured:135(20379755ns) {StartInitialHandshake:157(20379141ns) {DropHandshakes:469(776ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(97ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:476(717ns) {SwitchToState:178(259ns)}}}}}
2025-11-19T14:00:01.260754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435965840791638:2089] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(20602924ns) {Configure:104(20602536ns) {ProcessConfigured:135(20279709ns) {StartInitialHandshake:157(20279440ns) {DropHandshakes:469(425ns) {DropIncomingHandshake:441(95ns) DropOutgoingHandshake:456(87ns)} GenerateSessionVirtualId:988(580ns) PrepareNewSessionHandshake:476(680ns) {SwitchToState:178(172ns)}}}}}
2025-11-19T14:00:01.260804Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435965840791672:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.260823Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435965840791673:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.260931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435965840791640:2090] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20296203ns) {Configure:104(20295405ns) {ProcessConfigured:135(20071679ns) {StartInitialHandshake:157(20071468ns) {DropHandshakes:469(189ns) {DropIncomingHandshake:441(31ns) DropOutgoingHandshake:456(32ns)} GenerateSessionVirtualId:988(161ns) PrepareNewSessionHandshake:476(396ns) {SwitchToState:178(37ns)}}}}}
2025-11-19T14:00:01.263470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435967468770259:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.272696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435967468770261:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15403
2025-11-19T14:00:01.272904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435966231847686:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1611
2025-11-19T14:00:01.273316Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435966231847677:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.275855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435968173462515:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(61681649ns) {Configure:104(61681151ns) {ProcessConfigured:135(61459475ns) {StartInitialHandshake:157(61458823ns) {DropHandshakes:469(142ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(31ns)} GenerateSessionVirtualId:988(124ns) PrepareNewSessionHandshake:476(2838ns) {SwitchToState:178(1818ns)}}}}}
2025-11-19T14:00:01.275971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435968173462515:2086] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.276996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435968479439230:2092] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.277053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435968479439230:2092] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(30956211ns) {Configure:104(30955687ns) {ProcessConfigured:135(30611511ns) {StartInitialHandshake:157(30570626ns) {DropHandshakes:469(812ns) {DropIncomingHandshake:441(95ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(486ns) PrepareNewSessionHandshake:476(1147ns) {SwitchToState:178(321ns)}} IncomingHandshake:261(32780ns) {DropIncomingHandshake:441(23ns) PrepareNewSessionHandshake:476(144ns) {SwitchToState:178(98ns)} GenerateSessionVirtualId:988(219ns) IssueIncomingHandshakeReply:193(25999ns) {SwitchToState:178(42ns)}}}}}
2025-11-19T14:00:01.277189Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435968173462522:2090] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(30561040ns) {Configure:104(30559382ns) {ProcessConfigured:135(30229392ns) {StartInitialHandshake:157(30229031ns) {DropHandshakes:469(550ns) {DropIncomingHandshake:441(88ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(276ns) PrepareNewSessionHandshake:476(1173ns) {SwitchToState:178(22ns)}}}}}
2025-11-19T14:00:01.277217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435968173462522:2090] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.277196Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435964811522387:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.277210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435964811522387:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.277570Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435964811522390:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.277580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435964811522390:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.277645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435967468770182:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(43282173ns) {Configure:104(43281347ns) {ProcessConfigured:135(42855589ns) {StartInitialHandshake:157(42854208ns) {DropHandshakes:469(2030ns) {DropIncomingHandshake:441(420ns) DropOutgoingHandshake:456(404ns)} GenerateSessionVirtualId:988(311ns) PrepareNewSessionHandshake:476(1751ns) {SwitchToState:178(1228ns)}}}}}
2025-11-19T14:00:01.277829Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435966231847678:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.280221Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435968479439179:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(61184080ns) {Configure:104(61182932ns) {ProcessConfigured:135(60871453ns) {StartInitialHandshake:157(60871038ns) {DropHandshakes:469(600ns) {DropIncomingHandshake:441(89ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(1107ns) {SwitchToState:178(697ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.287047Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435968479439253:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.287076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435968479439253:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.287237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435968479439258:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54133
2025-11-19T14:00:01.287426Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435964811522387:2108] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:01.287723Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435968479439236:2101] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.288575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435966231847689:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1611
2025-11-19T14:00:01.287904Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435964811522395:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.287922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435964811522377:2100] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.287940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:01.287948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435964811522377:2100] [node 5] ICP22 created new session: [4:7574435964811522405:2100]
2025-11-19T14:00:01.287955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435964811522405:2100] [node 5] ICS09 handshake done sender: [4:7574435964811522387:2108] self: [4:7574435964811522379:0] peer: [5:7574435968479439256:0] socket: 114 qp: -1
2025-11-19T14:00:01.287963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435964811522405:2100] [node 5] ICS10 traffic start
2025-11-19T14:00:01.287986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:01.288299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435964811522325:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.288823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435968479439230:2092] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.288842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:01.288847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435968479439230:2092] [node 4] ICP22 created new session: [5:7574435968479439259:2092]
2025-11-19T14:00:01.288853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435968479439259:2092] [node 4] ICS09 handshake done sender: [5:7574435968479439236:2101] self: [5:7574435968479439256:0] peer: [4:7574435964811522379:0] socket: 120 qp: -1
2025-11-19T14:00:01.288858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435968479439259:2092] [node 4] ICS10 traffic start
2025-11-19T14:00:01.288864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:01.292467Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435965840791674:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.292593Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435966231847686:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.292622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435961936880308:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.292856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435965840791662:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.292866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435965840791662:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.292349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435968479439239:2102] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.293262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435968479439179:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.293366Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435964811522400:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.293889Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435964811522401:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.293936Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435964811522390:2109] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:01.294032Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435966231847689:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.294857Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435968479439253:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:01.294966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435968479439231:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.294983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:01.294991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435968479439231:2094] [node 6] ICP22 created new session: [5:7574435968479439264:2094]
2025-11-19T14:00:01.294998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435968479439264:2094] [node 6] ICS09 handshake done sender: [5:7574435968479439253:2114] self: [5:7574435968479439252:0] peer: [6:7574435968173462554:0] socket: 143 qp: -1
2025-11-19T14:00:01.295003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435968479439264:2094] [node 6] ICS10 traffic start
2025-11-19T14:00:01.295021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:01.295397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435964811522388:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.295411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:01.295416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435964811522388:2104] [node 6] ICP22 created new session: [4:7574435964811522411:2104]
2025-11-19T14:00:01.292349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435967468770252:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.292738Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435967468770253:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.292757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435967468770183:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.293731Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435967468770251:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.293735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435967468770251:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.295420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435964811522411:2104] [node 6] ICS09 handshake done sender: [4:7574435964811522390:2109] self: [4:7574435964811522389:0] peer: [6:7574435968173462552:0] socket: 115 qp: -1
2025-11-19T14:00:01.295425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435964811522411:2104] [node 6] ICS10 traffic start
2025-11-19T14:00:01.295431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:01.295637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435964811522323:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.296661Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435966231847681:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.296674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435966231847681:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.298062Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435965840791664:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.298079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435965840791664:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.297676Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435967468770246:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.297695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435967468770246:2106] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.298831Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435965840791675:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.298049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435966231847674:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.298059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435966231847674:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.300193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435964811522415:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4210
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.300856Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435965840791662:2107] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:01.300985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435961545824299:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.301016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:01.301023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435961545824299:2048] [node 2] ICP22 created new session: [1:7574435965840791683:2048]
2025-11-19T14:00:01.301029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435965840791683:2048] [node 2] ICS09 handshake done sender: [1:7574435965840791662:2107] self: [1:7574435965840791661:0] peer: [2:7574435966231847690:0] socket: 140 qp: -1
2025-11-19T14:00:01.301034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435965840791683:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:01.301041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:01.300851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435964811522324:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.300351Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435966231847674:2110] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:01.300644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435966231847693:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1611
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.301739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435964811522416:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4210
2025-11-19T14:00:01.302305Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435964811522400:2114] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:01.302017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435966231847664:2105] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.302035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:01.302040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435966231847664:2105] [node 4] ICP22 created new session: [2:7574435966231847694:2105]
2025-11-19T14:00:01.302046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435966231847694:2105] [node 4] ICS09 handshake done sender: [2:7574435966231847674:2110] self: [2:7574435966231847671:0] peer: [4:7574435964811522409:0] socket: 125 qp: -1
2025-11-19T14:00:01.302051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435966231847694:2105] [node 4] ICS10 traffic start
2025-11-19T14:00:01.302071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:01.302537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435964811522324:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.302555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:01.302560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435964811522324:2049] [node 2] ICP22 created new session: [4:7574435964811522417:2049]
2025-11-19T14:00:01.302565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435964811522417:2049] [node 2] ICS09 handshake done sender: [4:7574435964811522400:2114] self: [4:7574435964811522409:0] peer: [2:7574435966231847671:0] socket: 137 qp: -1
2025-11-19T14:00:01.302570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435964811522417:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:01.302575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.300325Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435967468770251:2108] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:01.300667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435967468770267:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15403
2025-11-19T14:00:01.301122Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435967468770267:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.301147Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435967468770261:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.301156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435967468770243:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.301164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:01.301168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435967468770243:2104] [node 5] ICP22 created new session: [3:7574435967468770269:2104]
2025-11-19T14:00:01.301173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435967468770269:2104] [node 5] ICS09 handshake done sender: [3:7574435967468770251:2108] self: [3:7574435967468770244:0] peer: [5:7574435968479439262:0] socket: 132 qp: -1
2025-11-19T14:00:01.301177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435967468770269:2104] [node 5] ICS10 traffic start
2025-11-19T14:00:01.301182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:01.301368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435967468770182:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.303393Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435966231847686:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:01.303578Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435964811522395:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:01.303497Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435967468770246:2106] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.303150Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435965840791660:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.303159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435965840791660:2106] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.304980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435965840791685:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15224
2025-11-19T14:00:01.305036Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435965840791664:2108] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.305168Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435965840791660:2106] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.305748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435961936880308:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.305762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:01.305767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435961936880308:2048] [node 1] ICP22 created new session: [2:7574435966231847698:2048]
2025-11-19T14:00:01.305772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435966231847698:2048] [node 1] ICS09 handshake done sender: [2:7574435966231847686:2118] self: [2:7574435966231847690:0] peer: [1:7574435965840791661:0] socket: 142 qp: -1
2025-11-19T14:00:01.305777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435966231847698:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:01.305807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:01.305950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435961936880308:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:01.306126Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435966231847683:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.307007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968173462557:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43504
2025-11-19T14:00:01.307026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968173462558:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43504
2025-11-19T14:00:01.306358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435965840791642:2091] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.306376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:01.306382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435965840791642:2091] [node 4] ICP22 created new session: [1:7574435965840791689:2091]
2025-11-19T14:00:01.306390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435965840791689:2091] [node 4] ICS09 handshake done sender: [1:7574435965840791664:2108] self: [1:7574435965840791663:0] peer: [4:7574435964811522413:0] socket: 141 qp: -1
2025-11-19T14:00:01.306395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435965840791689:2091] [node 4] ICS10 traffic start
2025-11-19T14:00:01.306401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:01.306623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435961545824300:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.306629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:01.306633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435961545824300:2049] [node 3] ICP22 created new session: [1:7574435965840791691:2049]
2025-11-19T14:00:01.306637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435965840791691:2049] [node 3] ICS09 handshake done sender: [1:7574435965840791660:2106] self: [1:7574435965840791644:0] peer: [3:7574435967468770273:0] socket: 139 qp: -1
2025-11-19T14:00:01.306641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435965840791691:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:01.306645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:01.307184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435964811522325:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.307198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:01.307201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435964811522325:2050] [node 3] ICP22 created new session: [4:7574435964811522420:2050]
2025-11-19T14:00:01.307206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435964811522420:2050] [node 3] ICS09 handshake done sender: [4:7574435964811522395:2111] self: [4:7574435964811522408:0] peer: [3:7574435967468770242:0] socket: 119 qp: -1
2025-11-19T14:00:01.307210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435964811522420:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:01.307215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.303954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435967468770239:2102] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.303968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:01.303974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435967468770239:2102] [node 4] ICP22 created new session: [3:7574435967468770275:2102]
2025-11-19T14:00:01.303979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435967468770275:2102] [node 4] ICS09 handshake done sender: [3:7574435967468770246:2106] self: [3:7574435967468770242:0] peer: [4:7574435964811522408:0] socket: 117 qp: -1
2025-11-19T14:00:01.303984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435967468770275:2102] [node 4] ICS10 traffic start
2025-11-19T14:00:01.303989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:01.304023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435967468770268:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15403
2025-11-19T14:00:01.305532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435967468770274:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15403
2025-11-19T14:00:01.307738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435967468770278:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15403
2025-11-19T14:00:01.307767Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435966231847681:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:01.307976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435961936880309:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.307991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:01.307997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435961936880309:2049] [node 3] ICP22 created new session: [2:7574435966231847701:2049]
2025-11-19T14:00:01.308003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435966231847701:2049] [node 3] ICS09 handshake done sender: [2:7574435966231847681:2114] self: [2:7574435966231847672:0] peer: [3:7574435967468770265:0] socket: 129 qp: -1
2025-11-19T14:00:01.308010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435966231847701:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:01.308027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:01.308199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968173462555:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43504
2025-11-19T14:00:01.308393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968173462556:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43504
2025-11-19T14:00:01.308556Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435968173462549:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.309051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435964811522419:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4210
2025-11-19T14:00:01.315878Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435968173462550:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:01.316002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435968173462515:2086] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.316023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:01.316030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435968173462515:2086] [node 4] ICP22 created new session: [6:7574435968173462559:2086]
2025-11-19T14:00:01.316037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435968173462559:2086] [node 4] ICS09 handshake done sender: [6:7574435968173462549:2113] self: [6:7574435968173462552:0] peer: [4:7574435964811522389:0] socket: 136 qp: -1
2025-11-19T14:00:01.316044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435968173462559:2086] [node 4] ICS10 traffic start
2025-11-19T14:00:01.316072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:01.316532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435968173462522:2090] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.316540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:01.316546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435968173462522:2090] [node 5] ICP22 created new session: [6:7574435968173462562:2090]
2025-11-19T14:00:01.316551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435968173462562:2090] [node 5] ICS09 handshake done sender: [6:7574435968173462550:2114] self: [6:7574435968173462554:0] peer: [5:7574435968479439252:0] socket: 144 qp: -1
2025-11-19T14:00:01.316556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435968173462562:2090] [node 5] ICS10 traffic start
2025-11-19T14:00:01.316561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:01.316730Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435967468770268:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.317339Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435967468770243:2104] [node 5] ICP35 event processing took too much time PendingConnection:278(16053802ns) {HandleHandshakeStatus:329(16053531ns) {DropIncomingHandshake:441(45ns) DropOutgoingHandshake:456(208ns) SwitchToState:178(79ns) ProcessPendingSessionEvents:495(12262203ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7285642ns max# 7206513ns}}}
2025-11-19T14:00:01.317359Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435967468770261:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:01.318185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435967468770243:2104] [node 5] ICP18 handshake (actor [3:7574435967468770267:2118]) is too old
2025-11-19T14:00:01.318301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435967468770182:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.318313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:01.318319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435967468770182:2048] [node 1] ICP22 created new session: [3:7574435967468770279:2048]
2025-11-19T14:00:01.318324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435967468770279:2048] [node 1] ICS09 handshake done sender: [3:7574435967468770261:2116] self: [3:7574435967468770273:0] peer: [1:7574435965840791644:0] socket: 149 qp: -1
2025-11-19T14:00:01.318332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435967468770279:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:01.318340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:01.319252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435967468770239:2102] [node 4] ICP18 handshake (actor [3:7574435967468770268:2119]) is too old
2025-11-19T14:00:01.320463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435968479439179:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11068) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435968479439239:2102] held: no
2025-11-19T14:00:01.320792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435964811522323:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30779) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435964811522401:2115] held: no
2025-11-19T14:00:01.321534Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435965840791685:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.321569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435961545824300:2049] [node 3] ICP18 handshake (actor [1:7574435965840791685:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.321755Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435967468770253:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:01.321355Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435964811522401:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:01.321650Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435968173462558:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.321838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435967468770183:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.321855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:01.321861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435967468770183:2049] [node 2] ICP22 created new session: [3:7574435967468770281:2049]
2025-11-19T14:00:01.321872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435967468770281:2049] [node 2] ICS09 handshake done sender: [3:7574435967468770253:2110] self: [3:7574435967468770265:0] peer: [2:7574435966231847672:0] socket: 131 qp: -1
2025-11-19T14:00:01.321880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435967468770281:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:01.321887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:01.322220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435968173462475:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.322232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968173462561:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43504
2025-11-19T14:00:01.322282Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435968173462557:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.322298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435968173462474:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.322595Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435968173462561:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.322621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435968173462473:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.322580Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435966231847684:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.322600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435966231847684:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.324381Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435967468770263:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.324391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435967468770263:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.325291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968173462567:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43504
2025-11-19T14:00:01.325696Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435965840791679:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.325711Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435965840791679:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.326486Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435965840791679:2118] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:01.326495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968173462568:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43504
2025-11-19T14:00:01.326557Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435968173462561:2121] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:01.326631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435968173462473:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.326655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:01.326663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435968173462473:2048] [node 1] ICP22 created new session: [6:7574435968173462569:2048]
2025-11-19T14:00:01.326670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435968173462569:2048] [node 1] ICS09 handshake done sender: [6:7574435968173462561:2121] self: [6:7574435968173462566:0] peer: [1:7574435965840791659:0] socket: 181 qp: -1
2025-11-19T14:00:01.326677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435968173462569:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:01.326684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:01.327342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435965840791640:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.327361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:01.327368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435965840791640:2090] [node 6] ICP22 created new session: [1:7574435965840791695:2090]
2025-11-19T14:00:01.327374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435965840791695:2090] [node 6] ICS09 handshake done sender: [1:7574435965840791679:2118] self: [1:7574435965840791659:0] peer: [6:7574435968173462566:0] socket: 138 qp: -1
2025-11-19T14:00:01.327379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435965840791695:2090] [node 6] ICS10 traffic start
2025-11-19T14:00:01.327386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:01.327824Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435966231847684:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:01.327900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435966231847669:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.327914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:01.327919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435966231847669:2107] [node 6] ICP22 created new session: [2:7574435966231847705:2107]
2025-11-19T14:00:01.327925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435966231847705:2107] [node 6] ICS09 handshake done sender: [2:7574435966231847684:2116] self: [2:7574435966231847673:0] peer: [6:7574435968173462565:0] socket: 153 qp: -1
2025-11-19T14:00:01.327929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435966231847705:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:01.327935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:01.328652Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435967468770263:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:01.329327Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435968173462558:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:01.329400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435968173462475:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.329416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:01.329420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435968173462475:2050] [node 3] ICP22 created new session: [6:7574435968173462571:2050]
2025-11-19T14:00:01.329425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435968173462571:2050] [node 3] ICS09 handshake done sender: [6:7574435968173462558:2120] self: [6:7574435968173462564:0] peer: [3:7574435967468770262:0] socket: 180 qp: -1
2025-11-19T14:00:01.329432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435968173462571:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:01.329438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:01.330250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435967468770259:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.330268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:01.330273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435967468770259:2105] [node 6] ICP22 created new session: [3:7574435967468770284:2105]
2025-11-19T14:00:01.330278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435967468770284:2105] [node 6] ICS09 handshake done sender: [3:7574435967468770263:2117] self: [3:7574435967468770262:0] peer: [6:7574435968173462564:0] socket: 177 qp: -1
2025-11-19T14:00:01.330283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435967468770284:2105] [node 6] ICS10 traffic start
2025-11-19T14:00:01.330288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.331598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968173462573:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43504
2025-11-19T14:00:01.331692Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435968173462557:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:01.331913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435968173462474:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.331933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:01.331943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435968173462474:2049] [node 2] ICP22 created new session: [6:7574435968173462574:2049]
2025-11-19T14:00:01.331949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435968173462574:2049] [node 2] ICS09 handshake done sender: [6:7574435968173462557:2119] self: [6:7574435968173462565:0] peer: [2:7574435966231847673:0] socket: 179 qp: -1
2025-11-19T14:00:01.331956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435968173462574:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:01.331963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.337293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435968479439266:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54133
2025-11-19T14:00:01.337393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435968479439268:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54133
::ffff:127.1.0.0
2025-11-19T14:00:01.339468Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435965840791676:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.339479Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435965840791676:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.340925Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435965840791676:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:01.338099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435964811522323:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.338130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:01.338139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435964811522323:2048] [node 1] ICP22 created new session: [4:7574435964811522422:2048]
2025-11-19T14:00:01.338145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435964811522422:2048] [node 1] ICS09 handshake done sender: [4:7574435964811522401:2115] self: [4:7574435964811522413:0] peer: [1:7574435965840791663:0] socket: 145 qp: -1
2025-11-19T14:00:01.338152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435964811522422:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:01.338159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:01.338965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435968479439267:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54133
2025-11-19T14:00:01.339259Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435968479439267:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.339297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435968479439177:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.340760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435968479439270:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54133
2025-11-19T14:00:01.341172Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435968479439267:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:01.341287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435968479439177:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.342489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435965840791638:2089] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.342514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:01.342519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435965840791638:2089] [node 5] ICP22 created new session: [1:7574435965840791698:2089]
2025-11-19T14:00:01.342529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435965840791698:2089] [node 5] ICS09 handshake done sender: [1:7574435965840791676:2117] self: [1:7574435965840791658:0] peer: [5:7574435968479439269:0] socket: 171 qp: -1
2025-11-19T14:00:01.342534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435965840791698:2089] [node 5] ICS10 traffic start
2025-11-19T14:00:01.342540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:01.342173Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435968479439268:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.342192Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435968479439239:2102] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:01.342228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435968479439178:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.342311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:01.342318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435968479439177:2048] [node 1] ICP22 created new session: [5:7574435968479439272:2048]
2025-11-19T14:00:01.342325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435968479439272:2048] [node 1] ICS09 handshake done sender: [5:7574435968479439267:2119] self: [5:7574435968479439269:0] peer: [1:7574435965840791658:0] socket: 152 qp: -1
2025-11-19T14:00:01.342332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435968479439272:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:01.342344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:01.342673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435968479439179:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.342680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:01.342685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435968479439179:2050] [node 3] ICP22 created new session: [5:7574435968479439274:2050]
2025-11-19T14:00:01.342689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435968479439274:2050] [node 3] ICS09 handshake done sender: [5:7574435968479439239:2102] self: [5:7574435968479439262:0] peer: [3:7574435967468770244:0] socket: 135 qp: -1
2025-11-19T14:00:01.342692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435968479439274:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:01.342697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:01.342337Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435966231847685:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.342351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435966231847685:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.348145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435968479439276:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54133
2025-11-19T14:00:01.348316Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435966231847685:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:01.348515Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435968479439268:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:01.348575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435968479439178:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.348591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:01.348599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435968479439178:2049] [node 2] ICP22 created new session: [5:7574435968479439277:2049]
2025-11-19T14:00:01.348604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435968479439277:2049] [node 2] ICS09 handshake done sender: [5:7574435968479439268:2120] self: [5:7574435968479439271:0] peer: [2:7574435966231847670:0] socket: 155 qp: -1
2025-11-19T14:00:01.348611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435968479439277:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:01.348617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:01.349543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435966231847663:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.349573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:01.349579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435966231847663:2102] [node 5] ICP22 created new session: [2:7574435966231847708:2102]
2025-11-19T14:00:01.349584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435966231847708:2102] [node 5] ICS09 handshake done sender: [2:7574435966231847685:2117] self: [2:7574435966231847670:0] peer: [5:7574435968479439271:0] socket: 158 qp: -1
2025-11-19T14:00:01.349589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435966231847708:2102] [node 5] ICS10 traffic start
2025-11-19T14:00:01.349596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:01.364226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:01.364336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435966231847698:2048] [node 1] ICS01 socket: 142 reason# EndOfStream
2025-11-19T14:00:01.364348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435961936880308:2048] [node 1] ICP30 unregister session Session# [2:7574435966231847698:2048] VirtualId# [2:7574435966231847690:0]
2025-11-19T14:00:01.364356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435966231847698:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:01.364397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:01.364420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:01.368134Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:01.368208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435967468770279:2048] [node 1] ICS01 socket: 149 reason# EndOfStream
2025-11-19T14:00:01.368214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435967468770182:2048] [node 1] ICP30 unregister session Session# [3:7574435967468770279:2048] VirtualId# [3:7574435967468770273:0]
2025-11-19T14:00:01.368220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435967468770279:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:01.368247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:01.368264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:01.373707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:01.373799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435968479439272:2048] [node 1] ICS01 socket: 152 reason# EndOfStream
2025-11-19T14:00:01.373807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435968479439177:2048] [node 1] ICP30 unregister session Session# [5:7574435968479439272:2048] VirtualId# [5:7574435968479439269:0]
2025-11-19T14:00:01.373816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435968479439272:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:01.373871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:01.373896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:01.373935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:01.374001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435964811522422:2048] [node 1] ICS01 socket: 145 reason# EndOfStream
2025-11-19T14:00:01.374008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435964811522323:2048] [node 1] ICP30 unregister session Session# [4:7574435964811522422:2048] VirtualId# [4:7574435964811522413:0]
2025-11-19T14:00:01.374012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435964811522422:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:01.374047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:01.374066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:01.375714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:01.375810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435968173462569:2048] [node 1] ICS01 socket: 181 reason# EndOfStream
2025-11-19T14:00:01.375817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435968173462473:2048] [node 1] ICP30 unregister session Session# [6:7574435968173462569:2048] VirtualId# [6:7574435968173462566:0]
2025-11-19T14:00:01.375822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435968173462569:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:01.375875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:01.375898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:01.388417Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:01.388537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435967468770281:2049] [node 2] ICS01 socket: 131 reason# EndOfStream
2025-11-19T14:00:01.388549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435967468770183:2049] [node 2] ICP30 unregister session Session# [3:7574435967468770281:2049] VirtualId# [3:7574435967468770265:0]
2025-11-19T14:00:01.388558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435967468770281:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:01.388602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:01.388625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:01.403936Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:01.404058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435964811522417:2049] [node 2] ICS01 socket: 137 reason# EndOfStream
2025-11-19T14:00:01.404070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435964811522324:2049] [node 2] ICP30 unregister session Session# [4:7574435964811522417:2049] VirtualId# [4:7574435964811522409:0]
2025-11-19T14:00:01.404080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435964811522417:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:01.404131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:01.404161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:01.404205Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:01.404268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435968173462574:2049] [node 2] ICS01 socket: 179 reason# EndOfStream
2025-11-19T14:00:01.404273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435968173462474:2049] [node 2] ICP30 unregister session Session# [6:7574435968173462574:2049] VirtualId# [6:7574435968173462565:0]
2025-11-19T14:00:01.404277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435968173462574:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:01.404304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:01.404324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:01.404753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:01.404820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435968479439277:2049] [node 2] ICS01 socket: 155 reason# EndOfStream
2025-11-19T14:00:01.404825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435968479439178:2049] [node 2] ICP30 unregister session Session# [5:7574435968479439277:2049] VirtualId# [5:7574435968479439271:0]
2025-11-19T14:00:01.404829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435968479439277:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:01.404861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:01.404881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:01.440843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:01.440962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435968479439274:2050] [node 3] ICS01 socket: 135 reason# EndOfStream
2025-11-19T14:00:01.440976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435968479439179:2050] [node 3] ICP30 unregister session Session# [5:7574435968479439274:2050] VirtualId# [5:7574435968479439262:0]
2025-11-19T14:00:01.440984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435968479439274:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:01.441029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:01.441052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:01.442239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:01.442314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435968173462571:2050] [node 3] ICS01 socket: 180 reason# EndOfStream
2025-11-19T14:00:01.442321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435968173462475:2050] [node 3] ICP30 unregister session Session# [6:7574435968173462571:2050] VirtualId# [6:7574435968173462564:0]
2025-11-19T14:00:01.442327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435968173462571:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:01.442371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:01.442392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:01.485586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:01.485705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435968173462559:2086] [node 4] ICS01 socket: 136 reason# EndOfStream
2025-11-19T14:00:01.485719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435968173462515:2086] [node 4] ICP30 unregister session Session# [6:7574435968173462559:2086] VirtualId# [6:7574435968173462552:0]
2025-11-19T14:00:01.485730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435968173462559:2086] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:01.485785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:01.485814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:01.503912Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T14:00:01.504021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435968173462562:2090] [node 5] ICS07 socket disconnect 144 reason# EndOfStream
2025-11-19T14:00:01.504029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435968173462562:2090] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:01.504075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T14:00:01.504096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435968173462562:2090] [node 5] ICS15 start handshake
2025-11-19T14:00:01.504570Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435968173462522:2090] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30163) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:01.504583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435968173462562:2090] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:01.504588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435968173462522:2090] [node 5] ICP30 unregister session Session# [6:7574435968173462562:2090] VirtualId# [6:7574435968173462554:0]
2025-11-19T14:00:01.504645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
2025-11-19T14:00:01.504652Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435968173462522:2090] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30163) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:01.504657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30163) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:01.695958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435968375843755:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:01.695971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435968375843756:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:01.712764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435967559194037:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:01.712777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435967559194038:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:01.782659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435968008436604:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:01.782694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435968008436605:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:01.783727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435969008803494:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:01.783739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435969008803495:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:01.783760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435969008803496:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:01.839892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435967340683853:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:01.839898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435967340683854:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:01.839908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435967340683855:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:01.873484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435968115845953:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:01.873500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435968115845954:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:01.873504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435968115845955:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.910169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435968008436664:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3321
2025-11-19T14:00:01.910642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435968115845989:2080] [node 4] ICP01 ready to work
2025-11-19T14:00:01.911624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435968375843816:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:01.908667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435967340683897:2082] [node 6] ICP01 ready to work
2025-11-19T14:00:01.908740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435967340683898:2087] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.915958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435967559194097:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52767
2025-11-19T14:00:01.916330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435968115846008:2093] [node 5] ICP01 ready to work
2025-11-19T14:00:01.920070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435967559194099:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.922307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435969008803541:2089] [node 6] ICP01 ready to work
2025-11-19T14:00:01.923954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968115846020:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37435
2025-11-19T14:00:01.923161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435968008436667:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3321
2025-11-19T14:00:01.924869Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435968008436664:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.925477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435968375843818:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:01.927909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435968375843822:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:01.933054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435968008436671:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.933954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968115846030:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37435
2025-11-19T14:00:01.935860Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435968375843755:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24324932ns) {Configure:104(24323226ns) {ProcessConfigured:135(18721545ns) {StartInitialHandshake:157(18721233ns) {DropHandshakes:469(437ns) {DropIncomingHandshake:441(46ns) DropOutgoingHandshake:456(121ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(797ns) {SwitchToState:178(462ns)}}}}}
2025-11-19T14:00:01.936534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435968375843816:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(24851765ns) {Configure:104(24851336ns) {ProcessConfigured:135(19317100ns) {StartInitialHandshake:157(19316273ns) {DropHandshakes:469(417ns) {DropIncomingHandshake:441(22ns) DropOutgoingHandshake:456(22ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:476(330ns) {SwitchToState:178(77ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.936866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435967340683921:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23303
2025-11-19T14:00:01.941609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435967559194101:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:01.941898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435967559194104:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:01.945150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435969008803537:2086] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.945966Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435968375843818:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20355646ns) {Configure:104(20354844ns) {ProcessConfigured:135(19991900ns) {StartInitialHandshake:157(19991272ns) {DropHandshakes:469(757ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(104ns)} GenerateSessionVirtualId:988(260ns) PrepareNewSessionHandshake:476(1042ns) {SwitchToState:178(445ns)}}}}}
2025-11-19T14:00:01.946793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435967559194099:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(18772913ns) {Configure:104(18772178ns) {ProcessConfigured:135(18370221ns) {StartInitialHandshake:157(18369749ns) {DropHandshakes:469(596ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(98ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(950ns) {SwitchToState:178(70ns)}}}}}
2025-11-19T14:00:01.947318Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435967559194097:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.946513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435968375843756:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18521315ns) {Configure:104(18519867ns) {ProcessConfigured:135(18163407ns) {StartInitialHandshake:157(18162804ns) {DropHandshakes:469(746ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(124ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:476(713ns) {SwitchToState:178(150ns)}}}}}
2025-11-19T14:00:01.946254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969008803570:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3960
2025-11-19T14:00:01.946343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969008803571:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3960
2025-11-19T14:00:01.946357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969008803572:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3960
2025-11-19T14:00:01.947855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435969008803496:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19779114ns) {Configure:104(19778158ns) {ProcessConfigured:135(19413575ns) {StartInitialHandshake:157(19413309ns) {DropHandshakes:469(426ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(43ns)} GenerateSessionVirtualId:988(139ns) PrepareNewSessionHandshake:476(571ns) {SwitchToState:178(228ns)}}}}}
2025-11-19T14:00:01.948257Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969008803570:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.949570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435968008436668:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:01.949707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435968008436669:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:01.950131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435968115845954:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(18868035ns) {Configure:104(18866976ns) {ProcessConfigured:135(18482400ns) {StartInitialHandshake:157(18482004ns) {DropHandshakes:469(614ns) {DropIncomingHandshake:441(150ns) DropOutgoingHandshake:456(139ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:476(946ns) {SwitchToState:178(172ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.953035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435968375843834:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
2025-11-19T14:00:01.953057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435968375843835:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
2025-11-19T14:00:01.953080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435968375843836:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.956287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969008803574:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3960
2025-11-19T14:00:01.956397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435968115846008:2093] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(40001183ns) {Configure:104(40000164ns) {ProcessConfigured:135(39710842ns) {StartInitialHandshake:157(39710169ns) {DropHandshakes:469(992ns) {DropIncomingHandshake:441(135ns) DropOutgoingHandshake:456(141ns)} GenerateSessionVirtualId:988(82ns) PrepareNewSessionHandshake:476(1158ns) {SwitchToState:178(437ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.957907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435967340683930:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23303
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.964375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435967559194118:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52767
2025-11-19T14:00:01.964403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435967559194119:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52767
2025-11-19T14:00:01.964414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435967559194120:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52767
2025-11-19T14:00:01.969542Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435968008436604:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(34049186ns) {Configure:104(34048161ns) {ProcessConfigured:135(33711863ns) {StartInitialHandshake:157(33710971ns) {DropHandshakes:469(548ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(120ns)} GenerateSessionVirtualId:988(245ns) PrepareNewSessionHandshake:476(1165ns) {SwitchToState:178(519ns)}}}}}
2025-11-19T14:00:01.975395Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435967340683930:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.975490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435967340683854:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.975825Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969008803574:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.975874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435969008803495:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.975721Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435967559194108:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.975737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435967559194108:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.976626Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435967559194108:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:01.976758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435967559194099:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.976782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:01.976790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435967559194099:2108] [node 5] ICP22 created new session: [2:7574435967559194122:2108]
2025-11-19T14:00:01.976796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435967559194122:2108] [node 5] ICS09 handshake done sender: [2:7574435967559194108:2111] self: [2:7574435967559194102:0] peer: [5:7574435967340683933:0] socket: 145 qp: -1
2025-11-19T14:00:01.976803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435967559194122:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:01.976828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.979441Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435968008436669:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(29593153ns) {Configure:104(29592539ns) {ProcessConfigured:135(29296532ns) {StartInitialHandshake:157(29281180ns) {DropHandshakes:469(860ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(249ns)} GenerateSessionVirtualId:988(482ns) PrepareNewSessionHandshake:476(1794ns) {SwitchToState:178(937ns)}} IncomingHandshake:261(5832ns) {DropIncomingHandshake:441(149ns) PrepareNewSessionHandshake:476(208ns) {SwitchToState:178(158ns)} GenerateSessionVirtualId:988(84ns) IssueIncomingHandshakeReply:193(230ns)}}}}
2025-11-19T14:00:01.979630Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435967559194119:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.980710Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435968375843836:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.978796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969008803578:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3960
2025-11-19T14:00:01.979788Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969008803578:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.980985Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969008803571:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.981010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435969008803496:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.981376Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435968008436667:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.981473Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435968375843834:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.981564Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435967559194114:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.981578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435967559194114:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.983900Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435967340683854:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8142) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435967340683930:2114] held: no
2025-11-19T14:00:01.984149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435968008436673:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.984163Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435968008436673:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.984522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969008803581:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3960
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.986551Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435967559194114:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:01.986854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435967559194101:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.986874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:01.986882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435967559194101:2109] [node 4] ICP22 created new session: [2:7574435967559194127:2109]
2025-11-19T14:00:01.986889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435967559194127:2109] [node 4] ICS09 handshake done sender: [2:7574435967559194114:2113] self: [2:7574435967559194105:0] peer: [4:7574435969008803577:0] socket: 148 qp: -1
2025-11-19T14:00:01.986897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435967559194127:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:01.986931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:01.987652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435967340683934:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23303
2025-11-19T14:00:01.987592Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435968008436681:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.987611Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435968008436681:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.981048Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435968115846030:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.981115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435968115845955:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.984930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968115846034:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37435
2025-11-19T14:00:01.988053Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435969008803574:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:01.987879Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435967340683930:2114] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:01.988140Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969008803572:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.988175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435969008803494:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.988385Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435967559194118:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.988409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435967559194101:2109] [node 4] ICP18 handshake (actor [2:7574435967559194118:2116]) is too old
2025-11-19T14:00:01.988438Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435967559194120:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.988468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435967559194037:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.988345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435969008803495:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.988364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:01.988370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435969008803495:2049] [node 2] ICP22 created new session: [4:7574435969008803583:2049]
2025-11-19T14:00:01.988376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435969008803583:2049] [node 2] ICS09 handshake done sender: [4:7574435969008803574:2117] self: [4:7574435969008803577:0] peer: [2:7574435967559194105:0] socket: 151 qp: -1
2025-11-19T14:00:01.988382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435969008803583:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:01.988400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:01.988908Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435968115846020:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.988978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435968115845989:2080] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.989097Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435967340683921:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.989135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435967340683853:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.989257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435967340683854:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:01.989288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:01.989295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435967340683854:2049] [node 2] ICP22 created new session: [5:7574435967340683937:2049]
2025-11-19T14:00:01.989301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435967340683937:2049] [node 2] ICS09 handshake done sender: [5:7574435967340683930:2114] self: [5:7574435967340683933:0] peer: [2:7574435967559194102:0] socket: 147 qp: -1
2025-11-19T14:00:01.989306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435967340683937:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:01.989318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:01.989171Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435968115846034:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.989200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435968115845954:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:01.992550Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435967559194116:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.992572Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435967559194116:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.994977Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969008803543:2090] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.994992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435969008803543:2090] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.993594Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435967559194116:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:01.996156Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435969008803543:2090] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:01.993708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435967559194104:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.993719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:01.993728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435967559194104:2110] [node 6] ICP22 created new session: [2:7574435967559194132:2110]
2025-11-19T14:00:01.993734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435967559194132:2110] [node 6] ICS09 handshake done sender: [2:7574435967559194116:2114] self: [2:7574435967559194106:0] peer: [6:7574435968115846036:0] socket: 158 qp: -1
2025-11-19T14:00:01.993746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435967559194132:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:01.993757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:01.996595Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435968008436673:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:01.996747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435968008436671:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:01.996765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:01.996772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435968008436671:2111] [node 6] ICP22 created new session: [3:7574435968008436688:2111]
2025-11-19T14:00:01.996779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435968008436688:2111] [node 6] ICS09 handshake done sender: [3:7574435968008436673:2112] self: [3:7574435968008436672:0] peer: [6:7574435968115846033:0] socket: 136 qp: -1
2025-11-19T14:00:01.996784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435968008436688:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:01.996791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:01.997113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435968115845955:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63756) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435968115846030:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:01.996756Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435968375843825:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.996774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435968375843825:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.996828Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435968375843830:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.996835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435968375843830:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.997337Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435968375843827:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:01.997343Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435968375843827:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:01.998138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435967340683939:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23303
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.000075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968115846037:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37435
2025-11-19T14:00:02.000150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968115846038:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37435
2025-11-19T14:00:02.000250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968115846039:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37435
2025-11-19T14:00:02.000282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435968115846040:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37435
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.000583Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435968115846020:2104] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:02.001152Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435968115846034:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:02.000961Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435968375843830:2112] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:02.001027Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435968375843827:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.002116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435972410813337:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37435
2025-11-19T14:00:02.001382Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435968375843835:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.001397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435968375843822:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.001408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:02.001415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435968375843822:2109] [node 4] ICP22 created new session: [1:7574435968375843841:2109]
2025-11-19T14:00:02.001424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435968375843841:2109] [node 4] ICS09 handshake done sender: [1:7574435968375843827:2111] self: [1:7574435968375843826:0] peer: [4:7574435969008803582:0] socket: 140 qp: -1
2025-11-19T14:00:02.001430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435968375843841:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:02.001469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:02.001745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435968375843822:2109] [node 4] ICP18 handshake (actor [1:7574435968375843835:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.002319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435968375843816:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.000835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969008803586:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3960
2025-11-19T14:00:02.002334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:02.002340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435968375843816:2107] [node 5] ICP22 created new session: [1:7574435968375843844:2107]
2025-11-19T14:00:02.004078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435969008803541:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.002346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435968375843844:2107] [node 5] ICS09 handshake done sender: [1:7574435968375843830:2112] self: [1:7574435968375843820:0] peer: [5:7574435967340683936:0] socket: 141 qp: -1
2025-11-19T14:00:02.004101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:02.002351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435968375843844:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:02.004107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435969008803541:2089] [node 6] ICP22 created new session: [4:7574435973303770883:2089]
2025-11-19T14:00:02.002358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:02.004113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435973303770883:2089] [node 6] ICS09 handshake done sender: [4:7574435969008803543:2090] self: [4:7574435969008803542:0] peer: [6:7574435968115846035:0] socket: 133 qp: -1
2025-11-19T14:00:02.004119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435973303770883:2089] [node 6] ICS10 traffic start
2025-11-19T14:00:02.004152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:02.004088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435972303403986:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3321
2025-11-19T14:00:02.004163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435972303403988:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3321
2025-11-19T14:00:02.004691Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435967340683921:2105] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:02.004821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435967340683853:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.004852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:02.004858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435967340683853:2048] [node 1] ICP22 created new session: [5:7574435971635651236:2048]
2025-11-19T14:00:02.004863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435971635651236:2048] [node 1] ICS09 handshake done sender: [5:7574435967340683921:2105] self: [5:7574435967340683936:0] peer: [1:7574435968375843820:0] socket: 142 qp: -1
2025-11-19T14:00:02.004878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435971635651236:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:02.004888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.006349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435968115846037:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.006366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435968115845954:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.006377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:02.006383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435968115845954:2049] [node 2] ICP22 created new session: [6:7574435972410813338:2049]
2025-11-19T14:00:02.006390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435972410813338:2049] [node 2] ICS09 handshake done sender: [6:7574435968115846034:2117] self: [6:7574435968115846036:0] peer: [2:7574435967559194106:0] socket: 159 qp: -1
2025-11-19T14:00:02.006396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435972410813338:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:02.006412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:02.006069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435972670811143:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
2025-11-19T14:00:02.007530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435968115845989:2080] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.007547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:02.007554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435968115845989:2080] [node 4] ICP22 created new session: [6:7574435972410813340:2080]
2025-11-19T14:00:02.007560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435972410813340:2080] [node 4] ICS09 handshake done sender: [6:7574435968115846020:2104] self: [6:7574435968115846035:0] peer: [4:7574435969008803542:0] socket: 135 qp: -1
2025-11-19T14:00:02.007567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435972410813340:2080] [node 4] ICS10 traffic start
2025-11-19T14:00:02.007574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:02.007851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435968115846008:2093] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.008149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435972303403986:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.008280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435968008436605:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.012034Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435968008436681:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:02.012336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435968008436668:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.012352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:02.012371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435968008436668:2108] [node 4] ICP22 created new session: [3:7574435972303403991:2108]
2025-11-19T14:00:02.012378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435972303403991:2108] [node 4] ICS09 handshake done sender: [3:7574435968008436681:2114] self: [3:7574435968008436679:0] peer: [4:7574435969008803579:0] socket: 154 qp: -1
2025-11-19T14:00:02.012408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435972303403991:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:02.012425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:02.013293Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435972410813337:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.013400Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435968115846030:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:02.013574Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435969008803572:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:02.013238Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435972303403988:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.013342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435968008436604:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.013830Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435968008436671:2111] [node 6] ICP35 event processing took too much time PendingConnection:278(16975213ns) {HandleHandshakeStatus:329(16974437ns) {DropIncomingHandshake:441(3847ns) DropOutgoingHandshake:456(436ns) SwitchToState:178(71ns) ProcessPendingSessionEvents:495(4384937ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 4361131ns max# 4290260ns}}}
2025-11-19T14:00:02.014272Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435972670811143:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.014336Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435968375843825:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:02.014437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435969008803494:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23581) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435969008803572:2115] held: no
2025-11-19T14:00:02.014521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435968115845953:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.014826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435973303770886:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3960
2025-11-19T14:00:02.015589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435969008803494:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.015599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:02.015605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435969008803494:2048] [node 1] ICP22 created new session: [4:7574435973303770888:2048]
2025-11-19T14:00:02.015611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435973303770888:2048] [node 1] ICS09 handshake done sender: [4:7574435969008803572:2115] self: [4:7574435969008803582:0] peer: [1:7574435968375843826:0] socket: 144 qp: -1
2025-11-19T14:00:02.015616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435973303770888:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:02.015622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.016050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435971854161430:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52767
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.016031Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435967340683912:2097] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.016044Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435967340683912:2097] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.016547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435968115845955:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.016562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:02.016568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435968115845955:2050] [node 3] ICP22 created new session: [6:7574435972410813345:2050]
2025-11-19T14:00:02.016575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435972410813345:2050] [node 3] ICS09 handshake done sender: [6:7574435968115846030:2114] self: [6:7574435968115846033:0] peer: [3:7574435968008436672:0] socket: 137 qp: -1
2025-11-19T14:00:02.016580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435972410813345:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:02.016588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:02.016944Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435967559194120:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:02.017098Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435967340683912:2097] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:02.017204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435967340683897:2082] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.017220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:02.017225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435967340683897:2082] [node 6] ICP22 created new session: [5:7574435971635651242:2082]
2025-11-19T14:00:02.017231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435971635651242:2082] [node 6] ICS09 handshake done sender: [5:7574435967340683912:2097] self: [5:7574435967340683911:0] peer: [6:7574435972410813342:0] socket: 122 qp: -1
2025-11-19T14:00:02.017236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435971635651242:2082] [node 6] ICS10 traffic start
2025-11-19T14:00:02.017242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:02.017568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435967559194037:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.017589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:02.017594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435967559194037:2048] [node 1] ICP22 created new session: [2:7574435971854161431:2048]
2025-11-19T14:00:02.017599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435971854161431:2048] [node 1] ICS09 handshake done sender: [2:7574435967559194120:2118] self: [2:7574435967559194130:0] peer: [1:7574435968375843819:0] socket: 153 qp: -1
2025-11-19T14:00:02.017605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435971854161431:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:02.017615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.018340Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435969008803578:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:02.018031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435968375843755:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.018051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:02.018056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435968375843755:2048] [node 2] ICP22 created new session: [1:7574435972670811145:2048]
2025-11-19T14:00:02.018060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435972670811145:2048] [node 2] ICS09 handshake done sender: [1:7574435968375843825:2110] self: [1:7574435968375843819:0] peer: [2:7574435967559194130:0] socket: 139 qp: -1
2025-11-19T14:00:02.018065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435972670811145:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:02.018072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:02.018493Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435968375843833:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.018507Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435968375843833:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.018439Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435967559194117:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.018452Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435967559194117:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.018436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435971635651240:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23303
2025-11-19T14:00:02.018520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435971635651239:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23303
2025-11-19T14:00:02.018889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435972303403995:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3321
2025-11-19T14:00:02.018900Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435971635651240:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.018936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435967340683898:2087] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.019201Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435972303403995:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.019286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435971635651238:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23303
2025-11-19T14:00:02.019573Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435971635651238:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.019605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435967340683855:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.019333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435969008803496:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.019352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:02.019359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435969008803496:2050] [node 3] ICP22 created new session: [4:7574435973303770890:2050]
2025-11-19T14:00:02.019365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435973303770890:2050] [node 3] ICS09 handshake done sender: [4:7574435969008803578:2119] self: [4:7574435969008803579:0] peer: [3:7574435968008436679:0] socket: 155 qp: -1
2025-11-19T14:00:02.019371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435973303770890:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:02.019377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:02.020232Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435968008436684:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.020242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435968008436684:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.020975Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435968008436684:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:02.021068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435968008436669:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.021085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:02.021090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435968008436669:2109] [node 5] ICP22 created new session: [3:7574435972303403997:2109]
2025-11-19T14:00:02.021095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435972303403997:2109] [node 5] ICS09 handshake done sender: [3:7574435968008436684:2116] self: [3:7574435968008436680:0] peer: [5:7574435971635651245:0] socket: 178 qp: -1
2025-11-19T14:00:02.021100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435972303403997:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:02.021140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:02.021960Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435968375843832:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.021979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435968375843832:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.022584Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435968115845954:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16124045ns) {HandleHandshakeStatus:329(16123260ns) {DropIncomingHandshake:441(202ns) DropOutgoingHandshake:456(1297ns) SwitchToState:178(86ns) ProcessPendingSessionEvents:495(10662263ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10619194ns max# 10550865ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.024334Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435971635651239:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.025056Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435967340683855:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63756) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435971635651238:2119] held: no
2025-11-19T14:00:02.025295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435971635651246:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23303
2025-11-19T14:00:02.025350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435967340683897:2082] [node 6] ICP18 handshake (actor [5:7574435971635651239:2120]) is too old
2025-11-19T14:00:02.026014Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435971635651238:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:02.023305Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969008803575:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.023322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435969008803575:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.026193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435967340683855:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.026206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:02.026234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435967340683855:2050] [node 3] ICP22 created new session: [5:7574435971635651247:2050]
2025-11-19T14:00:02.026239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435971635651247:2050] [node 3] ICS09 handshake done sender: [5:7574435971635651238:2119] self: [5:7574435971635651245:0] peer: [3:7574435968008436680:0] socket: 125 qp: -1
2025-11-19T14:00:02.026250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435971635651247:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:02.026259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:02.028849Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435968008436668:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(16401356ns) {HandleHandshakeStatus:329(16400464ns) {DropIncomingHandshake:441(179ns) DropOutgoingHandshake:456(180ns) SwitchToState:178(235ns) ProcessPendingSessionEvents:495(10184493ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10160241ns max# 10093955ns}}}
2025-11-19T14:00:02.029030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435968008436668:2108] [node 4] ICP18 handshake (actor [3:7574435972303403995:2119]) is too old
2025-11-19T14:00:02.027800Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435967559194117:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.029383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435968115846008:2093] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1883) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435968115846037:2118] held: no
2025-11-19T14:00:02.029763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435972410813347:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37435
2025-11-19T14:00:02.030054Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435968375843833:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:02.031139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435967559194038:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.031168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:02.031176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435967559194038:2049] [node 3] ICP22 created new session: [2:7574435971854161434:2049]
2025-11-19T14:00:02.031184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435971854161434:2049] [node 3] ICS09 handshake done sender: [2:7574435967559194117:2115] self: [2:7574435967559194107:0] peer: [3:7574435972303403990:0] socket: 161 qp: -1
2025-11-19T14:00:02.031191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435971854161434:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:02.031200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:02.031795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435968375843756:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.031820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:02.031826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435968375843756:2049] [node 3] ICP22 created new session: [1:7574435972670811149:2049]
2025-11-19T14:00:02.031834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435972670811149:2049] [node 3] ICS09 handshake done sender: [1:7574435968375843833:2114] self: [1:7574435968375843824:0] peer: [3:7574435972303403994:0] socket: 174 qp: -1
2025-11-19T14:00:02.031840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435972670811149:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:02.031847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:02.032300Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435968115846037:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:02.034749Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435969008803575:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:02.035728Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435968375843832:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:02.035896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435968008436605:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8142) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435972303403986:2117] held: no
2025-11-19T14:00:02.036211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435969008803537:2086] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.036233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:02.036240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435969008803537:2086] [node 5] ICP22 created new session: [4:7574435973303770893:2086]
2025-11-19T14:00:02.036247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435973303770893:2086] [node 5] ICS09 handshake done sender: [4:7574435969008803575:2118] self: [4:7574435969008803568:0] peer: [5:7574435971635651244:0] socket: 182 qp: -1
2025-11-19T14:00:02.036253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435973303770893:2086] [node 5] ICS10 traffic start
2025-11-19T14:00:02.036262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.036649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435972303403999:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3321
2025-11-19T14:00:02.036699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435972303404000:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3321
2025-11-19T14:00:02.036929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435968375843818:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.036952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:02.036959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435968375843818:2108] [node 6] ICP22 created new session: [1:7574435972670811152:2108]
2025-11-19T14:00:02.036965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435972670811152:2108] [node 6] ICS09 handshake done sender: [1:7574435968375843832:2113] self: [1:7574435968375843823:0] peer: [6:7574435972410813343:0] socket: 172 qp: -1
2025-11-19T14:00:02.036972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435972670811152:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:02.036980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:02.037656Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435967340683898:2087] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30735) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435971635651240:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.040175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435968115846008:2093] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.040203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:02.040212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435968115846008:2093] [node 5] ICP22 created new session: [6:7574435972410813348:2093]
2025-11-19T14:00:02.040220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435972410813348:2093] [node 5] ICS09 handshake done sender: [6:7574435968115846037:2118] self: [6:7574435972410813342:0] peer: [5:7574435967340683911:0] socket: 162 qp: -1
2025-11-19T14:00:02.040227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435972410813348:2093] [node 5] ICS10 traffic start
2025-11-19T14:00:02.040245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:02.039980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435971635651249:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23303
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.044161Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435968008436604:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23581) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435972303403988:2118] held: no
2025-11-19T14:00:02.044271Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435972303403986:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:02.044409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435972410813350:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37435
2025-11-19T14:00:02.045278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435968115845953:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23581) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435972410813337:2122] held: no
2025-11-19T14:00:02.044868Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435971635651240:2121] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:02.045670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435967340683898:2087] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.045684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:02.045693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435967340683898:2087] [node 4] ICP22 created new session: [5:7574435971635651250:2087]
2025-11-19T14:00:02.045701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435971635651250:2087] [node 4] ICS09 handshake done sender: [5:7574435971635651240:2121] self: [5:7574435971635651244:0] peer: [4:7574435969008803568:0] socket: 184 qp: -1
2025-11-19T14:00:02.045709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435971635651250:2087] [node 4] ICS10 traffic start
2025-11-19T14:00:02.045721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:02.052068Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435972410813337:2122] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:02.052262Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435972303403988:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:02.052323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435968008436605:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.052339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:02.052349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435968008436605:2049] [node 2] ICP22 created new session: [3:7574435972303404001:2049]
2025-11-19T14:00:02.052355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435972303404001:2049] [node 2] ICS09 handshake done sender: [3:7574435972303403986:2117] self: [3:7574435972303403990:0] peer: [2:7574435967559194107:0] socket: 163 qp: -1
2025-11-19T14:00:02.052362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435972303404001:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:02.052369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:02.052669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435968115845953:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.052683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:02.052688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435968115845953:2048] [node 1] ICP22 created new session: [6:7574435972410813351:2048]
2025-11-19T14:00:02.052693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435972410813351:2048] [node 1] ICS09 handshake done sender: [6:7574435972410813337:2122] self: [6:7574435972410813343:0] peer: [1:7574435968375843823:0] socket: 173 qp: -1
2025-11-19T14:00:02.052697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435972410813351:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:02.052708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:02.053088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435968008436604:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.053106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:02.053114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435968008436604:2048] [node 1] ICP22 created new session: [3:7574435972303404003:2048]
2025-11-19T14:00:02.053121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435972303404003:2048] [node 1] ICS09 handshake done sender: [3:7574435972303403988:2118] self: [3:7574435972303403994:0] peer: [1:7574435968375843824:0] socket: 176 qp: -1
2025-11-19T14:00:02.053129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435972303404003:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:02.053140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:02.082524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:02.082659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435972303404003:2048] [node 1] ICS01 socket: 176 reason# EndOfStream
2025-11-19T14:00:02.082667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435968008436604:2048] [node 1] ICP30 unregister session Session# [3:7574435972303404003:2048] VirtualId# [3:7574435972303403994:0]
2025-11-19T14:00:02.082674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435972303404003:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.082732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:02.082751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:02.083984Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:02.084101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435971854161431:2048] [node 1] ICS01 socket: 153 reason# EndOfStream
2025-11-19T14:00:02.084114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435967559194037:2048] [node 1] ICP30 unregister session Session# [2:7574435971854161431:2048] VirtualId# [2:7574435967559194130:0]
2025-11-19T14:00:02.084128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435971854161431:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.084204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:02.084247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:02.088910Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:02.089023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435971635651236:2048] [node 1] ICS01 socket: 142 reason# EndOfStream
2025-11-19T14:00:02.089032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435967340683853:2048] [node 1] ICP30 unregister session Session# [5:7574435971635651236:2048] VirtualId# [5:7574435967340683936:0]
2025-11-19T14:00:02.089040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435971635651236:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.089099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:02.089128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:02.089175Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:02.089243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435972410813351:2048] [node 1] ICS01 socket: 173 reason# EndOfStream
2025-11-19T14:00:02.089249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435968115845953:2048] [node 1] ICP30 unregister session Session# [6:7574435972410813351:2048] VirtualId# [6:7574435972410813343:0]
2025-11-19T14:00:02.089253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435972410813351:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.089289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:02.089314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:02.089352Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:02.089418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435973303770888:2048] [node 1] ICS01 socket: 144 reason# EndOfStream
2025-11-19T14:00:02.089422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435969008803494:2048] [node 1] ICP30 unregister session Session# [4:7574435973303770888:2048] VirtualId# [4:7574435969008803582:0]
2025-11-19T14:00:02.089427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435973303770888:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.089456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:02.089477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:02.095139Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:02.095242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435972303404001:2049] [node 2] ICS01 socket: 163 reason# EndOfStream
2025-11-19T14:00:02.095267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435968008436605:2049] [node 2] ICP30 unregister session Session# [3:7574435972303404001:2049] VirtualId# [3:7574435972303403990:0]
2025-11-19T14:00:02.095274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435972303404001:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.095319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:02.095342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:02.104350Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:02.104452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435969008803583:2049] [node 2] ICS01 socket: 151 reason# EndOfStream
2025-11-19T14:00:02.104462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435969008803495:2049] [node 2] ICP30 unregister session Session# [4:7574435969008803583:2049] VirtualId# [4:7574435969008803577:0]
2025-11-19T14:00:02.104468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435969008803583:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.104546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:02.104571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:02.104612Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:02.104682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435972410813338:2049] [node 2] ICS07 socket disconnect 159 reason# EndOfStream
2025-11-19T14:00:02.104688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435972410813338:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.104726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:02.104746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435972410813338:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:02.105151Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:02.105221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435967340683937:2049] [node 2] ICS01 socket: 147 reason# EndOfStream
2025-11-19T14:00:02.105227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435967340683854:2049] [node 2] ICP30 unregister session Session# [5:7574435967340683937:2049] VirtualId# [5:7574435967340683933:0]
2025-11-19T14:00:02.105232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435967340683937:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.105287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:02.105321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:02.105861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435968115845954:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8142) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:02.105878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435972410813338:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:02.105898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435968115845954:2049] [node 2] ICP30 unregister session Session# [6:7574435972410813338:2049] VirtualId# [6:7574435968115846036:0]
2025-11-19T14:00:02.105941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:02.105948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435968115845954:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8142) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:02.105951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8142) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:02.119736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435968115845954:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:02.119803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:02.119886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435972410813345:2050] [node 3] ICS01 socket: 137 reason# EndOfStream
2025-11-19T14:00:02.119898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435968115845955:2050] [node 3] ICP30 unregister session Session# [6:7574435972410813345:2050] VirtualId# [6:7574435968115846033:0]
2025-11-19T14:00:02.119906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435972410813345:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.119969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:02.119993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:02.120188Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:02.120256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435971635651247:2050] [node 3] ICS01 socket: 125 reason# EndOfStream
2025-11-19T14:00:02.120261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435967340683855:2050] [node 3] ICP30 unregister session Session# [5:7574435971635651247:2050] VirtualId# [5:7574435971635651245:0]
2025-11-19T14:00:02.120266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435971635651247:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.120315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:02.120341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:02.131653Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T14:00:02.131833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435971635651250:2087] [node 4] ICS01 socket: 184 reason# EndOfStream
2025-11-19T14:00:02.131843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435967340683898:2087] [node 4] ICP30 unregister session Session# [5:7574435971635651250:2087] VirtualId# [5:7574435971635651244:0]
2025-11-19T14:00:02.131852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435971635651250:2087] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.131914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:02.131938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:02.135870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:02.135977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435972410813340:2080] [node 4] ICS01 socket: 135 reason# EndOfStream
2025-11-19T14:00:02.135987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435968115845989:2080] [node 4] ICP30 unregister session Session# [6:7574435972410813340:2080] VirtualId# [6:7574435968115846035:0]
2025-11-19T14:00:02.135994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435972410813340:2080] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.136060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:02.136087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:02.290760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435972412020930:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:02.290773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435972412020931:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:02.328101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435969988140624:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:02.328124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435969988140625:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:02.330046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435970000455006:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:02.330060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435970000455007:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:02.347936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435969623606331:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:02.347951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435969623606332:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:02.347957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435969623606333:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:02.400928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435969941546291:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:02.400961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435969941546292:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:02.400966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435969941546293:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:02.399463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435971063388217:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:02.399479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435971063388218:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:02.399484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435971063388219:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:02.428403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435969623606349:2063] [node 5] ICP01 ready to work
2025-11-19T14:00:02.428291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435970000455054:2094] [node 6] ICP01 ready to work
2025-11-19T14:00:02.428390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435969941546310:2063] [node 4] ICP01 ready to work
2025-11-19T14:00:02.429207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435972412020994:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:02.430354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435969988140687:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:02.430442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435969988140688:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:02.430513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435969988140689:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:02.439793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435970000455070:2103] [node 5] ICP01 ready to work
2025-11-19T14:00:02.439921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435970000455071:2106] [node 4] ICP01 ready to work
2025-11-19T14:00:02.437575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435969623606386:2066] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.444484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435969941546310:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16007527ns) {Configure:104(16006616ns) {ProcessConfigured:135(15536341ns) {StartInitialHandshake:157(15535946ns) {DropHandshakes:469(509ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(92ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:476(1455ns) {SwitchToState:178(100ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.448137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435969941546341:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36370
2025-11-19T14:00:02.456183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435969988140624:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25608749ns) {Configure:104(25608019ns) {ProcessConfigured:135(25226320ns) {StartInitialHandshake:157(25225430ns) {DropHandshakes:469(596ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(172ns) PrepareNewSessionHandshake:476(1548ns) {SwitchToState:178(750ns)}}}}}
2025-11-19T14:00:02.459898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435969941546342:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36370
2025-11-19T14:00:02.460169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435969941546368:2097] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.462863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435970000455085:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19787
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.475503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969623606409:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
2025-11-19T14:00:02.487637Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435969988140625:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(31381714ns) {Configure:104(31379585ns) {ProcessConfigured:135(30978469ns) {StartInitialHandshake:157(30978237ns) {DropHandshakes:469(1035ns) {DropIncomingHandshake:441(132ns) DropOutgoingHandshake:456(344ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(626ns) {SwitchToState:178(374ns)}}}}}
2025-11-19T14:00:02.484571Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435972412020931:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(55264150ns) {Configure:104(55262980ns) {ProcessConfigured:135(54952587ns) {StartInitialHandshake:157(54951745ns) {DropHandshakes:469(394ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(315ns) PrepareNewSessionHandshake:476(1822ns) {SwitchToState:178(1047ns)}}}}}
2025-11-19T14:00:02.485356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435972412021001:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:02.485549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435972412021002:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:02.488381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435971063388282:2059] [node 5] ICP01 ready to work
2025-11-19T14:00:02.488631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435971063388283:2065] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.491588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435969988140700:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54897
2025-11-19T14:00:02.491610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435969988140701:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54897
2025-11-19T14:00:02.492096Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435969988140701:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.492294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435970000455086:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19787
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.492410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969623606410:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.493460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435972412021003:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19515
2025-11-19T14:00:02.493550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435972412021004:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19515
2025-11-19T14:00:02.495360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435972412021011:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19515
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.497108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969623606411:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.497219Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435969941546368:2097] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(36605892ns) {Configure:104(36604860ns) {ProcessConfigured:135(36217002ns) {StartInitialHandshake:157(36216581ns) {DropHandshakes:469(908ns) {DropIncomingHandshake:441(120ns) DropOutgoingHandshake:456(150ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(1804ns) {SwitchToState:178(1116ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.501942Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435972412021003:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.502223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435970000455087:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19787
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.499650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435971063388275:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5942
2025-11-19T14:00:02.499758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435971063388281:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5942
2025-11-19T14:00:02.503654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969623606412:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
2025-11-19T14:00:02.508323Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435971063388275:2105] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.508362Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435971063388281:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.509715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435971063388219:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.509739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435971063388283:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.512287Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435970000455087:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.512332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435970000455006:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.512435Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969623606411:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.512471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435969623606331:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.512662Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435972412020998:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.512673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435972412020998:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.513150Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435972412021000:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.513154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435972412021000:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.514146Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435972412020998:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:02.514183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969623606414:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
2025-11-19T14:00:02.514324Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435972412021000:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:02.514590Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435969623606411:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:02.514416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435972412020931:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.514442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:02.514449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435972412020931:2049] [node 3] ICP22 created new session: [1:7574435972412021015:2049]
2025-11-19T14:00:02.514455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435972412021015:2049] [node 3] ICS09 handshake done sender: [1:7574435972412020998:2111] self: [1:7574435972412020995:0] peer: [3:7574435970000455088:0] socket: 152 qp: -1
2025-11-19T14:00:02.514462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435972412021015:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:02.514472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:02.514770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435972412020994:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.514780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:02.514787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435972412020994:2108] [node 4] ICP22 created new session: [1:7574435972412021017:2108]
2025-11-19T14:00:02.514791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435972412021017:2108] [node 4] ICS09 handshake done sender: [1:7574435972412021000:2112] self: [1:7574435972412020999:0] peer: [4:7574435969623606413:0] socket: 153 qp: -1
2025-11-19T14:00:02.514796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435972412021017:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:02.514802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:02.514717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435969623606331:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.514734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:02.514741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435969623606331:2048] [node 1] ICP22 created new session: [4:7574435969623606415:2048]
2025-11-19T14:00:02.514746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435969623606415:2048] [node 1] ICS09 handshake done sender: [4:7574435969623606411:2117] self: [4:7574435969623606413:0] peer: [1:7574435972412020999:0] socket: 154 qp: -1
2025-11-19T14:00:02.514753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435969623606415:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:02.514761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.515982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435969988140706:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54897
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.516601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435970000455089:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19787
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.517421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435970000455090:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19787
2025-11-19T14:00:02.517502Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435970000455086:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.517522Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435970000455087:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:02.517633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435970000455006:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.517652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:02.517658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435970000455006:2048] [node 1] ICP22 created new session: [3:7574435970000455091:2048]
2025-11-19T14:00:02.517665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435970000455091:2048] [node 1] ICS09 handshake done sender: [3:7574435970000455087:2117] self: [3:7574435970000455088:0] peer: [1:7574435972412020995:0] socket: 155 qp: -1
2025-11-19T14:00:02.517671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435970000455091:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:02.517680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:02.518066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435970000455007:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.516802Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435969941546342:2094] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.516836Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435969941546341:2093] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.519671Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435969988140693:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.519686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435969988140693:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.520494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435969988140707:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54897
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.522049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435970000455095:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19787
2025-11-19T14:00:02.522447Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969623606410:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.522485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435969623606332:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.522683Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435969988140693:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:02.522879Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435969988140695:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.522887Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435969988140695:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.523272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435969988140625:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.523287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:02.523294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435969988140625:2049] [node 3] ICP22 created new session: [2:7574435969988140709:2049]
2025-11-19T14:00:02.523301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435969988140709:2049] [node 3] ICS09 handshake done sender: [2:7574435969988140693:2111] self: [2:7574435969988140692:0] peer: [3:7574435970000455093:0] socket: 147 qp: -1
2025-11-19T14:00:02.523309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435969988140709:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:02.523317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.524160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969623606419:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
2025-11-19T14:00:02.524251Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435970000455090:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.524443Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435969988140695:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:02.524610Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435969623606410:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:02.524410Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435972412021011:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.524686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435969623606332:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.524701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:02.524706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435969623606332:2049] [node 2] ICP22 created new session: [4:7574435969623606420:2049]
2025-11-19T14:00:02.524711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435969623606420:2049] [node 2] ICS09 handshake done sender: [4:7574435969623606410:2116] self: [4:7574435969623606418:0] peer: [2:7574435969988140694:0] socket: 150 qp: -1
2025-11-19T14:00:02.524717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435969623606420:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:02.524722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:02.526254Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435970000455086:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:02.526434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435970000455007:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.526445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:02.526450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435970000455007:2049] [node 2] ICP22 created new session: [3:7574435970000455097:2049]
2025-11-19T14:00:02.526455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435970000455097:2049] [node 2] ICS09 handshake done sender: [3:7574435970000455086:2116] self: [3:7574435970000455093:0] peer: [2:7574435969988140692:0] socket: 149 qp: -1
2025-11-19T14:00:02.526460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435970000455097:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:02.526466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:02.527694Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435969988140700:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.527721Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435969988140707:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.527938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435969988140687:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.527951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:02.527956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435969988140687:2107] [node 4] ICP22 created new session: [2:7574435969988140713:2107]
2025-11-19T14:00:02.527960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435969988140713:2107] [node 4] ICS09 handshake done sender: [2:7574435969988140695:2112] self: [2:7574435969988140694:0] peer: [4:7574435969623606418:0] socket: 148 qp: -1
2025-11-19T14:00:02.527965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435969988140713:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:02.527971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.529400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435971063388299:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5942
2025-11-19T14:00:02.529426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435971063388300:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5942
2025-11-19T14:00:02.529664Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969623606388:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.529673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435969623606388:2099] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.536998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435969941546371:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36370
2025-11-19T14:00:02.537058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435969941546293:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.537079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435969941546310:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.537159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435969941546372:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36370
2025-11-19T14:00:02.538111Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435969941546372:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.538141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435969941546291:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.538281Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435972412021004:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.538334Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435972412021006:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.538340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435972412021006:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.539196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435969941546376:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36370
2025-11-19T14:00:02.530581Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435970000455056:2095] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.530589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435970000455056:2095] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.541549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435969988140624:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.541573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435969988140625:2049] [node 3] ICP18 handshake (actor [2:7574435969988140707:2118]) is too old
2025-11-19T14:00:02.542043Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435969623606388:2099] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.542244Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969623606409:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.542279Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969623606385:2098] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.542283Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435969623606385:2098] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.542687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435969623606386:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.542698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:02.542705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435969623606386:2066] [node 6] ICP22 created new session: [4:7574435969623606424:2066]
2025-11-19T14:00:02.542710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435969623606424:2066] [node 6] ICS09 handshake done sender: [4:7574435969623606388:2099] self: [4:7574435969623606387:0] peer: [6:7574435971063388298:0] socket: 131 qp: -1
2025-11-19T14:00:02.542716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435969623606424:2066] [node 6] ICS10 traffic start
2025-11-19T14:00:02.542735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:02.543073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969623606423:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
2025-11-19T14:00:02.543309Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435970000455056:2095] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:02.543463Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435970000455085:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.543497Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435970000455075:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.543500Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435970000455075:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.544256Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435972412021006:2115] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:02.544378Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435969623606385:2098] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:02.544554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435970000455054:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.544569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:02.544576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435970000455054:2094] [node 6] ICP22 created new session: [3:7574435970000455101:2094]
2025-11-19T14:00:02.544582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435970000455101:2094] [node 6] ICS09 handshake done sender: [3:7574435970000455056:2095] self: [3:7574435970000455055:0] peer: [6:7574435971063388297:0] socket: 127 qp: -1
2025-11-19T14:00:02.544589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435970000455101:2094] [node 6] ICS10 traffic start
2025-11-19T14:00:02.544598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.547121Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969623606412:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.547159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435969623606349:2063] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.547173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:02.547177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435969623606349:2063] [node 5] ICP22 created new session: [4:7574435969623606428:2063]
2025-11-19T14:00:02.547182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435969623606428:2063] [node 5] ICS09 handshake done sender: [4:7574435969623606385:2098] self: [4:7574435969623606384:0] peer: [5:7574435969941546374:0] socket: 130 qp: -1
2025-11-19T14:00:02.547187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435969623606428:2063] [node 5] ICS10 traffic start
2025-11-19T14:00:02.547195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:02.546443Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435971063388300:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.546562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435971063388301:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5942
2025-11-19T14:00:02.546844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435971063388302:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5942
2025-11-19T14:00:02.546960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435971063388217:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.547013Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435971063388281:2111] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:02.547707Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435971063388275:2105] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:02.547813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435971063388283:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.547824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:02.547830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435971063388283:2065] [node 4] ICP22 created new session: [6:7574435971063388304:2065]
2025-11-19T14:00:02.547834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435971063388304:2065] [node 4] ICS09 handshake done sender: [6:7574435971063388281:2111] self: [6:7574435971063388298:0] peer: [4:7574435969623606387:0] socket: 136 qp: -1
2025-11-19T14:00:02.547842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435971063388304:2065] [node 4] ICS10 traffic start
2025-11-19T14:00:02.547861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:02.549392Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435970000455075:2109] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:02.548506Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435972412020991:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.548519Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435972412020991:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.549065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435972412021001:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.549080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:02.549086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435972412021001:2109] [node 5] ICP22 created new session: [1:7574435972412021024:2109]
2025-11-19T14:00:02.549094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435972412021024:2109] [node 5] ICS09 handshake done sender: [1:7574435972412021006:2115] self: [1:7574435972412021005:0] peer: [5:7574435969941546375:0] socket: 173 qp: -1
2025-11-19T14:00:02.549102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435972412021024:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:02.549111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:02.549382Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435972412021008:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.549388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435972412021008:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.548222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435971063388219:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.548241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:02.548247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435971063388219:2050] [node 3] ICP22 created new session: [6:7574435971063388306:2050]
2025-11-19T14:00:02.548252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435971063388306:2050] [node 3] ICS09 handshake done sender: [6:7574435971063388275:2105] self: [6:7574435971063388297:0] peer: [3:7574435970000455055:0] socket: 128 qp: -1
2025-11-19T14:00:02.548256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435971063388306:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:02.548261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:02.550662Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435969941546371:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.551702Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435969941546310:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17767) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435969941546341:2093] held: no
2025-11-19T14:00:02.551825Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435969941546372:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.551945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435969941546377:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36370
2025-11-19T14:00:02.551965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435969941546378:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36370
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.552114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435969941546379:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36370
2025-11-19T14:00:02.552355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435969941546291:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.552369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:02.552373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435969941546291:2048] [node 1] ICP22 created new session: [5:7574435969941546380:2048]
2025-11-19T14:00:02.552379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435969941546380:2048] [node 1] ICS09 handshake done sender: [5:7574435969941546372:2118] self: [5:7574435969941546375:0] peer: [1:7574435972412021005:0] socket: 178 qp: -1
2025-11-19T14:00:02.552385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435969941546380:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:02.552391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:02.553942Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435969941546341:2093] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:02.553920Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435969988140706:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.554189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435969623606386:2066] [node 6] ICP18 handshake (actor [4:7574435969623606412:2118]) is too old
2025-11-19T14:00:02.554655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435970000455070:2103] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.554686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:02.554697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435970000455070:2103] [node 5] ICP22 created new session: [3:7574435970000455105:2103]
2025-11-19T14:00:02.554702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435970000455105:2103] [node 5] ICS09 handshake done sender: [3:7574435970000455075:2109] self: [3:7574435970000455074:0] peer: [5:7574435969941546373:0] socket: 133 qp: -1
2025-11-19T14:00:02.554707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435970000455105:2103] [node 5] ICS10 traffic start
2025-11-19T14:00:02.554713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:02.556618Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435969623606423:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.556651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435969623606333:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.556859Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435970000455079:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.556867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435970000455079:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.557468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435969623606431:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26437
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.558670Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435971063388299:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.558933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435971063388218:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.558949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435971063388308:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5942
2025-11-19T14:00:02.559245Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435970000455079:2111] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:02.559439Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435969623606423:2121] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:02.559505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435969623606333:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.559520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:02.559525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435969623606333:2050] [node 3] ICP22 created new session: [4:7574435969623606432:2050]
2025-11-19T14:00:02.559530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435969623606432:2050] [node 3] ICS09 handshake done sender: [4:7574435969623606423:2121] self: [4:7574435969623606430:0] peer: [3:7574435970000455078:0] socket: 181 qp: -1
2025-11-19T14:00:02.559535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435969623606432:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:02.559542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:02.560779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435970000455071:2106] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.560812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:02.560818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435970000455071:2106] [node 4] ICP22 created new session: [3:7574435970000455108:2106]
2025-11-19T14:00:02.560823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435970000455108:2106] [node 4] ICS09 handshake done sender: [3:7574435970000455079:2111] self: [3:7574435970000455078:0] peer: [4:7574435969623606430:0] socket: 177 qp: -1
2025-11-19T14:00:02.560830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435970000455108:2106] [node 4] ICS10 traffic start
2025-11-19T14:00:02.560838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.562781Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435969988140697:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.562792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435969988140697:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.563304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435969988140718:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54897
2025-11-19T14:00:02.563459Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435969941546342:2094] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:02.564459Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435969988140697:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:02.563738Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435972412020991:2107] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:02.563947Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435972412021008:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:02.565085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435969988140688:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.565102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:02.565108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435969988140688:2108] [node 6] ICP22 created new session: [2:7574435969988140721:2108]
2025-11-19T14:00:02.565113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435969988140721:2108] [node 6] ICS09 handshake done sender: [2:7574435969988140697:2113] self: [2:7574435969988140696:0] peer: [6:7574435971063388310:0] socket: 171 qp: -1
2025-11-19T14:00:02.565117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435969988140721:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:02.565125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:02.565577Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435969988140700:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.566044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435972412020930:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.566067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:02.566074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435972412020930:2048] [node 2] ICP22 created new session: [1:7574435972412021029:2048]
2025-11-19T14:00:02.566081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435972412021029:2048] [node 2] ICS09 handshake done sender: [1:7574435972412020991:2107] self: [1:7574435972412020990:0] peer: [2:7574435969988140716:0] socket: 126 qp: -1
2025-11-19T14:00:02.566087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435972412021029:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:02.566094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:02.565670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435969988140720:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54897
2025-11-19T14:00:02.565679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435969988140624:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.565688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:02.565692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435969988140624:2048] [node 1] ICP22 created new session: [2:7574435969988140723:2048]
2025-11-19T14:00:02.565696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435969988140723:2048] [node 1] ICS09 handshake done sender: [2:7574435969988140700:2115] self: [2:7574435969988140716:0] peer: [1:7574435972412020990:0] socket: 144 qp: -1
2025-11-19T14:00:02.565700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435969988140723:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:02.565705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:02.568448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435969941546310:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.568467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:02.568473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435969941546310:2063] [node 4] ICP22 created new session: [5:7574435969941546384:2063]
2025-11-19T14:00:02.568479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435969941546384:2063] [node 4] ICS09 handshake done sender: [5:7574435969941546341:2093] self: [5:7574435969941546374:0] peer: [4:7574435969623606384:0] socket: 134 qp: -1
2025-11-19T14:00:02.568485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435969941546384:2063] [node 4] ICS10 traffic start
2025-11-19T14:00:02.568491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:02.568659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435969941546293:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.568666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:02.568670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435969941546293:2050] [node 3] ICP22 created new session: [5:7574435969941546386:2050]
2025-11-19T14:00:02.568674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435969941546386:2050] [node 3] ICS09 handshake done sender: [5:7574435969941546342:2094] self: [5:7574435969941546373:0] peer: [3:7574435970000455074:0] socket: 135 qp: -1
2025-11-19T14:00:02.568677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435969941546386:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:02.568684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:02.568800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435969941546293:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:02.569017Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435969941546377:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.569371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435971063388218:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29142) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435971063388299:2117] held: no
2025-11-19T14:00:02.569471Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435971063388300:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:02.567979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435972412021002:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.567994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:02.568000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435972412021002:2110] [node 6] ICP22 created new session: [1:7574435972412021031:2110]
2025-11-19T14:00:02.568005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435972412021031:2110] [node 6] ICS09 handshake done sender: [1:7574435972412021008:2116] self: [1:7574435972412021007:0] peer: [6:7574435971063388303:0] socket: 174 qp: -1
2025-11-19T14:00:02.568010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435972412021031:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:02.568017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.576677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435972412021028:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19515
2025-11-19T14:00:02.578014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435969941546292:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.578850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435971063388311:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5942
2025-11-19T14:00:02.579059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435971063388217:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.579078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:02.579085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435971063388217:2048] [node 1] ICP22 created new session: [6:7574435971063388312:2048]
2025-11-19T14:00:02.579092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435971063388312:2048] [node 1] ICS09 handshake done sender: [6:7574435971063388300:2118] self: [6:7574435971063388303:0] peer: [1:7574435972412021007:0] socket: 132 qp: -1
2025-11-19T14:00:02.579099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435971063388312:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:02.579108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:02.580471Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435972412021028:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.580542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435972412021002:2110] [node 6] ICP18 handshake (actor [1:7574435972412021028:2118]) is too old
2025-11-19T14:00:02.584155Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435969988140699:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.584172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435969988140699:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.585603Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435971063388299:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:02.588020Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435969988140699:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.588245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435971063388218:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.588287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:02.588296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435971063388218:2049] [node 2] ICP22 created new session: [6:7574435971063388315:2049]
2025-11-19T14:00:02.588302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435971063388315:2049] [node 2] ICS09 handshake done sender: [6:7574435971063388299:2117] self: [6:7574435971063388310:0] peer: [2:7574435969988140696:0] socket: 129 qp: -1
2025-11-19T14:00:02.588308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435971063388315:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:02.588317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:02.588565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435971063388314:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5942
2025-11-19T14:00:02.588979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435969988140689:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.588994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:02.589000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435969988140689:2109] [node 5] ICP22 created new session: [2:7574435969988140726:2109]
2025-11-19T14:00:02.589006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435969988140726:2109] [node 5] ICS09 handshake done sender: [2:7574435969988140699:2114] self: [2:7574435969988140698:0] peer: [5:7574435969941546389:0] socket: 145 qp: -1
2025-11-19T14:00:02.589011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435969988140726:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:02.589021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.589688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435969941546390:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36370
2025-11-19T14:00:02.589904Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435969941546377:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:02.590001Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435969988140720:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.590034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435969988140689:2109] [node 5] ICP18 handshake (actor [2:7574435969988140720:2120]) is too old
2025-11-19T14:00:02.590243Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435971063388314:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.590273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435971063388282:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:02.590510Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435969941546370:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:02.590521Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435969941546370:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:02.591135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435969941546292:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.591148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:02.591185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435969941546292:2049] [node 2] ICP22 created new session: [5:7574435969941546391:2049]
2025-11-19T14:00:02.591191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435969941546391:2049] [node 2] ICS09 handshake done sender: [5:7574435969941546377:2120] self: [5:7574435969941546389:0] peer: [2:7574435969988140698:0] socket: 184 qp: -1
2025-11-19T14:00:02.591198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435969941546391:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:02.591206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:02.612478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435971063388318:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5942
2025-11-19T14:00:02.612689Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435969941546370:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:02.612911Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435971063388314:2123] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:02.612978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435971063388282:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:02.613006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:02.613013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435971063388282:2059] [node 5] ICP22 created new session: [6:7574435971063388319:2059]
2025-11-19T14:00:02.613020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435971063388319:2059] [node 5] ICS09 handshake done sender: [6:7574435971063388314:2123] self: [6:7574435971063388317:0] peer: [5:7574435969941546369:0] socket: 143 qp: -1
2025-11-19T14:00:02.613026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435971063388319:2059] [node 5] ICS10 traffic start
2025-11-19T14:00:02.613037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:02.614105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435969941546368:2097] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:02.614125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:02.614131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435969941546368:2097] [node 6] ICP22 created new session: [5:7574435969941546394:2097]
2025-11-19T14:00:02.614136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435969941546394:2097] [node 6] ICS09 handshake done sender: [5:7574435969941546370:2116] self: [5:7574435969941546369:0] peer: [6:7574435971063388317:0] socket: 138 qp: -1
2025-11-19T14:00:02.614141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435969941546394:2097] [node 6] ICS10 traffic start
2025-11-19T14:00:02.614150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:02.624384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:02.624484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435969988140723:2048] [node 1] ICS01 socket: 144 reason# EndOfStream
2025-11-19T14:00:02.624494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435969988140624:2048] [node 1] ICP30 unregister session Session# [2:7574435969988140723:2048] VirtualId# [2:7574435969988140716:0]
2025-11-19T14:00:02.624502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435969988140723:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.624548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:02.624570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:02.625361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:02.625448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435970000455091:2048] [node 1] ICS01 socket: 155 reason# EndOfStream
2025-11-19T14:00:02.625460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435970000455006:2048] [node 1] ICP30 unregister session Session# [3:7574435970000455091:2048] VirtualId# [3:7574435970000455088:0]
2025-11-19T14:00:02.625467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435970000455091:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.625506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:02.625531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:02.636058Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:02.636169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435969623606415:2048] [node 1] ICS01 socket: 154 reason# EndOfStream
2025-11-19T14:00:02.636180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435969623606331:2048] [node 1] ICP30 unregister session Session# [4:7574435969623606415:2048] VirtualId# [4:7574435969623606413:0]
2025-11-19T14:00:02.636188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435969623606415:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.636261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:02.636295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:02.636186Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:02.636306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435969941546380:2048] [node 1] ICS01 socket: 178 reason# EndOfStream
2025-11-19T14:00:02.636314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435969941546291:2048] [node 1] ICP30 unregister session Session# [5:7574435969941546380:2048] VirtualId# [5:7574435969941546375:0]
2025-11-19T14:00:02.636320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435969941546380:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.636364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:02.636386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:02.636423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:02.636511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435971063388312:2048] [node 1] ICS01 socket: 132 reason# EndOfStream
2025-11-19T14:00:02.636515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435971063388217:2048] [node 1] ICP30 unregister session Session# [6:7574435971063388312:2048] VirtualId# [6:7574435971063388303:0]
2025-11-19T14:00:02.636520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435971063388312:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.636543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:02.636564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:02.643131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:02.643245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435970000455097:2049] [node 2] ICS01 socket: 149 reason# EndOfStream
2025-11-19T14:00:02.643254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435970000455007:2049] [node 2] ICP30 unregister session Session# [3:7574435970000455097:2049] VirtualId# [3:7574435970000455093:0]
2025-11-19T14:00:02.643272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435970000455097:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.643307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:02.643326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:02.647937Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:02.648070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435971063388315:2049] [node 2] ICS01 socket: 129 reason# EndOfStream
2025-11-19T14:00:02.648081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435971063388218:2049] [node 2] ICP30 unregister session Session# [6:7574435971063388315:2049] VirtualId# [6:7574435971063388310:0]
2025-11-19T14:00:02.648088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435971063388315:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.648154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:02.648180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:02.648226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:02.648317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435969941546391:2049] [node 2] ICS01 socket: 184 reason# EndOfStream
2025-11-19T14:00:02.648322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435969941546292:2049] [node 2] ICP30 unregister session Session# [5:7574435969941546391:2049] VirtualId# [5:7574435969941546389:0]
2025-11-19T14:00:02.648327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435969941546391:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.648361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:02.648386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:02.648518Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:02.648582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435969623606420:2049] [node 2] ICS01 socket: 150 reason# EndOfStream
2025-11-19T14:00:02.648595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435969623606332:2049] [node 2] ICP30 unregister session Session# [4:7574435969623606420:2049] VirtualId# [4:7574435969623606418:0]
2025-11-19T14:00:02.648600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435969623606420:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.648637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:02.648665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:02.672106Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:02.672204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435971063388306:2050] [node 3] ICS01 socket: 128 reason# EndOfStream
2025-11-19T14:00:02.672214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435971063388219:2050] [node 3] ICP30 unregister session Session# [6:7574435971063388306:2050] VirtualId# [6:7574435971063388297:0]
2025-11-19T14:00:02.672222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435971063388306:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.672279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:02.672307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:02.672349Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:02.672478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435969941546386:2050] [node 3] ICS01 socket: 135 reason# EndOfStream
2025-11-19T14:00:02.672483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435969941546293:2050] [node 3] ICP30 unregister session Session# [5:7574435969941546386:2050] VirtualId# [5:7574435969941546373:0]
2025-11-19T14:00:02.672488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435969941546386:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.672516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:02.672536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:02.705673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:02.705777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435971063388304:2065] [node 4] ICS01 socket: 136 reason# EndOfStream
2025-11-19T14:00:02.705788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435971063388283:2065] [node 4] ICP30 unregister session Session# [6:7574435971063388304:2065] VirtualId# [6:7574435971063388298:0]
2025-11-19T14:00:02.705797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435971063388304:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:02.705864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:02.705894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:02.946665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435970720847745:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:02.946688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435970720847746:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:02.976069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435971166505736:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:02.976086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435971166505737:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:03.028273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435975545482062:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:03.028290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435975545482063:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:03.063652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435975911921546:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:03.063668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435975911921547:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:03.063673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435975911921548:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:03.092176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435975141598665:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:03.092191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435975141598666:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:03.092196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435975141598667:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:03.119385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435974470707977:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:03.119382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435974470707976:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:03.119396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435974470707978:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:03.157335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435975015815102:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:03.159282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435974470708021:2087] [node 4] ICP01 ready to work
2025-11-19T14:00:03.160161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435975911921602:2091] [node 6] ICP01 ready to work
2025-11-19T14:00:03.164694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435975015815109:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:03.168479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435975545482125:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:03.168564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435975545482126:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:03.173987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435974470708045:2093] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.175047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435975015815117:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.177355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435975015815120:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41833
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.179132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974470708054:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13139
2025-11-19T14:00:03.176883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435975911921617:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59208
2025-11-19T14:00:03.179890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435975141598720:2097] [node 6] ICP01 ready to work
2025-11-19T14:00:03.180086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435975141598734:2104] [node 4] ICP01 ready to work
2025-11-19T14:00:03.184192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435971166505737:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18891437ns) {Configure:104(18889877ns) {ProcessConfigured:135(18508354ns) {StartInitialHandshake:157(18507634ns) {DropHandshakes:469(504ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(293ns) PrepareNewSessionHandshake:476(1744ns) {SwitchToState:178(1041ns)}}}}}
2025-11-19T14:00:03.184371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435975461473097:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:03.184517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435975461473098:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:03.184640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435975461473099:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.186254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974470708055:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13139
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.188036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435975545482125:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19420224ns) {Configure:104(19419662ns) {ProcessConfigured:135(19117880ns) {StartInitialHandshake:157(19117321ns) {DropHandshakes:469(817ns) {DropIncomingHandshake:441(163ns) DropOutgoingHandshake:456(76ns)} GenerateSessionVirtualId:988(62ns) PrepareNewSessionHandshake:476(1026ns) {SwitchToState:178(559ns)}}}}}
2025-11-19T14:00:03.188246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435975545482132:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:03.189122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435975545482133:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33885
2025-11-19T14:00:03.189135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435975545482134:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33885
2025-11-19T14:00:03.191834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435975911921609:2088] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.196272Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435975545482126:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16773446ns) {Configure:104(16771681ns) {ProcessConfigured:135(16423216ns) {StartInitialHandshake:157(16422464ns) {DropHandshakes:469(759ns) {DropIncomingHandshake:441(144ns) DropOutgoingHandshake:456(136ns)} GenerateSessionVirtualId:988(534ns) PrepareNewSessionHandshake:476(1232ns) {SwitchToState:178(885ns)}}}}}
2025-11-19T14:00:03.198597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435975141598736:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47713
2025-11-19T14:00:03.198620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435975141598737:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47713
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.209321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435975461473111:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9790
2025-11-19T14:00:03.209584Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435975461473111:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.208353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435975015815121:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41833
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.211871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435975141598720:2097] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19908887ns) {Configure:104(19908139ns) {ProcessConfigured:135(19450591ns) {StartInitialHandshake:157(19450085ns) {DropHandshakes:469(476ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(425ns) PrepareNewSessionHandshake:476(1003ns) {SwitchToState:178(630ns)}}}}}
2025-11-19T14:00:03.216252Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435974470708055:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.216299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435974470707977:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.216768Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435975141598736:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.216799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435975141598665:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.214720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435975911921623:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59208
2025-11-19T14:00:03.219936Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435975461473101:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.219965Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435975461473101:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.223934Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435975015815111:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.223962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435975015815111:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.225000Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435975015815120:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.225057Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435975015815111:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:03.225596Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435975911921617:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.225465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435975015815109:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.225483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:03.225492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435975015815109:2109] [node 5] ICP22 created new session: [1:7574435975015815123:2109]
2025-11-19T14:00:03.225498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435975015815123:2109] [node 5] ICS09 handshake done sender: [1:7574435975015815111:2111] self: [1:7574435975015815110:0] peer: [5:7574435975141598745:0] socket: 136 qp: -1
2025-11-19T14:00:03.225505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435975015815123:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:03.225514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:03.225749Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435975911921623:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.225833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435975911921546:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.225850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435975911921547:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.227865Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435975141598737:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.228038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435975141598746:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47713
2025-11-19T14:00:03.228362Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435975141598736:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:03.228396Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435974470708054:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.228460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435974470708021:2087] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.231476Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435975461473103:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.231491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435975461473103:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.232911Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435975461473101:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:03.226135Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435975015815108:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.226141Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435975015815108:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.235290Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435975015815108:2110] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:03.235392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435975015815102:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.235407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:03.235416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435975015815102:2107] [node 4] ICP22 created new session: [1:7574435975015815128:2107]
2025-11-19T14:00:03.235423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435975015815128:2107] [node 4] ICS09 handshake done sender: [1:7574435975015815108:2110] self: [1:7574435975015815107:0] peer: [4:7574435975911921626:0] socket: 135 qp: -1
2025-11-19T14:00:03.235430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435975015815128:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:03.235452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:03.240126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435975141598665:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.240166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:03.240177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435975141598665:2048] [node 1] ICP22 created new session: [5:7574435975141598748:2048]
2025-11-19T14:00:03.240184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435975141598748:2048] [node 1] ICS09 handshake done sender: [5:7574435975141598736:2112] self: [5:7574435975141598745:0] peer: [1:7574435975015815110:0] socket: 146 qp: -1
2025-11-19T14:00:03.240194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435975141598748:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:03.240206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.232677Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435975911921612:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.232696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435975911921612:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.240832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435975911921628:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59208
2025-11-19T14:00:03.240913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435975911921629:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59208
2025-11-19T14:00:03.240994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435975911921630:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59208
2025-11-19T14:00:03.240086Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435975545482134:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.240135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435975545482063:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.241142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435975545482144:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33885
2025-11-19T14:00:03.241192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435975545482145:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33885
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.252247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435975911921631:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59208
2025-11-19T14:00:03.252368Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435975911921628:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.252402Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435975911921617:2113] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:03.252555Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435975461473103:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:03.252720Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435975545482133:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.252919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435975461473099:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.252935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:03.252945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435975461473099:2109] [node 6] ICP22 created new session: [2:7574435975461473116:2109]
2025-11-19T14:00:03.252952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435975461473116:2109] [node 6] ICS09 handshake done sender: [2:7574435975461473101:2111] self: [2:7574435975461473100:0] peer: [6:7574435974470708056:0] socket: 148 qp: -1
2025-11-19T14:00:03.252959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435975461473116:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:03.252970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:03.253255Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435975461473096:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.253259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435975461473096:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.253899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435975461473097:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.253928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:03.253936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435975461473097:2107] [node 4] ICP22 created new session: [2:7574435975461473118:2107]
2025-11-19T14:00:03.253942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435975461473118:2107] [node 4] ICS09 handshake done sender: [2:7574435975461473103:2112] self: [2:7574435975461473102:0] peer: [4:7574435975911921627:0] socket: 149 qp: -1
2025-11-19T14:00:03.253948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435975461473118:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:03.253955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:03.254186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435975461473115:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9790
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.254797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435975545482147:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33885
2025-11-19T14:00:03.254930Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435975461473096:2110] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:03.255143Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435975545482134:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:03.255264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435975545482063:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.255296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:03.255301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435975545482063:2049] [node 2] ICP22 created new session: [3:7574435975545482148:2049]
2025-11-19T14:00:03.255306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435975545482148:2049] [node 2] ICS09 handshake done sender: [3:7574435975545482134:2113] self: [3:7574435975545482143:0] peer: [2:7574435975461473093:0] socket: 152 qp: -1
2025-11-19T14:00:03.255313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435975545482148:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:03.255319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.256259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974470708058:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13139
2025-11-19T14:00:03.257047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435975015815130:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41833
2025-11-19T14:00:03.257351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435975015815102:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(21859470ns) {HandleHandshakeStatus:329(21858435ns) {DropIncomingHandshake:441(118ns) DropOutgoingHandshake:456(353ns) SwitchToState:178(278ns) ProcessPendingSessionEvents:495(21754888ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21734798ns max# 21667831ns}}}
2025-11-19T14:00:03.257795Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435974470708055:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.257897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435974470707977:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.257914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:03.257919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435974470707977:2049] [node 2] ICP22 created new session: [6:7574435974470708060:2049]
2025-11-19T14:00:03.257925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435974470708060:2049] [node 2] ICS09 handshake done sender: [6:7574435974470708055:2116] self: [6:7574435974470708056:0] peer: [2:7574435975461473100:0] socket: 150 qp: -1
2025-11-19T14:00:03.257931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435974470708060:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:03.257939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:03.258311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974470708059:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13139
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.259959Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435975545482145:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.260084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435975545482062:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.260106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435975545482150:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33885
2025-11-19T14:00:03.260454Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435975545482144:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.261454Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435975015815121:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.263298Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435975461473115:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.263584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435971166505737:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.263594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:03.263601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435971166505737:2049] [node 3] ICP22 created new session: [2:7574435975461473126:2049]
2025-11-19T14:00:03.263607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435975461473126:2049] [node 3] ICS09 handshake done sender: [2:7574435975461473096:2110] self: [2:7574435975461473093:0] peer: [3:7574435975545482143:0] socket: 147 qp: -1
2025-11-19T14:00:03.263612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435975461473126:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:03.263620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:03.266794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435975911921546:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.266818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:03.266825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435975911921546:2048] [node 1] ICP22 created new session: [4:7574435975911921633:2048]
2025-11-19T14:00:03.266831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435975911921633:2048] [node 1] ICS09 handshake done sender: [4:7574435975911921617:2113] self: [4:7574435975911921626:0] peer: [1:7574435975015815107:0] socket: 140 qp: -1
2025-11-19T14:00:03.266840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435975911921633:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:03.266848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:03.267231Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435975911921629:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.267249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435975911921548:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.267480Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435975911921623:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.269402Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435975545482131:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.269416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435975545482131:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.269500Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435975911921612:2110] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:03.269174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974470708063:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13139
2025-11-19T14:00:03.269199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974470708064:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13139
2025-11-19T14:00:03.272286Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435974470708054:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.272398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974470708065:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13139
2025-11-19T14:00:03.272410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435974470708021:2087] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.272421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:03.272427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435974470708021:2087] [node 4] ICP22 created new session: [6:7574435974470708066:2087]
2025-11-19T14:00:03.272432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435974470708066:2087] [node 4] ICS09 handshake done sender: [6:7574435974470708054:2115] self: [6:7574435974470708057:0] peer: [4:7574435975911921603:0] socket: 145 qp: -1
2025-11-19T14:00:03.272438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435974470708066:2087] [node 4] ICS10 traffic start
2025-11-19T14:00:03.272445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:03.272246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435975911921602:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.272264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:03.272269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435975911921602:2091] [node 6] ICP22 created new session: [4:7574435975911921637:2091]
2025-11-19T14:00:03.272275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435975911921637:2091] [node 6] ICS09 handshake done sender: [4:7574435975911921612:2110] self: [4:7574435975911921603:0] peer: [6:7574435974470708057:0] socket: 141 qp: -1
2025-11-19T14:00:03.272281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435975911921637:2091] [node 6] ICS10 traffic start
2025-11-19T14:00:03.272288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.271337Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435975545482131:2111] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:03.271451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435975545482125:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.271471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:03.271478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435975545482125:2107] [node 4] ICP22 created new session: [3:7574435975545482155:2107]
2025-11-19T14:00:03.271484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435975545482155:2107] [node 4] ICS09 handshake done sender: [3:7574435975545482131:2111] self: [3:7574435975545482127:0] peer: [4:7574435975911921635:0] socket: 154 qp: -1
2025-11-19T14:00:03.271490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435975545482155:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:03.271500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:03.272907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435975461473123:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9790
2025-11-19T14:00:03.273361Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435975015815116:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.273378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435975015815116:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.279907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435975911921547:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.279944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:03.279957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435975911921547:2049] [node 2] ICP22 created new session: [4:7574435975911921639:2049]
2025-11-19T14:00:03.279970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435975911921639:2049] [node 2] ICS09 handshake done sender: [4:7574435975911921623:2115] self: [4:7574435975911921627:0] peer: [2:7574435975461473102:0] socket: 156 qp: -1
2025-11-19T14:00:03.279982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435975911921639:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:03.280007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:03.281610Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435975545482150:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.282865Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435974470708059:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.282894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435974470707976:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.282965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435975141598751:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47713
2025-11-19T14:00:03.282993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435975141598752:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47713
2025-11-19T14:00:03.283041Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435975015815116:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.284629Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435974470708064:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.284662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435974470707978:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.284862Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435975015815119:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.284875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435975015815119:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.285378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435970720847746:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.285394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:03.285400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435970720847746:2049] [node 3] ICP22 created new session: [1:7574435975015815134:2049]
2025-11-19T14:00:03.285406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435975015815134:2049] [node 3] ICS09 handshake done sender: [1:7574435975015815116:2113] self: [1:7574435975015815104:0] peer: [3:7574435975545482151:0] socket: 165 qp: -1
2025-11-19T14:00:03.285413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435975015815134:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:03.285422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.286137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974470708070:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13139
2025-11-19T14:00:03.286235Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435975015815119:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:03.286396Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435974470708059:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:03.286464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435974470707976:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.286477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:03.286482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435974470707976:2048] [node 1] ICP22 created new session: [6:7574435974470708071:2048]
2025-11-19T14:00:03.286487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435974470708071:2048] [node 1] ICS09 handshake done sender: [6:7574435974470708059:2118] self: [6:7574435974470708068:0] peer: [1:7574435975015815118:0] socket: 182 qp: -1
2025-11-19T14:00:03.286493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435974470708071:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:03.286500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:03.284519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435975545482159:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33885
2025-11-19T14:00:03.286718Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435975545482138:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.286726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435975545482138:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.288219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435975141598753:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47713
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.288076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435975911921548:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23940) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435975911921628:2117] held: no
2025-11-19T14:00:03.288409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435975911921642:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59208
2025-11-19T14:00:03.288871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435975461473124:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9790
2025-11-19T14:00:03.288925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435975461473125:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9790
2025-11-19T14:00:03.289487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435971166505736:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.289504Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435975461473099:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:03.289796Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435971166505737:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:03.293655Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435975911921628:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:03.295410Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435975141598751:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.295453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435975141598666:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.295591Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435975461473105:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.295601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435975461473105:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.296153Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435974470708065:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.296192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435974470708045:2093] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.298308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435975461473097:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:03.298409Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435975141598753:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.298444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435975141598734:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.298567Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435975911921625:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.298577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435975911921625:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.299525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435975141598756:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47713
2025-11-19T14:00:03.296548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435975015815117:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.296578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:03.296587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435975015815117:2112] [node 6] ICP22 created new session: [1:7574435975015815137:2112]
2025-11-19T14:00:03.296594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435975015815137:2112] [node 6] ICS09 handshake done sender: [1:7574435975015815119:2114] self: [1:7574435975015815118:0] peer: [6:7574435974470708068:0] socket: 181 qp: -1
2025-11-19T14:00:03.296603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435975015815137:2112] [node 6] ICS10 traffic start
2025-11-19T14:00:03.296614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:03.297058Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435975015815130:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.297084Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435975015815105:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.297089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435975015815105:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.300024Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435975015815105:2108] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:03.300106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435970720847745:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.300124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:03.300134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435970720847745:2048] [node 2] ICP22 created new session: [1:7574435975015815141:2048]
2025-11-19T14:00:03.300139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435975015815141:2048] [node 2] ICS09 handshake done sender: [1:7574435975015815105:2108] self: [1:7574435975015815103:0] peer: [2:7574435975461473128:0] socket: 134 qp: -1
2025-11-19T14:00:03.300145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435975015815141:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:03.300151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:03.300288Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435975911921625:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:03.300327Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435975141598744:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.300339Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435975141598744:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.304279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435975461473130:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9790
2025-11-19T14:00:03.304589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435975911921548:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.304610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:03.304618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435975911921548:2050] [node 3] ICP22 created new session: [4:7574435975911921644:2050]
2025-11-19T14:00:03.304626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435975911921644:2050] [node 3] ICS09 handshake done sender: [4:7574435975911921628:2117] self: [4:7574435975911921635:0] peer: [3:7574435975545482127:0] socket: 162 qp: -1
2025-11-19T14:00:03.304633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435975911921644:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:03.304643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:03.304868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435975911921609:2088] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.304879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:03.304883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435975911921609:2088] [node 5] ICP22 created new session: [4:7574435975911921646:2088]
2025-11-19T14:00:03.304888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435975911921646:2088] [node 5] ICS09 handshake done sender: [4:7574435975911921625:2116] self: [4:7574435975911921624:0] peer: [5:7574435975141598755:0] socket: 163 qp: -1
2025-11-19T14:00:03.304894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435975911921646:2088] [node 5] ICS10 traffic start
2025-11-19T14:00:03.304899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:03.306104Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435975461473105:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:03.306216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435975461473098:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.306249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:03.306255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435975461473098:2108] [node 5] ICP22 created new session: [2:7574435975461473131:2108]
2025-11-19T14:00:03.306261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435975461473131:2108] [node 5] ICS09 handshake done sender: [2:7574435975461473105:2113] self: [2:7574435975461473104:0] peer: [5:7574435975141598754:0] socket: 177 qp: -1
2025-11-19T14:00:03.306267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435975461473131:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:03.306273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:03.306460Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435975461473115:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:03.306772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435971166505736:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.306785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:03.306789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435971166505736:2048] [node 1] ICP22 created new session: [2:7574435975461473133:2048]
2025-11-19T14:00:03.306794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435975461473133:2048] [node 1] ICS09 handshake done sender: [2:7574435975461473115:2116] self: [2:7574435975461473128:0] peer: [1:7574435975015815103:0] socket: 160 qp: -1
2025-11-19T14:00:03.306798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435975461473133:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:03.306803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:03.307961Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435975545482062:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27043) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435975545482145:2118] held: no
2025-11-19T14:00:03.308237Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435975545482145:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.316147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435975545482062:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.316173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:03.316182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435975545482062:2048] [node 1] ICP22 created new session: [3:7574435975545482161:2048]
2025-11-19T14:00:03.316190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435975545482161:2048] [node 1] ICS09 handshake done sender: [3:7574435975545482145:2118] self: [3:7574435975545482151:0] peer: [1:7574435975015815104:0] socket: 175 qp: -1
2025-11-19T14:00:03.316197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435975545482161:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:03.316203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:03.317864Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435975545482138:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:03.318571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435975545482132:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.318586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:03.318592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435975545482132:2109] [node 6] ICP22 created new session: [3:7574435975545482163:2109]
2025-11-19T14:00:03.318598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435975545482163:2109] [node 6] ICS09 handshake done sender: [3:7574435975545482138:2115] self: [3:7574435975545482137:0] peer: [6:7574435974470708069:0] socket: 188 qp: -1
2025-11-19T14:00:03.318603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435975545482163:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:03.318610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:03.316708Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435975141598752:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.318355Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435975141598753:2120] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:03.317989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974470708074:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13139
2025-11-19T14:00:03.319026Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435974470708064:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:03.319082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435974470707978:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.319090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:03.319095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435974470707978:2050] [node 3] ICP22 created new session: [6:7574435974470708075:2050]
2025-11-19T14:00:03.319100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435974470708075:2050] [node 3] ICS09 handshake done sender: [6:7574435974470708064:2120] self: [6:7574435974470708069:0] peer: [3:7574435975545482137:0] socket: 190 qp: -1
2025-11-19T14:00:03.319106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435974470708075:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:03.319113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.322650Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435975545482142:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.322662Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435975545482142:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.323494Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435975545482142:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:03.323586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435975545482126:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.323604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:03.323610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435975545482126:2108] [node 5] ICP22 created new session: [3:7574435975545482166:2108]
2025-11-19T14:00:03.323616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435975545482166:2108] [node 5] ICS09 handshake done sender: [3:7574435975545482142:2116] self: [3:7574435975545482130:0] peer: [5:7574435975141598761:0] socket: 151 qp: -1
2025-11-19T14:00:03.323620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435975545482166:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:03.323627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:03.321013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435975141598666:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15910) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435975141598751:2118] held: no
2025-11-19T14:00:03.321135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435975141598734:2104] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18663) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435975141598753:2120] held: no
2025-11-19T14:00:03.321207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435975141598734:2104] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.321216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:03.321221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435975141598734:2104] [node 4] ICP22 created new session: [5:7574435975141598759:2104]
2025-11-19T14:00:03.321226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435975141598759:2104] [node 4] ICS09 handshake done sender: [5:7574435975141598753:2120] self: [5:7574435975141598755:0] peer: [4:7574435975911921624:0] socket: 197 qp: -1
2025-11-19T14:00:03.321232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435975141598759:2104] [node 4] ICS10 traffic start
2025-11-19T14:00:03.321238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:03.321725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435975141598757:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47713
2025-11-19T14:00:03.322423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435975141598667:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.323896Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435975141598744:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.324139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974470708077:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13139
2025-11-19T14:00:03.324339Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435974470708065:2121] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:03.324416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435974470708045:2093] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.324430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:03.324437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435974470708045:2093] [node 5] ICP22 created new session: [6:7574435974470708078:2093]
2025-11-19T14:00:03.324442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435974470708078:2093] [node 5] ICS09 handshake done sender: [6:7574435974470708065:2121] self: [6:7574435974470708073:0] peer: [5:7574435975141598735:0] socket: 192 qp: -1
2025-11-19T14:00:03.324448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435974470708078:2093] [node 5] ICS10 traffic start
2025-11-19T14:00:03.324455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:03.324093Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435975141598751:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.328867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435975141598666:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.328888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:03.328897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435975141598666:2049] [node 2] ICP22 created new session: [5:7574435975141598763:2049]
2025-11-19T14:00:03.328902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435975141598763:2049] [node 2] ICS09 handshake done sender: [5:7574435975141598751:2118] self: [5:7574435975141598754:0] peer: [2:7574435975461473104:0] socket: 184 qp: -1
2025-11-19T14:00:03.328908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435975141598763:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:03.328916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:03.329133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435975141598762:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47713
2025-11-19T14:00:03.329581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435975141598667:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23940) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435975141598752:2119] held: no
2025-11-19T14:00:03.329671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435975141598720:2097] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.329682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:03.329686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435975141598720:2097] [node 6] ICP22 created new session: [5:7574435975141598765:2097]
2025-11-19T14:00:03.329691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435975141598765:2097] [node 6] ICS09 handshake done sender: [5:7574435975141598744:2116] self: [5:7574435975141598735:0] peer: [6:7574435974470708073:0] socket: 191 qp: -1
2025-11-19T14:00:03.329695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435975141598765:2097] [node 6] ICS10 traffic start
2025-11-19T14:00:03.329702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:03.330343Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435975141598752:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:03.331013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435975141598667:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.331025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:03.331030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435975141598667:2050] [node 3] ICP22 created new session: [5:7574435975141598767:2050]
2025-11-19T14:00:03.331034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435975141598767:2050] [node 3] ICS09 handshake done sender: [5:7574435975141598752:2119] self: [5:7574435975141598761:0] peer: [3:7574435975545482130:0] socket: 172 qp: -1
2025-11-19T14:00:03.331039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435975141598767:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:03.331045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:03.349304Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:03.349408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435975461473133:2048] [node 1] ICS01 socket: 160 reason# EndOfStream
2025-11-19T14:00:03.349418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435971166505736:2048] [node 1] ICP30 unregister session Session# [2:7574435975461473133:2048] VirtualId# [2:7574435975461473128:0]
2025-11-19T14:00:03.349424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435975461473133:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:03.349456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:03.349476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:03.349927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:03.349990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435975545482161:2048] [node 1] ICS01 socket: 175 reason# EndOfStream
2025-11-19T14:00:03.349996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435975545482062:2048] [node 1] ICP30 unregister session Session# [3:7574435975545482161:2048] VirtualId# [3:7574435975545482151:0]
2025-11-19T14:00:03.350001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435975545482161:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:03.350026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:03.350043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:03.363838Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:03.363945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435975911921633:2048] [node 1] ICS01 socket: 140 reason# EndOfStream
2025-11-19T14:00:03.363955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435975911921546:2048] [node 1] ICP30 unregister session Session# [4:7574435975911921633:2048] VirtualId# [4:7574435975911921626:0]
2025-11-19T14:00:03.363963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435975911921633:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:03.364023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:03.364050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:03.364262Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:03.364355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435974470708071:2048] [node 1] ICS01 socket: 182 reason# EndOfStream
2025-11-19T14:00:03.364362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435974470707976:2048] [node 1] ICP30 unregister session Session# [6:7574435974470708071:2048] VirtualId# [6:7574435974470708068:0]
2025-11-19T14:00:03.364369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435974470708071:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:03.364423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:03.364448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:03.364544Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:03.364984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435975141598748:2048] [node 1] ICS01 socket: 146 reason# EndOfStream
2025-11-19T14:00:03.364994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435975141598665:2048] [node 1] ICP30 unregister session Session# [5:7574435975141598748:2048] VirtualId# [5:7574435975141598745:0]
2025-11-19T14:00:03.364999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435975141598748:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:03.365033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:03.365054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:03.372876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:03.372973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435975545482148:2049] [node 2] ICS01 socket: 152 reason# EndOfStream
2025-11-19T14:00:03.372982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435975545482063:2049] [node 2] ICP30 unregister session Session# [3:7574435975545482148:2049] VirtualId# [3:7574435975545482143:0]
2025-11-19T14:00:03.372989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435975545482148:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:03.373030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:03.373052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:03.385623Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:03.385729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435974470708060:2049] [node 2] ICS01 socket: 150 reason# EndOfStream
2025-11-19T14:00:03.385739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435974470707977:2049] [node 2] ICP30 unregister session Session# [6:7574435974470708060:2049] VirtualId# [6:7574435974470708056:0]
2025-11-19T14:00:03.385747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435974470708060:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:03.385803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:03.385837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:03.387898Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:03.387990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435975911921639:2049] [node 2] ICS01 socket: 156 reason# EndOfStream
2025-11-19T14:00:03.387998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435975911921547:2049] [node 2] ICP30 unregister session Session# [4:7574435975911921639:2049] VirtualId# [4:7574435975911921627:0]
2025-11-19T14:00:03.388005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435975911921639:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:03.388061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:03.388085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:03.392014Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:03.392108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435975141598763:2049] [node 2] ICS01 socket: 184 reason# EndOfStream
2025-11-19T14:00:03.392117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435975141598666:2049] [node 2] ICP30 unregister session Session# [5:7574435975141598763:2049] VirtualId# [5:7574435975141598754:0]
2025-11-19T14:00:03.392126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435975141598763:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:03.392182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:03.392207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:03.411837Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T14:00:03.411941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435975911921644:2050] [node 3] ICS01 socket: 162 reason# EndOfStream
2025-11-19T14:00:03.411951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435975911921548:2050] [node 3] ICP30 unregister session Session# [4:7574435975911921644:2050] VirtualId# [4:7574435975911921635:0]
2025-11-19T14:00:03.411958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435975911921644:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:03.412003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:03.412027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:03.411856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:03.411942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435975141598767:2050] [node 3] ICS01 socket: 172 reason# EndOfStream
2025-11-19T14:00:03.411952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435975141598667:2050] [node 3] ICP30 unregister session Session# [5:7574435975141598767:2050] VirtualId# [5:7574435975141598761:0]
2025-11-19T14:00:03.411958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435975141598767:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:03.412002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:03.412027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:03.412061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:03.412123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435974470708075:2050] [node 3] ICS01 socket: 190 reason# EndOfStream
2025-11-19T14:00:03.412129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435974470707978:2050] [node 3] ICP30 unregister session Session# [6:7574435974470708075:2050] VirtualId# [6:7574435974470708069:0]
2025-11-19T14:00:03.412133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435974470708075:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:03.412159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:03.412177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:03.431858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:03.431973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435974470708066:2087] [node 4] ICS01 socket: 145 reason# EndOfStream
2025-11-19T14:00:03.431984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435974470708021:2087] [node 4] ICP30 unregister session Session# [6:7574435974470708066:2087] VirtualId# [6:7574435974470708057:0]
2025-11-19T14:00:03.431992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435974470708066:2087] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:03.432063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:03.432092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:03.592062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435973559293433:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:03.592092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435973559293434:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:03.660358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435976574421467:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:03.660382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435976574421468:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:03.692143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435976584938193:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:03.692163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435976584938194:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:03.739099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435974033190295:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:03.739112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435974033190296:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:03.739117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435974033190297:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:03.811918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435976586332766:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:03.811931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435976586332767:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:03.811935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435976586332768:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:03.874907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435974469318258:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:03.874920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435974469318259:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:03.874925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435974469318260:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:03.886982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435974033190331:2082] [node 6] ICP01 ready to work
2025-11-19T14:00:03.886916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435973559293462:2075] [node 4] ICP01 ready to work
2025-11-19T14:00:03.887812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435974469318280:2062] [node 5] ICP01 ready to work
2025-11-19T14:00:03.887337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435976584938218:2071] [node 5] ICP01 ready to work
2025-11-19T14:00:03.888152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435976584938226:2074] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.887146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435976574421505:2083] [node 5] ICP01 ready to work
2025-11-19T14:00:03.887238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435976574421506:2084] [node 6] ICP01 ready to work
2025-11-19T14:00:03.889608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435973559293472:2080] [node 6] ICP01 ready to work
2025-11-19T14:00:03.889692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435973559293473:2082] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.897509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435973559293489:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45611
2025-11-19T14:00:03.897543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435974033190351:2094] [node 5] ICP01 ready to work
2025-11-19T14:00:03.888703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435976584938229:2077] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.898549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435976574421521:2089] [node 4] ICP01 ready to work
2025-11-19T14:00:03.899315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435976574421519:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50979
2025-11-19T14:00:03.899330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435976574421520:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50979
2025-11-19T14:00:03.902728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435974033190332:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45658
2025-11-19T14:00:03.905277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435974033190332:2083] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.905310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435974033190295:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.904928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435976584938226:2074] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16691079ns) {Configure:104(16689685ns) {ProcessConfigured:135(16403556ns) {StartInitialHandshake:157(16402863ns) {DropHandshakes:469(548ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(91ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:476(1468ns) {SwitchToState:178(1097ns)}}}}}
2025-11-19T14:00:03.905454Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435973559293464:2076] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.905465Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435973559293464:2076] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.907597Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435976574421520:2096] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.905802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435976584938229:2077] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(17053810ns) {Configure:104(17053302ns) {ProcessConfigured:135(16828631ns) {StartInitialHandshake:157(16828326ns) {DropHandshakes:469(366ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:476(525ns) {SwitchToState:178(385ns)}}}}}
2025-11-19T14:00:03.907060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435976584938258:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14377
2025-11-19T14:00:03.907076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435976584938260:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14377
2025-11-19T14:00:03.909029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435976586332793:2070] [node 4] ICP01 ready to work
2025-11-19T14:00:03.913610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435974469318290:2072] [node 4] ICP01 ready to work
2025-11-19T14:00:03.915135Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435973559293434:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(23509922ns) {Configure:104(23509283ns) {ProcessConfigured:135(14882493ns) {StartInitialHandshake:157(14881462ns) {DropHandshakes:469(553ns) {DropIncomingHandshake:441(137ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(1761ns) {SwitchToState:178(1040ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.915365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435976574421506:2084] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27144510ns) {Configure:104(15062275ns) {ProcessConfigured:135(14728259ns) {StartInitialHandshake:157(14728083ns) {DropHandshakes:469(515ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(154ns) PrepareNewSessionHandshake:476(1109ns) {SwitchToState:178(298ns)}}}}}
2025-11-19T14:00:03.915687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435973559293513:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45611
2025-11-19T14:00:03.916619Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435976584938260:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.916693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435976584938193:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(28068801ns) {Configure:104(14301391ns) {ProcessConfigured:135(13949864ns) {StartInitialHandshake:157(13947927ns) {DropHandshakes:469(402ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(42ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:476(939ns) {SwitchToState:178(708ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.918281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435974033190375:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45658
2025-11-19T14:00:03.918305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435974033190376:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45658
2025-11-19T14:00:03.919721Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435974033190376:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.919786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435974033190297:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.920154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435976586332806:2075] [node 6] ICP01 ready to work
2025-11-19T14:00:03.924915Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435973559293464:2076] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:03.927964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435974469318280:2062] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(40065904ns) {Configure:104(40065425ns) {ProcessConfigured:135(39824094ns) {StartInitialHandshake:157(39823046ns) {DropHandshakes:469(252ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(32ns)} GenerateSessionVirtualId:988(108ns) PrepareNewSessionHandshake:476(2164ns) {SwitchToState:178(1523ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.931859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435976586332768:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(22723752ns) {Configure:104(22721954ns) {ProcessConfigured:135(22199253ns) {StartInitialHandshake:157(22198465ns) {DropHandshakes:469(754ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(128ns)} GenerateSessionVirtualId:988(297ns) PrepareNewSessionHandshake:476(1424ns) {SwitchToState:178(339ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.933649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435976584938275:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14377
2025-11-19T14:00:03.933359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974469318334:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12655
2025-11-19T14:00:03.933419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974469318335:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12655
2025-11-19T14:00:03.933454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974469318336:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12655
2025-11-19T14:00:03.933485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974469318337:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12655
2025-11-19T14:00:03.939044Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435974469318334:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.939067Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435974469318335:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.939091Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435974469318336:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.939125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435974469318258:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.939137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435974469318260:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.940167Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435976584938262:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.940193Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435976584938262:2107] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.940241Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435976584938258:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.944347Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435976586332766:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24024917ns) {Configure:104(24024099ns) {ProcessConfigured:135(23688686ns) {StartInitialHandshake:157(23687420ns) {DropHandshakes:469(983ns) {DropIncomingHandshake:441(88ns) DropOutgoingHandshake:456(161ns)} GenerateSessionVirtualId:988(275ns) PrepareNewSessionHandshake:476(1001ns) {SwitchToState:178(649ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.945082Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435976584938275:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.945130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435974033190378:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45658
2025-11-19T14:00:03.944913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435976574421549:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50979
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.945511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435976586332844:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26428
2025-11-19T14:00:03.945558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435976586332845:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26428
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.947818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435974033190379:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45658
2025-11-19T14:00:03.947889Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435974033190332:2083] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:03.948286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435974033190295:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.948316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:03.948326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435974033190295:2048] [node 1] ICP22 created new session: [4:7574435974033190380:2048]
2025-11-19T14:00:03.948333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435974033190380:2048] [node 1] ICS09 handshake done sender: [4:7574435974033190332:2083] self: [4:7574435974033190374:0] peer: [1:7574435973559293463:0] socket: 139 qp: -1
2025-11-19T14:00:03.948341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435974033190380:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:03.948352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:03.948611Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435976586332845:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.948641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435976586332767:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.948772Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435974033190379:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.949246Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435973559293489:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.949535Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435976584938257:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.949551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435976584938257:2103] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.949650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435976586332843:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26428
2025-11-19T14:00:03.949756Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435976586332844:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.949787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435976586332766:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.950044Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435974033190375:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.950176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435974033190296:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.950466Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435976586332843:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.950490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435976586332768:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.948093Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435976574421519:2095] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.948931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435976574421518:2094] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.948935Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435976574421518:2094] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.949844Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435976574421549:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.949892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435976574421467:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.949372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435973559293462:2075] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.949392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:03.949402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435973559293462:2075] [node 4] ICP22 created new session: [1:7574435973559293517:2075]
2025-11-19T14:00:03.949409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435973559293517:2075] [node 4] ICS09 handshake done sender: [1:7574435973559293464:2076] self: [1:7574435973559293463:0] peer: [4:7574435974033190374:0] socket: 138 qp: -1
2025-11-19T14:00:03.949415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435973559293517:2075] [node 4] ICS10 traffic start
2025-11-19T14:00:03.949429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:03.949488Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435973559293480:2087] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.949498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435973559293480:2087] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.956171Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435974469318337:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.956272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435974469318280:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.957109Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435976586332833:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.957134Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435976586332833:2108] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.960630Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435973559293478:2086] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.960656Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435973559293478:2086] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.961000Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435976584938220:2072] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.961020Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435976584938220:2072] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.961320Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435974469318290:2072] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(47509858ns) {Configure:104(47508491ns) {ProcessConfigured:135(47045261ns) {StartInitialHandshake:157(47044338ns) {DropHandshakes:469(813ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(146ns)} GenerateSessionVirtualId:988(361ns) PrepareNewSessionHandshake:476(1379ns) {SwitchToState:178(858ns)}}}}}
2025-11-19T14:00:03.961362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435974469318290:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.960937Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435976574421525:2098] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.960955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435976574421525:2098] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.962219Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435976574421518:2094] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:03.962393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435976574421505:2083] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.962408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:03.962415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435976574421505:2083] [node 5] ICP22 created new session: [2:7574435976574421553:2083]
2025-11-19T14:00:03.962421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435976574421553:2083] [node 5] ICS09 handshake done sender: [2:7574435976574421518:2094] self: [2:7574435976574421510:0] peer: [5:7574435976586332847:0] socket: 153 qp: -1
2025-11-19T14:00:03.962427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435976574421553:2083] [node 5] ICS10 traffic start
2025-11-19T14:00:03.962436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:03.962561Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435976584938262:2107] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:03.962000Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435973559293513:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.967547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435976584938229:2077] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.967584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:03.967594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435976584938229:2077] [node 4] ICP22 created new session: [3:7574435976584938279:2077]
2025-11-19T14:00:03.967605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435976584938279:2077] [node 4] ICS09 handshake done sender: [3:7574435976584938262:2107] self: [3:7574435976584938230:0] peer: [4:7574435974033190377:0] socket: 161 qp: -1
2025-11-19T14:00:03.967617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435976584938279:2077] [node 4] ICS10 traffic start
2025-11-19T14:00:03.967649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.968072Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435974033190337:2087] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.968089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435974033190337:2087] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.968751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435974033190385:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45658
2025-11-19T14:00:03.969019Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435974033190295:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20628127ns) {HandleHandshakeStatus:329(20626826ns) {DropIncomingHandshake:441(250ns) DropOutgoingHandshake:456(6716ns) SwitchToState:178(46ns) ProcessPendingSessionEvents:495(18932472ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18911050ns max# 18834144ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.969347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974469318344:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12655
2025-11-19T14:00:03.969508Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435974033190337:2087] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:03.969701Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435974469318334:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:03.969820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435974469318290:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.969840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:03.969846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435974469318290:2072] [node 4] ICP22 created new session: [6:7574435974469318345:2072]
2025-11-19T14:00:03.969852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435974469318345:2072] [node 4] ICS09 handshake done sender: [6:7574435974469318334:2113] self: [6:7574435974469318343:0] peer: [4:7574435974033190336:0] socket: 162 qp: -1
2025-11-19T14:00:03.969859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435974469318345:2072] [node 4] ICS10 traffic start
2025-11-19T14:00:03.969880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.971649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435974033190331:2082] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.971669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:03.971679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435974033190331:2082] [node 6] ICP22 created new session: [4:7574435974033190387:2082]
2025-11-19T14:00:03.971685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435974033190387:2082] [node 6] ICS09 handshake done sender: [4:7574435974033190337:2087] self: [4:7574435974033190336:0] peer: [6:7574435974469318343:0] socket: 144 qp: -1
2025-11-19T14:00:03.971693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435974033190387:2082] [node 6] ICS10 traffic start
2025-11-19T14:00:03.971702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:03.972492Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435974033190376:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:03.972648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435974033190297:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.972663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:03.972668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435974033190297:2050] [node 3] ICP22 created new session: [4:7574435974033190389:2050]
2025-11-19T14:00:03.972673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435974033190389:2050] [node 3] ICS09 handshake done sender: [4:7574435974033190376:2117] self: [4:7574435974033190377:0] peer: [3:7574435976584938230:0] socket: 163 qp: -1
2025-11-19T14:00:03.972678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435974033190389:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:03.972684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.972108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435976586332850:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26428
2025-11-19T14:00:03.972134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435976586332851:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26428
2025-11-19T14:00:03.973269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435976586332852:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26428
2025-11-19T14:00:03.973561Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435976586332845:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:03.975389Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435973559293480:2087] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:03.975551Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435973559293478:2086] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:03.975645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435973559293472:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.975662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:03.975668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435973559293472:2080] [node 6] ICP22 created new session: [1:7574435973559293522:2080]
2025-11-19T14:00:03.975674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435973559293522:2080] [node 6] ICS09 handshake done sender: [1:7574435973559293480:2087] self: [1:7574435973559293479:0] peer: [6:7574435974469318338:0] socket: 148 qp: -1
2025-11-19T14:00:03.975679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435973559293522:2080] [node 6] ICS10 traffic start
2025-11-19T14:00:03.975686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:03.975939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435973559293473:2082] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.975946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:03.975953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435973559293473:2082] [node 5] ICP22 created new session: [1:7574435973559293524:2082]
2025-11-19T14:00:03.975958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435973559293524:2082] [node 5] ICS09 handshake done sender: [1:7574435973559293478:2086] self: [1:7574435973559293477:0] peer: [5:7574435976586332848:0] socket: 147 qp: -1
2025-11-19T14:00:03.975963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435973559293524:2082] [node 5] ICS10 traffic start
2025-11-19T14:00:03.975969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:03.973754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435976586332767:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.973779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:03.973786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435976586332767:2049] [node 2] ICP22 created new session: [5:7574435976586332853:2049]
2025-11-19T14:00:03.973793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435976586332853:2049] [node 2] ICS09 handshake done sender: [5:7574435976586332845:2116] self: [5:7574435976586332847:0] peer: [2:7574435976574421510:0] socket: 175 qp: -1
2025-11-19T14:00:03.973800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435976586332853:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:03.973808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.979240Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435974469318258:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11186) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435974469318335:2114] held: no
2025-11-19T14:00:03.979431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974469318348:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12655
2025-11-19T14:00:03.977018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435973559293462:2075] [node 4] ICP35 event processing took too much time PendingConnection:278(51854165ns) {HandleHandshakeStatus:329(51853072ns) {DropIncomingHandshake:441(564ns) DropOutgoingHandshake:456(284ns) SwitchToState:178(40ns) ProcessPendingSessionEvents:495(15990845ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15964828ns max# 15894909ns}}}
2025-11-19T14:00:03.980453Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435973559293485:2091] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.980464Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435973559293485:2091] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:03.981065Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435974469318335:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:03.981215Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435976574421525:2098] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.984156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435976574421521:2089] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.984182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:03.984192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435976574421521:2089] [node 4] ICP22 created new session: [2:7574435976574421557:2089]
2025-11-19T14:00:03.984198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435976574421557:2089] [node 4] ICS09 handshake done sender: [2:7574435976574421525:2098] self: [2:7574435976574421524:0] peer: [4:7574435974033190384:0] socket: 155 qp: -1
2025-11-19T14:00:03.984205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435976574421557:2089] [node 4] ICS10 traffic start
2025-11-19T14:00:03.984213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:03.984479Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435976586332850:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.984508Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435976586332852:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.984563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435976586332856:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26428
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.988917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435976584938282:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14377
2025-11-19T14:00:03.988972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435976584938283:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14377
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.989820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435973559293527:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45611
2025-11-19T14:00:03.989843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435973559293528:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45611
2025-11-19T14:00:03.991487Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435973559293485:2091] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:03.987713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435976586332793:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:03.987981Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435976586332833:2108] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:03.988075Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435976586332844:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:03.988731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435976586332766:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.988747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:03.988757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435976586332766:2048] [node 1] ICP22 created new session: [5:7574435976586332860:2048]
2025-11-19T14:00:03.988764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435976586332860:2048] [node 1] ICS09 handshake done sender: [5:7574435976586332844:2115] self: [5:7574435976586332848:0] peer: [1:7574435973559293477:0] socket: 174 qp: -1
2025-11-19T14:00:03.988769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435976586332860:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:03.988790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:03.990533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435976586332806:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.990550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:03.990557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435976586332806:2075] [node 6] ICP22 created new session: [5:7574435976586332863:2075]
2025-11-19T14:00:03.990562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435976586332863:2075] [node 6] ICS09 handshake done sender: [5:7574435976586332833:2108] self: [5:7574435976586332832:0] peer: [6:7574435974469318341:0] socket: 169 qp: -1
2025-11-19T14:00:03.990568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435976586332863:2075] [node 6] ICS10 traffic start
2025-11-19T14:00:03.990574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:03.991990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435973559293433:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.992016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:03.992024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435973559293433:2048] [node 2] ICP22 created new session: [1:7574435973559293530:2048]
2025-11-19T14:00:03.992041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435973559293530:2048] [node 2] ICS09 handshake done sender: [1:7574435973559293485:2091] self: [1:7574435973559293484:0] peer: [2:7574435976574421551:0] socket: 166 qp: -1
2025-11-19T14:00:03.992049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435973559293530:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:03.992058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:03.992686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435976574421505:2083] [node 5] ICP35 event processing took too much time PendingConnection:278(30194890ns) {HandleHandshakeStatus:329(30194158ns) {DropIncomingHandshake:441(142ns) DropOutgoingHandshake:456(579ns) SwitchToState:178(178ns) ProcessPendingSessionEvents:495(12042664ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12017441ns max# 11945825ns}}}
2025-11-19T14:00:03.994612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435976584938229:2077] [node 4] ICP35 event processing took too much time PendingConnection:278(26973355ns) {HandleHandshakeStatus:329(26972279ns) {DropIncomingHandshake:441(4187ns) DropOutgoingHandshake:456(284ns) SwitchToState:178(996ns) ProcessPendingSessionEvents:495(13942943ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13918285ns max# 13845632ns}}}
2025-11-19T14:00:03.994688Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435976584938257:2103] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:03.994702Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435976584938282:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.994762Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435976584938220:2072] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:03.995091Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435974033190353:2095] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:03.995104Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435974033190353:2095] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.996828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435974469318258:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:03.996849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:03.996857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435974469318258:2048] [node 1] ICP22 created new session: [6:7574435974469318349:2048]
2025-11-19T14:00:03.996864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435974469318349:2048] [node 1] ICS09 handshake done sender: [6:7574435974469318335:2114] self: [6:7574435974469318338:0] peer: [1:7574435973559293479:0] socket: 170 qp: -1
2025-11-19T14:00:03.996871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435974469318349:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:03.996879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:03.997282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974469318351:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12655
2025-11-19T14:00:03.997356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974469318352:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12655
2025-11-19T14:00:03.996260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435974033190296:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9159) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435974033190375:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.997889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435976586332862:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26428
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:03.999617Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435974033190353:2095] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:03.997975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435976584938218:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:03.998004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:03.998011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435976584938218:2071] [node 5] ICP22 created new session: [3:7574435976584938286:2071]
2025-11-19T14:00:03.998018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435976584938286:2071] [node 5] ICS09 handshake done sender: [3:7574435976584938220:2072] self: [3:7574435976584938219:0] peer: [5:7574435976586332849:0] socket: 140 qp: -1
2025-11-19T14:00:03.998025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435976584938286:2071] [node 5] ICS10 traffic start
2025-11-19T14:00:03.998033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:04.001389Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435976586332843:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:04.001501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435976586332768:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.001520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:04.001528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435976586332768:2050] [node 3] ICP22 created new session: [5:7574435976586332866:2050]
2025-11-19T14:00:04.001536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435976586332866:2050] [node 3] ICS09 handshake done sender: [5:7574435976586332843:2114] self: [5:7574435976586332849:0] peer: [3:7574435976584938219:0] socket: 143 qp: -1
2025-11-19T14:00:04.001542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435976586332866:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:04.001550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:04.003198Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435976574421467:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11186) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435976574421549:2117] held: no
2025-11-19T14:00:04.005887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435974033190351:2094] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.005912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:04.005924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435974033190351:2094] [node 5] ICP22 created new session: [4:7574435978328157689:2094]
2025-11-19T14:00:04.005930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435978328157689:2094] [node 5] ICS09 handshake done sender: [4:7574435974033190353:2095] self: [4:7574435974033190352:0] peer: [5:7574435976586332858:0] socket: 178 qp: -1
2025-11-19T14:00:04.005935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435978328157689:2094] [node 5] ICS10 traffic start
2025-11-19T14:00:04.005942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.006531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435974469318353:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12655
2025-11-19T14:00:04.006588Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435974469318336:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.006836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435980869388856:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50979
2025-11-19T14:00:04.006983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435976584938193:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.007255Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435976584938283:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.007479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435976584938194:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.008235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435974469318280:2062] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15463) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435974469318337:2116] held: no
2025-11-19T14:00:04.008335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435974469318260:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.008347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:04.008354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435974469318260:2050] [node 3] ICP22 created new session: [6:7574435978764285650:2050]
2025-11-19T14:00:04.008359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435978764285650:2050] [node 3] ICS09 handshake done sender: [6:7574435974469318336:2115] self: [6:7574435974469318339:0] peer: [3:7574435976584938228:0] socket: 171 qp: -1
2025-11-19T14:00:04.008366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435978764285650:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:04.008373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.008367Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435973559293512:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.008382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435973559293512:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.009734Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435973559293512:2115] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:04.009874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435973559293434:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.009892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:04.009900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435973559293434:2049] [node 3] ICP22 created new session: [1:7574435977854260829:2049]
2025-11-19T14:00:04.009906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435977854260829:2049] [node 3] ICS09 handshake done sender: [1:7574435973559293512:2115] self: [1:7574435973559293496:0] peer: [3:7574435980879905584:0] socket: 150 qp: -1
2025-11-19T14:00:04.009913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435977854260829:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:04.009922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:04.009700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435976584938226:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.009724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:04.009730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435976584938226:2074] [node 6] ICP22 created new session: [3:7574435980879905588:2074]
2025-11-19T14:00:04.009736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435980879905588:2074] [node 6] ICS09 handshake done sender: [3:7574435976584938257:2103] self: [3:7574435976584938228:0] peer: [6:7574435974469318339:0] socket: 160 qp: -1
2025-11-19T14:00:04.009741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435980879905588:2074] [node 6] ICS10 traffic start
2025-11-19T14:00:04.009749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:04.010989Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435974469318337:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.012347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435980879905587:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14377
2025-11-19T14:00:04.012652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435974469318280:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.012672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:04.012679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435974469318280:2062] [node 5] ICP22 created new session: [6:7574435978764285652:2062]
2025-11-19T14:00:04.012685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435978764285652:2062] [node 5] ICS09 handshake done sender: [6:7574435974469318337:2116] self: [6:7574435974469318341:0] peer: [5:7574435976586332832:0] socket: 172 qp: -1
2025-11-19T14:00:04.012691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435978764285652:2062] [node 5] ICS10 traffic start
2025-11-19T14:00:04.012697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:04.012844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435976586332793:2070] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23218) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435976586332850:2118] held: no
2025-11-19T14:00:04.013287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435976586332865:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26428
2025-11-19T14:00:04.013379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435974033190392:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45658
2025-11-19T14:00:04.013466Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435976586332850:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:04.014216Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435976574421508:2085] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.014229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435976574421508:2085] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.014664Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435976574421549:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:04.015009Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435974469318353:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.015039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435974469318259:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.015129Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435974033190375:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:04.015700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435974033190296:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.015714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:04.015721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435974033190296:2049] [node 2] ICP22 created new session: [4:7574435978328157691:2049]
2025-11-19T14:00:04.015727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435978328157691:2049] [node 2] ICS09 handshake done sender: [4:7574435974033190375:2116] self: [4:7574435974033190384:0] peer: [2:7574435976574421524:0] socket: 157 qp: -1
2025-11-19T14:00:04.015733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435978328157691:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:04.015754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.016634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435980869388857:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50979
2025-11-19T14:00:04.016845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435976586332793:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.016863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:04.016870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435976586332793:2070] [node 4] ICP22 created new session: [5:7574435980881300164:2070]
2025-11-19T14:00:04.016882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435980881300164:2070] [node 4] ICS09 handshake done sender: [5:7574435976586332850:2118] self: [5:7574435976586332858:0] peer: [4:7574435974033190352:0] socket: 179 qp: -1
2025-11-19T14:00:04.016888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435980881300164:2070] [node 4] ICS10 traffic start
2025-11-19T14:00:04.016895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:04.017427Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435976584938282:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:04.017682Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435976574421548:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.017691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435976574421548:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.018299Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435976574421508:2085] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:04.018310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435976574421467:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.018319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:04.018324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435976574421467:2048] [node 1] ICP22 created new session: [2:7574435980869388859:2048]
2025-11-19T14:00:04.018329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435980869388859:2048] [node 1] ICS09 handshake done sender: [2:7574435976574421549:2117] self: [2:7574435976574421551:0] peer: [1:7574435973559293484:0] socket: 176 qp: -1
2025-11-19T14:00:04.018334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435980869388859:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:04.018340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:04.018629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435976574421468:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.018647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:04.018655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435976574421468:2049] [node 3] ICP22 created new session: [2:7574435980869388862:2049]
2025-11-19T14:00:04.018661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435980869388862:2049] [node 3] ICS09 handshake done sender: [2:7574435976574421508:2085] self: [2:7574435976574421507:0] peer: [3:7574435980879905586:0] socket: 191 qp: -1
2025-11-19T14:00:04.018667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435980869388862:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:04.018676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:04.018962Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435976574421548:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:04.018967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435976584938193:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.018984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:04.018989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435976584938193:2048] [node 1] ICP22 created new session: [3:7574435980879905591:2048]
2025-11-19T14:00:04.018993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435980879905591:2048] [node 1] ICS09 handshake done sender: [3:7574435976584938282:2118] self: [3:7574435980879905584:0] peer: [1:7574435973559293496:0] socket: 195 qp: -1
2025-11-19T14:00:04.018998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435980879905591:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:04.019003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.020589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435980879905590:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14377
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.021130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435978764285655:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12655
2025-11-19T14:00:04.022128Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435974469318353:2123] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:04.022321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435974469318259:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.022340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:04.022347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435974469318259:2049] [node 2] ICP22 created new session: [6:7574435978764285656:2049]
2025-11-19T14:00:04.022353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435978764285656:2049] [node 2] ICS09 handshake done sender: [6:7574435974469318353:2123] self: [6:7574435978764285654:0] peer: [2:7574435976574421535:0] socket: 204 qp: -1
2025-11-19T14:00:04.022359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435978764285656:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:04.022366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:04.023479Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435976584938194:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9159) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435976584938283:2119] held: no
2025-11-19T14:00:04.023826Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435976584938283:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:04.019901Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435973559293528:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.020146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435973559293434:2049] [node 3] ICP18 handshake (actor [1:7574435973559293528:2118]) is too old
2025-11-19T14:00:04.023906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435976584938194:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.023920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:04.023924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435976584938194:2049] [node 2] ICP22 created new session: [3:7574435980879905593:2049]
2025-11-19T14:00:04.023930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435980879905593:2049] [node 2] ICS09 handshake done sender: [3:7574435976584938283:2119] self: [3:7574435980879905586:0] peer: [2:7574435976574421507:0] socket: 196 qp: -1
2025-11-19T14:00:04.023935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435980879905593:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:04.023942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:04.020688Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435980869388857:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.020716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435976574421468:2049] [node 3] ICP18 handshake (actor [2:7574435980869388857:2119]) is too old
2025-11-19T14:00:04.026419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435976574421506:2084] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.026446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:04.026454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435976574421506:2084] [node 6] ICP22 created new session: [2:7574435980869388864:2084]
2025-11-19T14:00:04.026460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435980869388864:2084] [node 6] ICS09 handshake done sender: [2:7574435976574421548:2116] self: [2:7574435976574421535:0] peer: [6:7574435978764285654:0] socket: 159 qp: -1
2025-11-19T14:00:04.026466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435980869388864:2084] [node 6] ICS10 traffic start
2025-11-19T14:00:04.026476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:04.041122Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:04.041211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435980869388859:2048] [node 1] ICS01 socket: 176 reason# EndOfStream
2025-11-19T14:00:04.041220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435976574421467:2048] [node 1] ICP30 unregister session Session# [2:7574435980869388859:2048] VirtualId# [2:7574435976574421551:0]
2025-11-19T14:00:04.041226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435980869388859:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.041260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:04.041278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:04.041647Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:04.041711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435980879905591:2048] [node 1] ICS01 socket: 195 reason# EndOfStream
2025-11-19T14:00:04.041716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435976584938193:2048] [node 1] ICP30 unregister session Session# [3:7574435980879905591:2048] VirtualId# [3:7574435980879905584:0]
2025-11-19T14:00:04.041720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435980879905591:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.041745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:04.041761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:04.054407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:04.054522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435974469318349:2048] [node 1] ICS01 socket: 170 reason# EndOfStream
2025-11-19T14:00:04.054533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435974469318258:2048] [node 1] ICP30 unregister session Session# [6:7574435974469318349:2048] VirtualId# [6:7574435974469318338:0]
2025-11-19T14:00:04.054540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435974469318349:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.054600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:04.054618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:04.056053Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:04.056140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435974033190380:2048] [node 1] ICS01 socket: 139 reason# EndOfStream
2025-11-19T14:00:04.056151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435974033190295:2048] [node 1] ICP30 unregister session Session# [4:7574435974033190380:2048] VirtualId# [4:7574435974033190374:0]
2025-11-19T14:00:04.056157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435974033190380:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.056213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:04.056243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:04.056212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:04.056325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435976586332860:2048] [node 1] ICS01 socket: 174 reason# EndOfStream
2025-11-19T14:00:04.056336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435976586332766:2048] [node 1] ICP30 unregister session Session# [5:7574435976586332860:2048] VirtualId# [5:7574435976586332848:0]
2025-11-19T14:00:04.056344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435976586332860:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.056397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:04.056428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:04.068823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:04.068918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435980879905593:2049] [node 2] ICS01 socket: 196 reason# EndOfStream
2025-11-19T14:00:04.068930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435976584938194:2049] [node 2] ICP30 unregister session Session# [3:7574435980879905593:2049] VirtualId# [3:7574435980879905586:0]
2025-11-19T14:00:04.068937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435980879905593:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.068988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:04.069007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:04.078947Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:04.079065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435978764285656:2049] [node 2] ICS01 socket: 204 reason# EndOfStream
2025-11-19T14:00:04.079074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435974469318259:2049] [node 2] ICP30 unregister session Session# [6:7574435978764285656:2049] VirtualId# [6:7574435978764285654:0]
2025-11-19T14:00:04.079083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435978764285656:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.079144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:04.079171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:04.078959Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:04.079090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435976586332853:2049] [node 2] ICS01 socket: 175 reason# EndOfStream
2025-11-19T14:00:04.079103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435976586332767:2049] [node 2] ICP30 unregister session Session# [5:7574435976586332853:2049] VirtualId# [5:7574435976586332847:0]
2025-11-19T14:00:04.079111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435976586332853:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.079172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:04.079204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:04.082613Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:04.082705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435978328157691:2049] [node 2] ICS01 socket: 157 reason# EndOfStream
2025-11-19T14:00:04.082714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435974033190296:2049] [node 2] ICP30 unregister session Session# [4:7574435978328157691:2049] VirtualId# [4:7574435974033190384:0]
2025-11-19T14:00:04.082721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435978328157691:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.082783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:04.082808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:04.089880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:04.089968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435978764285650:2050] [node 3] ICS01 socket: 171 reason# EndOfStream
2025-11-19T14:00:04.089977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435974469318260:2050] [node 3] ICP30 unregister session Session# [6:7574435978764285650:2050] VirtualId# [6:7574435974469318339:0]
2025-11-19T14:00:04.089983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435978764285650:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.090036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:04.090062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:04.095128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:04.095223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435976586332866:2050] [node 3] ICS01 socket: 143 reason# EndOfStream
2025-11-19T14:00:04.095232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435976586332768:2050] [node 3] ICP30 unregister session Session# [5:7574435976586332866:2050] VirtualId# [5:7574435976586332849:0]
2025-11-19T14:00:04.095240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435976586332866:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.095318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:04.095345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:04.117161Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:04.117289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435974469318345:2072] [node 4] ICS01 socket: 162 reason# EndOfStream
2025-11-19T14:00:04.117299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435974469318290:2072] [node 4] ICP30 unregister session Session# [6:7574435974469318345:2072] VirtualId# [6:7574435974469318343:0]
2025-11-19T14:00:04.117306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435974469318345:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.117371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:04.117399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:04.508194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435979142547674:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:04.508216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435979142547675:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:04.528226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435978209642983:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:04.528240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435978209642984:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:04.572438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435980617292432:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:04.572455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435980617292433:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:04.586594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435980386459993:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:04.586608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435980386459994:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:04.586613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435980386459995:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:04.608049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435981945788623:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:04.608061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435981945788624:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:04.608066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435981945788625:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:04.628127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435981786624339:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:04.628140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435981786624340:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:04.628143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435981786624341:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:04.680126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435980386460033:2085] [node 6] ICP01 ready to work
2025-11-19T14:00:04.680420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435980386460039:2088] [node 5] ICP01 ready to work
2025-11-19T14:00:04.680256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435980617292493:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:04.682147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435981786624356:2061] [node 4] ICP01 ready to work
2025-11-19T14:00:04.683384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435981786624365:2067] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.684111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435980386460047:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
2025-11-19T14:00:04.692976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435981945788641:2062] [node 4] ICP01 ready to work
2025-11-19T14:00:04.694560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435981945788656:2067] [node 6] ICP01 ready to work
2025-11-19T14:00:04.698628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435978209643050:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.704397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435981786624341:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(16461070ns) {Configure:104(16459934ns) {ProcessConfigured:135(16168959ns) {StartInitialHandshake:157(16168388ns) {DropHandshakes:469(520ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:476(1298ns) {SwitchToState:178(929ns)}}}}}
2025-11-19T14:00:04.705572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435979142547744:2110] [node 5] ICP01 ready to work
2025-11-19T14:00:04.705732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435979142547745:2111] [node 4] ICP01 ready to work
2025-11-19T14:00:04.704182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435978209643058:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63518
2025-11-19T14:00:04.704207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435978209643059:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63518
2025-11-19T14:00:04.704281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435978209643060:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63518
2025-11-19T14:00:04.704294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435978209643061:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63518
2025-11-19T14:00:04.705194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435980617292501:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:04.707895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435979142547743:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:04.708549Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435978209643059:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.709173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435980617292504:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27739
2025-11-19T14:00:04.709192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435980617292505:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27739
2025-11-19T14:00:04.709202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435980617292506:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27739
2025-11-19T14:00:04.710015Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435980617292505:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.716395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435981786624417:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51965
2025-11-19T14:00:04.716577Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435980386459995:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(20606788ns) {Configure:104(20602563ns) {ProcessConfigured:135(20191477ns) {StartInitialHandshake:157(20190897ns) {DropHandshakes:469(1852ns) {DropIncomingHandshake:441(391ns) DropOutgoingHandshake:456(283ns)} GenerateSessionVirtualId:988(555ns) PrepareNewSessionHandshake:476(1190ns) {SwitchToState:178(630ns)}}}}}
2025-11-19T14:00:04.716889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435980386460039:2088] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(36394108ns) {Configure:104(36393155ns) {ProcessConfigured:135(31238026ns) {StartInitialHandshake:157(31237398ns) {DropHandshakes:469(1087ns) {DropIncomingHandshake:441(172ns) DropOutgoingHandshake:456(480ns)} GenerateSessionVirtualId:988(312ns) PrepareNewSessionHandshake:476(969ns) {SwitchToState:178(513ns)}}}}}
2025-11-19T14:00:04.716063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435981945788701:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24351
2025-11-19T14:00:04.716089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435981945788702:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24351
2025-11-19T14:00:04.720363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435978209643048:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.722449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435979142547750:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1880
2025-11-19T14:00:04.722533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435979142547751:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1880
2025-11-19T14:00:04.722896Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435981786624417:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.722954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435981786624356:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-11-19T14:00:04.721323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435978209643047:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:04.723658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435980617292501:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18378116ns) {Configure:104(18376922ns) {ProcessConfigured:135(17991410ns) {StartInitialHandshake:157(17990742ns) {DropHandshakes:469(15000ns) {DropIncomingHandshake:441(14041ns) DropOutgoingHandshake:456(128ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:476(1028ns) {SwitchToState:178(409ns)}}}}}
::ffff:127.1.0.0
2025-11-19T14:00:04.723862Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435980386460043:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.723877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435980386460043:2094] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.724861Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435980386460043:2094] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:04.725048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435980386460033:2085] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.725064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:04.725071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435980386460033:2085] [node 6] ICP22 created new session: [4:7574435980386460071:2085]
2025-11-19T14:00:04.725077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435980386460071:2085] [node 6] ICS09 handshake done sender: [4:7574435980386460043:2094] self: [4:7574435980386460042:0] peer: [6:7574435981786624418:0] socket: 144 qp: -1
2025-11-19T14:00:04.725084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435980386460071:2085] [node 6] ICS10 traffic start
2025-11-19T14:00:04.725093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:04.726705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435979142547744:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(20832978ns) {Configure:104(20832168ns) {ProcessConfigured:135(20454166ns) {StartInitialHandshake:157(20453552ns) {DropHandshakes:469(460ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(756ns) {SwitchToState:178(178ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.728991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435980386459993:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(20920188ns) {Configure:104(20916366ns) {ProcessConfigured:135(20541397ns) {StartInitialHandshake:157(20541011ns) {DropHandshakes:469(479ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(81ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:476(985ns) {SwitchToState:178(477ns)}}}}}
2025-11-19T14:00:04.729165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435980386460074:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
2025-11-19T14:00:04.729388Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435981945788701:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.729453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435981945788625:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.727253Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435980617292432:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(20100196ns) {Configure:104(20099659ns) {ProcessConfigured:135(19761286ns) {StartInitialHandshake:157(19760505ns) {DropHandshakes:469(615ns) {DropIncomingHandshake:441(121ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(283ns) PrepareNewSessionHandshake:476(575ns) {SwitchToState:178(354ns)}}}}}
2025-11-19T14:00:04.727630Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435979142547745:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19676703ns) {Configure:104(19673067ns) {ProcessConfigured:135(19310393ns) {StartInitialHandshake:157(19309754ns) {DropHandshakes:469(464ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(49ns)} GenerateSessionVirtualId:988(339ns) PrepareNewSessionHandshake:476(748ns) {SwitchToState:178(153ns)}}}}}
2025-11-19T14:00:04.727722Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435979142547750:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.729673Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435980617292504:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.729709Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435980617292496:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.729712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435980617292496:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.729679Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435980386459994:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(21166046ns) {Configure:104(21165647ns) {ProcessConfigured:135(20936064ns) {StartInitialHandshake:157(20935414ns) {DropHandshakes:469(150ns) {DropIncomingHandshake:441(29ns) DropOutgoingHandshake:456(22ns)} GenerateSessionVirtualId:988(150ns) PrepareNewSessionHandshake:476(894ns) {SwitchToState:178(383ns)}}}}}
2025-11-19T14:00:04.731863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435979142547752:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1880
2025-11-19T14:00:04.730695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435981945788703:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24351
2025-11-19T14:00:04.736091Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435979142547751:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.736199Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435981945788703:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.736250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435981945788641:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.744203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435980617292432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.744337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435980617292513:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27739
2025-11-19T14:00:04.744965Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435979142547736:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.744978Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435979142547736:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.745952Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435978209643061:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.746123Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435980617292506:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.746151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435980617292433:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.750928Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435980386460047:2097] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.750984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435980386460033:2085] [node 6] ICP18 handshake (actor [4:7574435980386460047:2097]) is too old
2025-11-19T14:00:04.751193Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435980617292513:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.751284Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435978209643058:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.752327Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435978209643052:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.752342Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435978209643052:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.752426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435980617292509:2114] [node 6] ICP01 ready to work
2025-11-19T14:00:04.753253Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435978209643060:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.753277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435978209642983:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.753471Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435980617292496:2108] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:04.753589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435980617292493:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.753607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:04.753613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435980617292493:2107] [node 5] ICP22 created new session: [3:7574435980617292519:2107]
2025-11-19T14:00:04.753620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435980617292519:2107] [node 5] ICS09 handshake done sender: [3:7574435980617292496:2108] self: [3:7574435980617292494:0] peer: [5:7574435981945788704:0] socket: 158 qp: -1
2025-11-19T14:00:04.753626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435980617292519:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:04.753645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:04.752070Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435979142547752:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.751357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435981786624356:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9140) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435981786624417:2115] held: no
2025-11-19T14:00:04.756076Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435980386460068:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.756098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435980386460068:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.757427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435981786624419:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51965
2025-11-19T14:00:04.757535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435981786624420:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51965
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.758170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435979142547759:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1880
2025-11-19T14:00:04.760026Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435979142547759:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.761029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435978209643048:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(39566701ns) {Configure:104(39564978ns) {ProcessConfigured:135(39206022ns) {StartInitialHandshake:157(39205104ns) {DropHandshakes:469(1450ns) {DropIncomingHandshake:441(157ns) DropOutgoingHandshake:456(756ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(1410ns) {SwitchToState:178(1102ns)}}}}}
2025-11-19T14:00:04.761182Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435980386460074:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.761218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435980386459995:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.758801Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435981786624417:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:04.759017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435981786624356:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.759035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:04.759043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435981786624356:2061] [node 4] ICP22 created new session: [6:7574435981786624421:2061]
2025-11-19T14:00:04.759050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435981786624421:2061] [node 4] ICS09 handshake done sender: [6:7574435981786624417:2115] self: [6:7574435981786624418:0] peer: [4:7574435980386460042:0] socket: 157 qp: -1
2025-11-19T14:00:04.759060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435981786624421:2061] [node 4] ICS10 traffic start
2025-11-19T14:00:04.759086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.767065Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435978209643052:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:04.767251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435978209642984:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.767271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:04.767278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435978209642984:2049] [node 3] ICP22 created new session: [2:7574435978209643071:2049]
2025-11-19T14:00:04.767286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435978209643071:2049] [node 3] ICS09 handshake done sender: [2:7574435978209643052:2111] self: [2:7574435978209643051:0] peer: [3:7574435980617292515:0] socket: 161 qp: -1
2025-11-19T14:00:04.767304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435978209643071:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:04.767312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:04.765514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435981945788625:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23404) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435981945788701:2115] held: no
2025-11-19T14:00:04.767762Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435981945788702:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.767795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435981945788624:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.765948Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435979142547738:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.765977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435979142547738:2108] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.766566Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435979142547736:2107] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:04.766701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435979142547762:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1880
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.768764Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435981786624420:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.769043Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435980386460068:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:04.768424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435981945788707:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24351
2025-11-19T14:00:04.770957Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435980617292510:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.770976Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435980617292510:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.769010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435981786624339:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.771099Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435978209643054:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.771113Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435978209643054:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.769221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435980386460039:2088] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.769245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:04.769252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435980386460039:2088] [node 5] ICP22 created new session: [4:7574435980386460080:2088]
2025-11-19T14:00:04.769259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435980386460080:2088] [node 5] ICS09 handshake done sender: [4:7574435980386460068:2113] self: [4:7574435980386460048:0] peer: [5:7574435981945788705:0] socket: 171 qp: -1
2025-11-19T14:00:04.769268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435980386460080:2088] [node 5] ICS10 traffic start
2025-11-19T14:00:04.769279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.771408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435978209643075:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63518
2025-11-19T14:00:04.771536Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435979142547738:2108] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:04.771679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435979142547674:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.771701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:04.771707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435979142547674:2048] [node 2] ICP22 created new session: [1:7574435979142547765:2048]
2025-11-19T14:00:04.771714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435979142547765:2048] [node 2] ICS09 handshake done sender: [1:7574435979142547738:2108] self: [1:7574435979142547734:0] peer: [2:7574435978209643068:0] socket: 155 qp: -1
2025-11-19T14:00:04.771725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435979142547765:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:04.771734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.772944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435980386460082:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
2025-11-19T14:00:04.774352Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435981945788701:2115] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:04.774501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435981945788625:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.774527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:04.774534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435981945788625:2050] [node 3] ICP22 created new session: [5:7574435981945788708:2050]
2025-11-19T14:00:04.774543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435981945788708:2050] [node 3] ICS09 handshake done sender: [5:7574435981945788701:2115] self: [5:7574435981945788704:0] peer: [3:7574435980617292494:0] socket: 159 qp: -1
2025-11-19T14:00:04.774551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435981945788708:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:04.774569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:04.774758Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435980617292433:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7928) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435980617292506:2113] held: no
2025-11-19T14:00:04.775071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435979142547675:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.775095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:04.775105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435979142547675:2049] [node 3] ICP22 created new session: [1:7574435979142547768:2049]
2025-11-19T14:00:04.775112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435979142547768:2049] [node 3] ICS09 handshake done sender: [1:7574435979142547736:2107] self: [1:7574435979142547735:0] peer: [3:7574435980617292514:0] socket: 145 qp: -1
2025-11-19T14:00:04.775120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435979142547768:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:04.775129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:04.775360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435978209643074:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63518
2025-11-19T14:00:04.775797Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435980386460082:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.775854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435980386459993:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.780847Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435979142547749:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.780875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435979142547749:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.785197Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435978209642983:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22535) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435978209643060:2115] held: no
2025-11-19T14:00:04.785348Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435978209643074:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.785374Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435978209643060:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:04.785633Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435980617292510:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:04.785845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435980386460084:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
2025-11-19T14:00:04.785897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435980386460085:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.786500Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435980386460074:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.786011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435981786624425:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51965
2025-11-19T14:00:04.786659Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435980386460084:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.786698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574435980386460039:2088] [node 5] ICP18 handshake (actor [4:7574435980386460084:2118]) is too old
2025-11-19T14:00:04.786552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435981945788711:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24351
2025-11-19T14:00:04.786603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435981945788712:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24351
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.787080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435980617292501:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.787111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:04.787118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435980617292501:2110] [node 4] ICP22 created new session: [3:7574435980617292524:2110]
2025-11-19T14:00:04.787126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435980617292524:2110] [node 4] ICS09 handshake done sender: [3:7574435980617292510:2115] self: [3:7574435980617292502:0] peer: [4:7574435980386460077:0] socket: 178 qp: -1
2025-11-19T14:00:04.787133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435980617292524:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:04.787143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:04.787248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435981945788713:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24351
2025-11-19T14:00:04.787966Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435981786624425:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.788051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435981786624340:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.788224Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435981945788711:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.787557Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435979142547755:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.787573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435979142547755:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.787647Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435979142547762:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.789678Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435979142547754:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.789685Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435979142547754:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.790690Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435979142547749:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.790990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435980386459995:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.791011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:04.791019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435980386459995:2050] [node 3] ICP22 created new session: [4:7574435980386460086:2050]
2025-11-19T14:00:04.791027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435980386460086:2050] [node 3] ICS09 handshake done sender: [4:7574435980386460074:2115] self: [4:7574435980386460077:0] peer: [3:7574435980617292502:0] socket: 180 qp: -1
2025-11-19T14:00:04.791034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435980386460086:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:04.791048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:04.792000Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435980386459995:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:04.790940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435980617292526:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27739
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.792323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435980617292527:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27739
2025-11-19T14:00:04.792931Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435980617292504:2111] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:04.788559Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435981945788713:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.788600Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435981945788641:2062] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9140) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435981945788703:2117] held: no
2025-11-19T14:00:04.788782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435981945788623:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.788821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435981945788714:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24351
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.800625Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435978209643054:2112] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:04.800582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435981786624427:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51965
2025-11-19T14:00:04.800787Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435981786624420:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:04.800872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435981786624339:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.800891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:04.800898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435981786624339:2048] [node 1] ICP22 created new session: [6:7574435981786624428:2048]
2025-11-19T14:00:04.800905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435981786624428:2048] [node 1] ICS09 handshake done sender: [6:7574435981786624420:2117] self: [6:7574435981786624423:0] peer: [1:7574435979142547748:0] socket: 184 qp: -1
2025-11-19T14:00:04.800913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435981786624428:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:04.800923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:04.799913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435979142547675:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(32864099ns) {HandleHandshakeStatus:329(32863039ns) {DropIncomingHandshake:441(357ns) DropOutgoingHandshake:456(83ns) SwitchToState:178(251ns) ProcessPendingSessionEvents:495(12401087ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12370528ns max# 12245784ns}}}
2025-11-19T14:00:04.800006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435979142547675:2049] [node 3] ICP18 handshake (actor [1:7574435979142547762:2119]) is too old
2025-11-19T14:00:04.801537Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435980617292506:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:04.801817Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435979142547754:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:04.802634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435979142547744:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.802662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:04.802672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435979142547744:2110] [node 5] ICP22 created new session: [1:7574435979142547772:2110]
2025-11-19T14:00:04.802679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435979142547772:2110] [node 5] ICS09 handshake done sender: [1:7574435979142547754:2116] self: [1:7574435979142547746:0] peer: [5:7574435981945788715:0] socket: 190 qp: -1
2025-11-19T14:00:04.802687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435979142547772:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:04.802698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:04.801628Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435978209643065:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.801645Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435978209643065:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.802442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435978209642983:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.802456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:04.802464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435978209642983:2048] [node 1] ICP22 created new session: [2:7574435978209643078:2048]
2025-11-19T14:00:04.802471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435978209643078:2048] [node 1] ICS09 handshake done sender: [2:7574435978209643060:2115] self: [2:7574435978209643068:0] peer: [1:7574435979142547734:0] socket: 165 qp: -1
2025-11-19T14:00:04.802478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435978209643078:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:04.802486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:04.802760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435978209643050:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.802771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:04.802775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435978209643050:2109] [node 5] ICP22 created new session: [2:7574435978209643080:2109]
2025-11-19T14:00:04.802781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435978209643080:2109] [node 5] ICS09 handshake done sender: [2:7574435978209643054:2112] self: [2:7574435978209643053:0] peer: [5:7574435981945788706:0] socket: 162 qp: -1
2025-11-19T14:00:04.802785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435978209643080:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:04.802791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:04.803344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435980617292433:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.803386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:04.803393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435980617292433:2049] [node 2] ICP22 created new session: [3:7574435980617292529:2049]
2025-11-19T14:00:04.803400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435980617292529:2049] [node 2] ICS09 handshake done sender: [3:7574435980617292506:2113] self: [3:7574435980617292515:0] peer: [2:7574435978209643051:0] socket: 168 qp: -1
2025-11-19T14:00:04.803407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435980617292529:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:04.803415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.804326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435981786624430:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51965
2025-11-19T14:00:04.804513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435979142547743:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.804535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:04.804542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435979142547743:2109] [node 6] ICP22 created new session: [1:7574435979142547774:2109]
2025-11-19T14:00:04.804548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435979142547774:2109] [node 6] ICS09 handshake done sender: [1:7574435979142547749:2112] self: [1:7574435979142547748:0] peer: [6:7574435981786624423:0] socket: 179 qp: -1
2025-11-19T14:00:04.804554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435979142547774:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:04.804562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:04.805100Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435978209643065:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.805275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435980386460088:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
2025-11-19T14:00:04.805476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435978209643047:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.805498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:04.805503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435978209643047:2108] [node 6] ICP22 created new session: [2:7574435978209643084:2108]
2025-11-19T14:00:04.805508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435978209643084:2108] [node 6] ICS09 handshake done sender: [2:7574435978209643065:2117] self: [2:7574435978209643064:0] peer: [6:7574435981786624426:0] socket: 189 qp: -1
2025-11-19T14:00:04.805513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435978209643084:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:04.805521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.808781Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435979142547755:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:04.809036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435980617292432:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.809057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:04.809064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435980617292432:2048] [node 1] ICP22 created new session: [3:7574435980617292531:2048]
2025-11-19T14:00:04.809069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435980617292531:2048] [node 1] ICS09 handshake done sender: [3:7574435980617292504:2111] self: [3:7574435980617292514:0] peer: [1:7574435979142547735:0] socket: 156 qp: -1
2025-11-19T14:00:04.809076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435980617292531:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:04.809083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:04.818466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435981786624432:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51965
2025-11-19T14:00:04.819941Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435981945788703:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:04.820073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435981786624340:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7928) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435981786624425:2118] held: no
2025-11-19T14:00:04.820738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435981945788623:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22535) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435981945788711:2119] held: no
2025-11-19T14:00:04.824111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435979142547745:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.824154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:04.824165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435979142547745:2111] [node 4] ICP22 created new session: [1:7574435979142547777:2111]
2025-11-19T14:00:04.824175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435979142547777:2111] [node 4] ICS09 handshake done sender: [1:7574435979142547755:2117] self: [1:7574435979142547747:0] peer: [4:7574435980386460083:0] socket: 150 qp: -1
2025-11-19T14:00:04.824185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435979142547777:2111] [node 4] ICS10 traffic start
2025-11-19T14:00:04.824198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:04.825975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435980386459993:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22535) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435980386460082:2117] held: no
2025-11-19T14:00:04.828789Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435981786624425:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:04.828888Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435981786624430:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.828926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435981786624341:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.829692Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435981945788702:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:04.831196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435981945788717:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24351
2025-11-19T14:00:04.829993Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435980617292432:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(20842739ns) {HandleHandshakeStatus:329(20841656ns) {DropIncomingHandshake:441(431ns) DropOutgoingHandshake:456(1596ns) SwitchToState:178(110ns) ProcessPendingSessionEvents:495(20746965ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20724035ns max# 20646250ns}}}
2025-11-19T14:00:04.830074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435980617292522:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.830079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435980617292522:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:04.830631Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574435980617292432:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:04.831003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435980617292533:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27739
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.833439Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435978209643069:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.833451Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435978209643069:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.832469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435981945788641:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.832486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:04.832492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435981945788641:2062] [node 4] ICP22 created new session: [5:7574435981945788718:2062]
2025-11-19T14:00:04.832498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435981945788718:2062] [node 4] ICS09 handshake done sender: [5:7574435981945788703:2117] self: [5:7574435981945788705:0] peer: [4:7574435980386460048:0] socket: 175 qp: -1
2025-11-19T14:00:04.832506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435981945788718:2062] [node 4] ICS10 traffic start
2025-11-19T14:00:04.832513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.834792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435981786624434:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51965
2025-11-19T14:00:04.832275Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435980386460088:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.832320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435980386460089:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
2025-11-19T14:00:04.832378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435980386459994:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.834574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435980386460091:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46115
2025-11-19T14:00:04.834953Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435978209643069:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:04.835233Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435981945788624:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7928) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435981945788702:2116] held: no
2025-11-19T14:00:04.835528Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435981945788711:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:04.835877Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435980386460088:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:04.835987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435978209643048:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.836010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:04.836016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435978209643048:2107] [node 4] ICP22 created new session: [2:7574435978209643087:2107]
2025-11-19T14:00:04.836022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435978209643087:2107] [node 4] ICS09 handshake done sender: [2:7574435978209643069:2118] self: [2:7574435978209643063:0] peer: [4:7574435980386460090:0] socket: 170 qp: -1
2025-11-19T14:00:04.836028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435978209643087:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:04.836036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.837138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435981786624435:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51965
2025-11-19T14:00:04.837219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435981945788624:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.837234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:04.837241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435981945788624:2049] [node 2] ICP22 created new session: [5:7574435981945788721:2049]
2025-11-19T14:00:04.837248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435981945788721:2049] [node 2] ICS09 handshake done sender: [5:7574435981945788702:2116] self: [5:7574435981945788706:0] peer: [2:7574435978209643053:0] socket: 169 qp: -1
2025-11-19T14:00:04.837254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435981945788721:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:04.837273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:04.837754Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435980617292522:2118] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:04.837971Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435980386460082:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:04.838191Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435981786624430:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:04.838274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435981786624340:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.838288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:04.838296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435981786624340:2049] [node 2] ICP22 created new session: [6:7574435981786624436:2049]
2025-11-19T14:00:04.838302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435981786624436:2049] [node 2] ICS09 handshake done sender: [6:7574435981786624425:2118] self: [6:7574435981786624426:0] peer: [2:7574435978209643064:0] socket: 192 qp: -1
2025-11-19T14:00:04.838309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435981786624436:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:04.838316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:04.838671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435980386459994:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.838687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:04.838692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435980386459994:2049] [node 2] ICP22 created new session: [4:7574435980386460092:2049]
2025-11-19T14:00:04.838699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435980386460092:2049] [node 2] ICS09 handshake done sender: [4:7574435980386460088:2120] self: [4:7574435980386460090:0] peer: [2:7574435978209643063:0] socket: 209 qp: -1
2025-11-19T14:00:04.838703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435980386460092:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:04.838709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:04.840412Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435980617292533:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.840432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435980617292509:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.840441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:04.840446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435980617292509:2114] [node 6] ICP22 created new session: [3:7574435980617292535:2114]
2025-11-19T14:00:04.840453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435980617292535:2114] [node 6] ICS09 handshake done sender: [3:7574435980617292522:2118] self: [3:7574435980617292518:0] peer: [6:7574435981786624433:0] socket: 172 qp: -1
2025-11-19T14:00:04.840458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435980617292535:2114] [node 6] ICS10 traffic start
2025-11-19T14:00:04.840466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:04.840735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435980617292509:2114] [node 6] ICP18 handshake (actor [3:7574435980617292533:2121]) is too old
2025-11-19T14:00:04.841042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435981786624341:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.841063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:04.841069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435981786624341:2050] [node 3] ICP22 created new session: [6:7574435981786624438:2050]
2025-11-19T14:00:04.841075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435981786624438:2050] [node 3] ICS09 handshake done sender: [6:7574435981786624430:2120] self: [6:7574435981786624433:0] peer: [3:7574435980617292518:0] socket: 207 qp: -1
2025-11-19T14:00:04.841081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435981786624438:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:04.841087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:04.842796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435980386459993:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.842812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:04.842818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435980386459993:2048] [node 1] ICP22 created new session: [4:7574435980386460094:2048]
2025-11-19T14:00:04.842824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435980386460094:2048] [node 1] ICS09 handshake done sender: [4:7574435980386460082:2117] self: [4:7574435980386460083:0] peer: [1:7574435979142547747:0] socket: 174 qp: -1
2025-11-19T14:00:04.842829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435980386460094:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:04.842837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:04.845545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435981945788623:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.845563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:04.845570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435981945788623:2048] [node 1] ICP22 created new session: [5:7574435981945788723:2048]
2025-11-19T14:00:04.845575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435981945788723:2048] [node 1] ICS09 handshake done sender: [5:7574435981945788711:2119] self: [5:7574435981945788715:0] peer: [1:7574435979142547746:0] socket: 166 qp: -1
2025-11-19T14:00:04.845582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435981945788723:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:04.845588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:04.846068Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435981786624435:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.846105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435981786624365:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:04.846617Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435981945788700:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:04.846628Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435981945788700:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:04.847372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435981786624441:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51965
2025-11-19T14:00:04.847612Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435981945788700:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:04.847883Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435981786624435:2123] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:04.848061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435981945788656:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:04.848079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:04.848085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435981945788656:2067] [node 6] ICP22 created new session: [5:7574435981945788726:2067]
2025-11-19T14:00:04.848091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435981945788726:2067] [node 6] ICS09 handshake done sender: [5:7574435981945788700:2114] self: [5:7574435981945788699:0] peer: [6:7574435981786624440:0] socket: 210 qp: -1
2025-11-19T14:00:04.848097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435981945788726:2067] [node 6] ICS10 traffic start
2025-11-19T14:00:04.848105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:04.848609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435981786624365:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:04.848627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:04.848632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435981786624365:2067] [node 5] ICP22 created new session: [6:7574435981786624442:2067]
2025-11-19T14:00:04.848638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435981786624442:2067] [node 5] ICS09 handshake done sender: [6:7574435981786624435:2123] self: [6:7574435981786624440:0] peer: [5:7574435981945788699:0] socket: 211 qp: -1
2025-11-19T14:00:04.848644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435981786624442:2067] [node 5] ICS10 traffic start
2025-11-19T14:00:04.848650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:04.867384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:04.867486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435978209643078:2048] [node 1] ICS01 socket: 165 reason# EndOfStream
2025-11-19T14:00:04.867497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435978209642983:2048] [node 1] ICP30 unregister session Session# [2:7574435978209643078:2048] VirtualId# [2:7574435978209643068:0]
2025-11-19T14:00:04.867504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435978209643078:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.867555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:04.867577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:04.868099Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:04.868177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435980617292531:2048] [node 1] ICS07 socket disconnect 156 reason# EndOfStream
2025-11-19T14:00:04.868182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435980617292531:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.868217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:04.868236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435980617292531:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:04.883972Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:04.884078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435981945788723:2048] [node 1] ICS01 socket: 166 reason# EndOfStream
2025-11-19T14:00:04.884088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435981945788623:2048] [node 1] ICP30 unregister session Session# [5:7574435981945788723:2048] VirtualId# [5:7574435981945788715:0]
2025-11-19T14:00:04.884096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435981945788723:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.884151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:04.884177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:04.884226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:04.884293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435980386460094:2048] [node 1] ICS01 socket: 174 reason# EndOfStream
2025-11-19T14:00:04.884299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435980386459993:2048] [node 1] ICP30 unregister session Session# [4:7574435980386460094:2048] VirtualId# [4:7574435980386460083:0]
2025-11-19T14:00:04.884304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435980386460094:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.884345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:04.884369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:04.884517Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:04.884607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435981786624428:2048] [node 1] ICS07 socket disconnect 184 reason# EndOfStream
2025-11-19T14:00:04.884612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435981786624428:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.884655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:04.884678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435981786624428:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:04.889327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435981786624339:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22535) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:04.889359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435981786624428:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:04.889365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435981786624339:2048] [node 1] ICP30 unregister session Session# [6:7574435981786624428:2048] VirtualId# [6:7574435981786624423:0]
2025-11-19T14:00:04.889415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:04.889422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435981786624339:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22535) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:04.889425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22535) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:04.890451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435980617292432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22535) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T14:00:04.890467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435980617292531:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:04.893446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435980617292432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22535) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:04.893464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435980617292531:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:04.893470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435980617292432:2048] [node 1] ICP30 unregister session Session# [3:7574435980617292531:2048] VirtualId# [3:7574435980617292514:0]
2025-11-19T14:00:04.893519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:04.893525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435980617292432:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22535) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:04.893527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22535) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:04.894571Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:04.894647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435980617292529:2049] [node 2] ICS01 socket: 168 reason# EndOfStream
2025-11-19T14:00:04.894653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435980617292433:2049] [node 2] ICP30 unregister session Session# [3:7574435980617292529:2049] VirtualId# [3:7574435980617292515:0]
2025-11-19T14:00:04.894659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435980617292529:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.894698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:04.894718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:04.899975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435981786624339:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:04.900013Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:04.900096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435981786624436:2049] [node 2] ICS01 socket: 192 reason# EndOfStream
2025-11-19T14:00:04.900105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435981786624340:2049] [node 2] ICP30 unregister session Session# [6:7574435981786624436:2049] VirtualId# [6:7574435981786624426:0]
2025-11-19T14:00:04.900111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435981786624436:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.900181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:04.900206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:04.907524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:04.907594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435981945788721:2049] [node 2] ICS01 socket: 169 reason# EndOfStream
2025-11-19T14:00:04.907600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435981945788624:2049] [node 2] ICP30 unregister session Session# [5:7574435981945788721:2049] VirtualId# [5:7574435981945788706:0]
2025-11-19T14:00:04.907606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435981945788721:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.907637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:04.907660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:04.907217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:04.907336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435980386460092:2049] [node 2] ICS01 socket: 209 reason# EndOfStream
2025-11-19T14:00:04.907346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435980386459994:2049] [node 2] ICP30 unregister session Session# [4:7574435980386460092:2049] VirtualId# [4:7574435980386460090:0]
2025-11-19T14:00:04.907353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435980386460092:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.907420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:04.907448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:04.927461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:04.927563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435981786624438:2050] [node 3] ICS01 socket: 207 reason# EndOfStream
2025-11-19T14:00:04.927573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435981786624341:2050] [node 3] ICP30 unregister session Session# [6:7574435981786624438:2050] VirtualId# [6:7574435981786624433:0]
2025-11-19T14:00:04.927581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435981786624438:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.927654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:04.927683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:04.927800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:04.927869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435981945788708:2050] [node 3] ICS01 socket: 159 reason# EndOfStream
2025-11-19T14:00:04.927874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435981945788625:2050] [node 3] ICP30 unregister session Session# [5:7574435981945788708:2050] VirtualId# [5:7574435981945788704:0]
2025-11-19T14:00:04.927879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435981945788708:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.927916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:04.927941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:04.938711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:04.938827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435981786624421:2061] [node 4] ICS01 socket: 157 reason# EndOfStream
2025-11-19T14:00:04.938838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435981786624356:2061] [node 4] ICP30 unregister session Session# [6:7574435981786624421:2061] VirtualId# [6:7574435981786624418:0]
2025-11-19T14:00:04.938846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435981786624421:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:04.938903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:04.938931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:05.028039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435982539515683:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:05.028053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435982539515684:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:05.084262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435982122152420:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:05.084280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435982122152421:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:05.057006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435983752486603:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:05.057020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435983752486604:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:05.116262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435984165006127:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:05.116278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435984165006128:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:05.116284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435984165006129:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:05.144553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435982357640617:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:05.144567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435982357640618:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:05.144571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435982357640619:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:05.171999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435982364865268:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:05.172014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435982364865269:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:05.172020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435982364865270:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.225162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435983752486663:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61270
2025-11-19T14:00:05.223513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435982364865319:2097] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.226840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435984165006188:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47444
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.227973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982357640678:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50724
2025-11-19T14:00:05.228336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435982539515748:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:05.228443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435982539515749:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:05.232018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435983752486666:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:05.232244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435983752486667:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:05.232489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435983752486669:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:05.225947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435982364865327:2100] [node 5] ICP01 ready to work
2025-11-19T14:00:05.239057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435984165006188:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.241217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435982364865337:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43277
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.244513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435982539515758:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5892
2025-11-19T14:00:05.245179Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435982364865337:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.246601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435984165006190:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:05.247084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435984165006190:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.245654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435982364865268:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.250144Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435982539515744:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.250155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435982539515744:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.251325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435982539515755:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.251345Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435982539515755:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.251480Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435982539515744:2107] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:05.251921Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435982364865321:2098] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.251940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574435982364865321:2098] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.253677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435982122152489:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:05.248296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435983752486663:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.249690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435983752486603:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.254172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435982539515761:2114] [node 4] ICP01 ready to work
2025-11-19T14:00:05.255981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982357640679:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50724
2025-11-19T14:00:05.256045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982357640680:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50724
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.258435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435982364865341:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43277
2025-11-19T14:00:05.259112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435982122152486:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:05.256503Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435982539515758:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.257635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435982539515765:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5892
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.257908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435982357640683:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:05.258038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435982357640684:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.258685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435983752486684:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61270
2025-11-19T14:00:05.260264Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435982357640679:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.260291Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435982357640678:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.260544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435982364865342:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43277
2025-11-19T14:00:05.259643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435982122152487:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.262589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435984165006199:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47444
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.262632Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435982364865341:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.262694Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435982364865321:2098] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:05.262879Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435984165006188:2108] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:05.262974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435984165006190:2109] [node 6] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.262992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:05.262999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435984165006190:2109] [node 6] ICP22 created new session: [4:7574435984165006200:2109]
2025-11-19T14:00:05.263006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435984165006200:2109] [node 6] ICS09 handshake done sender: [4:7574435984165006188:2108] self: [4:7574435984165006191:0] peer: [6:7574435982364865320:0] socket: 154 qp: -1
2025-11-19T14:00:05.263013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435984165006200:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:05.263033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:05.263002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574435982364865319:2097] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.263022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:05.263028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435982364865319:2097] [node 4] ICP22 created new session: [6:7574435982364865351:2097]
2025-11-19T14:00:05.263035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435982364865351:2097] [node 4] ICS09 handshake done sender: [6:7574435982364865321:2098] self: [6:7574435982364865320:0] peer: [4:7574435984165006191:0] socket: 150 qp: -1
2025-11-19T14:00:05.263043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435982364865351:2097] [node 4] ICS10 traffic start
2025-11-19T14:00:05.263052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:05.268206Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435983752486674:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.268224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435983752486674:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.268619Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435983752486663:2107] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:05.270739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435982357640617:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.271768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435982364865349:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43277
2025-11-19T14:00:05.271855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435982364865269:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.271830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435982539515683:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.271865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:05.271878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435982539515683:2048] [node 2] ICP22 created new session: [1:7574435982539515767:2048]
2025-11-19T14:00:05.271887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435982539515767:2048] [node 2] ICS09 handshake done sender: [1:7574435982539515744:2107] self: [1:7574435982539515743:0] peer: [2:7574435983752486679:0] socket: 151 qp: -1
2025-11-19T14:00:05.271896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435982539515767:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:05.271910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.275588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435982364865354:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43277
2025-11-19T14:00:05.271900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435983752486685:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61270
2025-11-19T14:00:05.271967Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435983752486674:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:05.276210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435983752486669:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.276226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:05.276232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435983752486669:2110] [node 6] ICP22 created new session: [2:7574435983752486687:2110]
2025-11-19T14:00:05.276239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435983752486687:2110] [node 6] ICS09 handshake done sender: [2:7574435983752486674:2112] self: [2:7574435983752486673:0] peer: [6:7574435982364865350:0] socket: 164 qp: -1
2025-11-19T14:00:05.276245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435983752486687:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:05.276255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:05.271783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435982122152493:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34606
2025-11-19T14:00:05.277368Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435982357640680:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.278935Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435982364865349:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.278983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435982364865327:2100] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.280039Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435982539515765:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.280540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435983752486603:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.280565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:05.280572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435983752486603:2048] [node 1] ICP22 created new session: [2:7574435983752486690:2048]
2025-11-19T14:00:05.280578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435983752486690:2048] [node 1] ICS09 handshake done sender: [2:7574435983752486663:2107] self: [2:7574435983752486679:0] peer: [1:7574435982539515743:0] socket: 152 qp: -1
2025-11-19T14:00:05.280587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435983752486690:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:05.280618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.281911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435982364865356:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43277
2025-11-19T14:00:05.282340Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435982364865337:2110] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:05.282512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435982364865268:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.282527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:05.282533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435982364865268:2048] [node 1] ICP22 created new session: [6:7574435982364865357:2048]
2025-11-19T14:00:05.282539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435982364865357:2048] [node 1] ICS09 handshake done sender: [6:7574435982364865337:2110] self: [6:7574435982364865338:0] peer: [1:7574435982539515751:0] socket: 158 qp: -1
2025-11-19T14:00:05.282545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435982364865357:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:05.282556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.283742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435984165006202:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47444
2025-11-19T14:00:05.285147Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435984165006202:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.287389Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435982357640686:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.287406Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435982357640686:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.282080Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435982539515755:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:05.283042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435982539515749:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.283053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:05.283060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435982539515749:2109] [node 6] ICP22 created new session: [1:7574435982539515770:2109]
2025-11-19T14:00:05.283065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435982539515770:2109] [node 6] ICS09 handshake done sender: [1:7574435982539515755:2111] self: [1:7574435982539515751:0] peer: [6:7574435982364865338:0] socket: 157 qp: -1
2025-11-19T14:00:05.283071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435982539515770:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:05.283089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:05.288360Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435982357640686:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:05.288708Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435982364865341:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:05.288755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435982357640683:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.288780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:05.288788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435982357640683:2111] [node 6] ICP22 created new session: [5:7574435982357640697:2111]
2025-11-19T14:00:05.288796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435982357640697:2111] [node 6] ICS09 handshake done sender: [5:7574435982357640686:2113] self: [5:7574435982357640685:0] peer: [6:7574435982364865355:0] socket: 177 qp: -1
2025-11-19T14:00:05.288803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435982357640697:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:05.288813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:05.288924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435982364865269:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.288944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:05.288950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435982364865269:2049] [node 2] ICP22 created new session: [6:7574435982364865359:2049]
2025-11-19T14:00:05.288957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435982364865359:2049] [node 2] ICS09 handshake done sender: [6:7574435982364865341:2113] self: [6:7574435982364865350:0] peer: [2:7574435983752486673:0] socket: 166 qp: -1
2025-11-19T14:00:05.288963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435982364865359:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:05.288969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:05.291944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435982122152421:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(44765135ns) {Configure:104(44764166ns) {ProcessConfigured:135(44422834ns) {StartInitialHandshake:157(44422057ns) {DropHandshakes:469(694ns) {DropIncomingHandshake:441(81ns) DropOutgoingHandshake:456(151ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:476(1962ns) {SwitchToState:178(825ns)}}}}}
2025-11-19T14:00:05.292063Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435982122152493:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.292161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435982122152420:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.292324Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435982539515757:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.292340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435982539515757:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.292755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435984165006127:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(33760405ns) {Configure:104(33759844ns) {ProcessConfigured:135(33478189ns) {StartInitialHandshake:157(33477490ns) {DropHandshakes:469(640ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:476(954ns) {SwitchToState:178(129ns)}}}}}
2025-11-19T14:00:05.292842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435984165006127:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.293113Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435982357640619:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(33899904ns) {Configure:104(33898718ns) {ProcessConfigured:135(33619812ns) {StartInitialHandshake:157(33619384ns) {DropHandshakes:469(463ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(182ns) PrepareNewSessionHandshake:476(756ns) {SwitchToState:178(413ns)}}}}}
2025-11-19T14:00:05.293372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435982122152486:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(34128016ns) {Configure:104(34127537ns) {ProcessConfigured:135(33776834ns) {StartInitialHandshake:157(33776704ns) {DropHandshakes:469(422ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(256ns) PrepareNewSessionHandshake:476(748ns) {SwitchToState:178(601ns)}}}}}
2025-11-19T14:00:05.297445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435982357640618:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(37754080ns) {Configure:104(37752341ns) {ProcessConfigured:135(37477649ns) {StartInitialHandshake:157(37477195ns) {DropHandshakes:469(248ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(543ns) {SwitchToState:178(341ns)}}}}}
2025-11-19T14:00:05.297514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435982357640618:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.297075Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435984165006129:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(37428886ns) {Configure:104(37428308ns) {ProcessConfigured:135(37118812ns) {StartInitialHandshake:157(37117928ns) {DropHandshakes:469(413ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(81ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:476(1281ns) {SwitchToState:178(615ns)}}}}}
2025-11-19T14:00:05.297690Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435982122152487:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(37542060ns) {Configure:104(37541326ns) {ProcessConfigured:135(37274597ns) {StartInitialHandshake:157(37273988ns) {DropHandshakes:469(291ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(1311ns) {SwitchToState:178(206ns)}}}}}
2025-11-19T14:00:05.297740Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435983752486672:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.297755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435983752486672:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.299770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982357640704:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50724
2025-11-19T14:00:05.300106Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435983752486672:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.300255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435983752486667:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.300275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:05.300284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435983752486667:2109] [node 5] ICP22 created new session: [2:7574435983752486693:2109]
2025-11-19T14:00:05.300292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435983752486693:2109] [node 5] ICS09 handshake done sender: [2:7574435983752486672:2111] self: [2:7574435983752486671:0] peer: [5:7574435982357640703:0] socket: 162 qp: -1
2025-11-19T14:00:05.300299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435983752486693:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:05.300311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:05.302440Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435982357640680:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:05.302559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435982357640618:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.302577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:05.302584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435982357640618:2049] [node 2] ICP22 created new session: [5:7574435982357640705:2049]
2025-11-19T14:00:05.302592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435982357640705:2049] [node 2] ICS09 handshake done sender: [5:7574435982357640680:2110] self: [5:7574435982357640703:0] peer: [2:7574435983752486671:0] socket: 171 qp: -1
2025-11-19T14:00:05.302599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435982357640705:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:05.302624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:05.302711Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435982539515753:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.302740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435982539515753:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.302852Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435982539515764:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.302863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435982539515764:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.300871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435984165006208:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47444
2025-11-19T14:00:05.302795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435984165006197:2110] [node 5] ICP01 ready to work
2025-11-19T14:00:05.303175Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435984165006208:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.303214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435984165006128:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.316479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435982122152501:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34606
2025-11-19T14:00:05.316547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435982122152502:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34606
2025-11-19T14:00:05.316559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435982122152503:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34606
2025-11-19T14:00:05.316592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435982122152504:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34606
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.318815Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435982364865327:2100] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9414) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435982364865349:2117] held: no
2025-11-19T14:00:05.318991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435982364865361:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43277
2025-11-19T14:00:05.319896Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435983752486683:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.319921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435983752486683:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.322337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435984165006197:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19405044ns) {Configure:104(19404223ns) {ProcessConfigured:135(19083337ns) {StartInitialHandshake:157(19082556ns) {DropHandshakes:469(726ns) {DropIncomingHandshake:441(134ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(245ns) PrepareNewSessionHandshake:476(1056ns) {SwitchToState:178(682ns)}}}}}
2025-11-19T14:00:05.322659Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435982539515683:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(70782661ns) {HandleHandshakeStatus:329(70781621ns) {DropIncomingHandshake:441(327ns) DropOutgoingHandshake:456(650ns) SwitchToState:178(185ns) ProcessPendingSessionEvents:495(19290151ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19271854ns max# 19201708ns}}}
2025-11-19T14:00:05.322809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435982539515683:2048] [node 2] ICP18 handshake (actor [1:7574435982539515765:2116]) is too old
2025-11-19T14:00:05.322741Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435982122152501:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.322860Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435982122152503:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.322939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435982122152505:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34606
2025-11-19T14:00:05.323395Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435982357640683:2111] [node 6] ICP35 event processing took too much time PendingConnection:278(34554072ns) {HandleHandshakeStatus:329(34552896ns) {DropIncomingHandshake:441(4988ns) DropOutgoingHandshake:456(161ns) SwitchToState:178(202ns) ProcessPendingSessionEvents:495(19631664ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19610881ns max# 19545433ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.324155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435982357640618:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(21511705ns) {HandleHandshakeStatus:329(21511091ns) {DropIncomingHandshake:441(315ns) DropOutgoingHandshake:456(2530ns) SwitchToState:178(80ns) ProcessPendingSessionEvents:495(103465ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 84116ns max# 17103ns}}}
2025-11-19T14:00:05.320443Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435982364865269:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(31419406ns) {HandleHandshakeStatus:329(31418340ns) {DropIncomingHandshake:441(226ns) DropOutgoingHandshake:456(4966ns) SwitchToState:178(108ns) ProcessPendingSessionEvents:495(17625812ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17606280ns max# 17529713ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.325222Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435982539515757:2112] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:05.325443Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435983752486685:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.325564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435983752486696:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61270
2025-11-19T14:00:05.325602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435983752486697:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61270
2025-11-19T14:00:05.325632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435983752486698:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61270
2025-11-19T14:00:05.326084Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435982539515753:2110] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:05.326310Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435982539515764:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:05.326309Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435983752486697:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.326344Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435983752486678:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.326347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435983752486678:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.326734Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435982364865349:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:05.326869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435982364865327:2100] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.326883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:05.326889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435982364865327:2100] [node 5] ICP22 created new session: [6:7574435982364865362:2100]
2025-11-19T14:00:05.326893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435982364865362:2100] [node 5] ICS09 handshake done sender: [6:7574435982364865349:2117] self: [6:7574435982364865355:0] peer: [5:7574435982357640685:0] socket: 181 qp: -1
2025-11-19T14:00:05.323917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435984165006212:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47444
2025-11-19T14:00:05.326899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435982364865362:2100] [node 5] ICS10 traffic start
2025-11-19T14:00:05.326905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:05.327214Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435982364865342:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.326443Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435984165006202:2113] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:05.326521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435984165006127:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.326537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:05.326545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435984165006127:2048] [node 1] ICP22 created new session: [4:7574435984165006213:2048]
2025-11-19T14:00:05.326552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435984165006213:2048] [node 1] ICS09 handshake done sender: [4:7574435984165006202:2113] self: [4:7574435984165006204:0] peer: [1:7574435982539515763:0] socket: 192 qp: -1
2025-11-19T14:00:05.326559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435984165006213:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:05.326565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:05.324656Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435982122152504:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.325754Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435982122152502:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.325847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435982122152421:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.327203Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435982122152493:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:05.327254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435982122152420:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.327266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:05.327272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435982122152420:2048] [node 1] ICP22 created new session: [3:7574435982122152510:2048]
2025-11-19T14:00:05.327277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435982122152510:2048] [node 1] ICS09 handshake done sender: [3:7574435982122152493:2112] self: [3:7574435982122152497:0] peer: [1:7574435982539515752:0] socket: 174 qp: -1
2025-11-19T14:00:05.327282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435982122152510:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:05.327287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:05.327241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435982364865270:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.327447Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435983752486669:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:05.327671Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435983752486667:2109] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:05.328572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982357640707:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50724
2025-11-19T14:00:05.328762Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435983752486683:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:05.334503Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435983752486678:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:05.334785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435982539515748:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.334807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:05.334813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435982539515748:2108] [node 5] ICP22 created new session: [1:7574435982539515778:2108]
2025-11-19T14:00:05.334820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435982539515778:2108] [node 5] ICS09 handshake done sender: [1:7574435982539515753:2110] self: [1:7574435982539515750:0] peer: [5:7574435982357640693:0] socket: 156 qp: -1
2025-11-19T14:00:05.334827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435982539515778:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:05.334835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:05.335107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435982539515761:2114] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.335114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:05.335121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435982539515761:2114] [node 4] ICP22 created new session: [1:7574435982539515780:2114]
2025-11-19T14:00:05.335124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435982539515780:2114] [node 4] ICS09 handshake done sender: [1:7574435982539515764:2115] self: [1:7574435982539515763:0] peer: [4:7574435984165006204:0] socket: 191 qp: -1
2025-11-19T14:00:05.335129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435982539515780:2114] [node 4] ICS10 traffic start
2025-11-19T14:00:05.335135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:05.335337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435982539515684:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.335350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:05.335353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435982539515684:2049] [node 3] ICP22 created new session: [1:7574435982539515782:2049]
2025-11-19T14:00:05.335356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435982539515782:2049] [node 3] ICS09 handshake done sender: [1:7574435982539515757:2112] self: [1:7574435982539515752:0] peer: [3:7574435982122152497:0] socket: 170 qp: -1
2025-11-19T14:00:05.335360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435982539515782:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:05.335365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.336047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435982539515776:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5892
2025-11-19T14:00:05.336121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435982539515777:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5892
2025-11-19T14:00:05.336554Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435982539515776:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.336585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435982539515748:2108] [node 5] ICP18 handshake (actor [1:7574435982539515776:2117]) is too old
2025-11-19T14:00:05.336822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435983752486666:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.336841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:05.336846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435983752486666:2108] [node 4] ICP22 created new session: [2:7574435983752486703:2108]
2025-11-19T14:00:05.336856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435983752486703:2108] [node 4] ICS09 handshake done sender: [2:7574435983752486683:2115] self: [2:7574435983752486670:0] peer: [4:7574435984165006209:0] socket: 201 qp: -1
2025-11-19T14:00:05.336862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435983752486703:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:05.336869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:05.336926Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435982357640679:2109] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:05.337028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435983752486604:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.337041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:05.337044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435983752486604:2049] [node 3] ICP22 created new session: [2:7574435983752486705:2049]
2025-11-19T14:00:05.337049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435983752486705:2049] [node 3] ICS09 handshake done sender: [2:7574435983752486678:2114] self: [2:7574435983752486677:0] peer: [3:7574435982122152509:0] socket: 188 qp: -1
2025-11-19T14:00:05.337054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435983752486705:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:05.337059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.340153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435982357640617:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.340175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:05.340183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435982357640617:2048] [node 1] ICP22 created new session: [5:7574435982357640710:2048]
2025-11-19T14:00:05.340191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435982357640710:2048] [node 1] ICS09 handshake done sender: [5:7574435982357640679:2109] self: [5:7574435982357640693:0] peer: [1:7574435982539515750:0] socket: 160 qp: -1
2025-11-19T14:00:05.340196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435982357640710:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:05.340204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.340238Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435982122152491:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.340257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435982122152491:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.341010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435982122152512:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34606
2025-11-19T14:00:05.341344Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435982122152491:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:05.341765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435982122152489:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.341777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:05.341782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435982122152489:2110] [node 6] ICP22 created new session: [3:7574435982122152514:2110]
2025-11-19T14:00:05.341787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435982122152514:2110] [node 6] ICS09 handshake done sender: [3:7574435982122152491:2111] self: [3:7574435982122152490:0] peer: [6:7574435982364865364:0] socket: 169 qp: -1
2025-11-19T14:00:05.341792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435982122152514:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:05.341811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:05.342442Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435982122152502:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:05.343458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435982122152421:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.343467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:05.343474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435982122152421:2049] [node 2] ICP22 created new session: [3:7574435982122152517:2049]
2025-11-19T14:00:05.343479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435982122152517:2049] [node 2] ICS09 handshake done sender: [3:7574435982122152502:2117] self: [3:7574435982122152509:0] peer: [2:7574435983752486677:0] socket: 203 qp: -1
2025-11-19T14:00:05.343484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435982122152517:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:05.343490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:05.341325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435982364865365:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43277
2025-11-19T14:00:05.342203Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435982364865342:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:05.342593Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435982364865270:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11911) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435982364865342:2114] held: no
2025-11-19T14:00:05.342692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435982364865270:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.342700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:05.342706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435982364865270:2050] [node 3] ICP22 created new session: [6:7574435982364865366:2050]
2025-11-19T14:00:05.342710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435982364865366:2050] [node 3] ICS09 handshake done sender: [6:7574435982364865342:2114] self: [6:7574435982364865364:0] peer: [3:7574435982122152490:0] socket: 173 qp: -1
2025-11-19T14:00:05.342716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435982364865366:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:05.342722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.348974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435984165006215:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47444
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.353371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435984165006216:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47444
2025-11-19T14:00:05.353895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435984165006128:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22255) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435984165006208:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.355801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435982357640684:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(67095646ns) {Configure:104(54858818ns) {ProcessConfigured:135(54502944ns) {StartInitialHandshake:157(54502310ns) {DropHandshakes:469(419ns) {DropIncomingHandshake:441(35ns) DropOutgoingHandshake:456(43ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:476(548ns) {SwitchToState:178(51ns)}}}}}
2025-11-19T14:00:05.356062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982357640712:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50724
2025-11-19T14:00:05.360084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435983752486604:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(22934658ns) {HandleHandshakeStatus:329(22933686ns) {DropIncomingHandshake:441(2442ns) DropOutgoingHandshake:456(49ns) SwitchToState:178(60ns) ProcessPendingSessionEvents:495(22866989ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22838186ns max# 22762842ns}}}
2025-11-19T14:00:05.360379Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435984165006216:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.360410Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435984165006208:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:05.361922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435984165006128:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.361940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:05.361948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435984165006128:2049] [node 2] ICP22 created new session: [4:7574435984165006218:2049]
2025-11-19T14:00:05.361954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435984165006218:2049] [node 2] ICS09 handshake done sender: [4:7574435984165006208:2116] self: [4:7574435984165006209:0] peer: [2:7574435983752486670:0] socket: 202 qp: -1
2025-11-19T14:00:05.361961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435984165006218:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:05.361968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:05.361363Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435982122152420:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(34015394ns) {HandleHandshakeStatus:329(34015007ns) {DropIncomingHandshake:441(114ns) DropOutgoingHandshake:456(1190ns) SwitchToState:178(69ns) ProcessPendingSessionEvents:495(21278912ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21257386ns max# 21201170ns}}}
2025-11-19T14:00:05.363640Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435982357640617:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(23388125ns) {HandleHandshakeStatus:329(23387404ns) {DropIncomingHandshake:441(421ns) DropOutgoingHandshake:456(1844ns) SwitchToState:178(69ns) ProcessPendingSessionEvents:495(23308291ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23288084ns max# 23209480ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.364875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435984165006220:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47444
2025-11-19T14:00:05.364928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435983752486666:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(28013947ns) {HandleHandshakeStatus:329(28013029ns) {DropIncomingHandshake:441(3825ns) DropOutgoingHandshake:456(123ns) SwitchToState:178(56ns) ProcessPendingSessionEvents:495(11035361ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11012809ns max# 10930792ns}}}
2025-11-19T14:00:05.366761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435982539515761:2114] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:05.367800Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435982357640712:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.367849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435982357640619:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.367946Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435982122152499:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.367958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435982122152499:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.368803Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435982122152499:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:05.368886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435982122152486:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.368905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:05.368913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435982122152486:2108] [node 5] ICP22 created new session: [3:7574435982122152521:2108]
2025-11-19T14:00:05.368918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435982122152521:2108] [node 5] ICS09 handshake done sender: [3:7574435982122152499:2114] self: [3:7574435982122152494:0] peer: [5:7574435982357640714:0] socket: 161 qp: -1
2025-11-19T14:00:05.368924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435982122152521:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:05.368931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.372051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982357640713:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50724
2025-11-19T14:00:05.372312Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435982357640713:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.372382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982357640715:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50724
2025-11-19T14:00:05.372597Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435982357640712:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:05.373683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435982357640619:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.373707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:05.373714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435982357640619:2050] [node 3] ICP22 created new session: [5:7574435982357640718:2050]
2025-11-19T14:00:05.373720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435982357640718:2050] [node 3] ICS09 handshake done sender: [5:7574435982357640712:2120] self: [5:7574435982357640714:0] peer: [3:7574435982122152494:0] socket: 175 qp: -1
2025-11-19T14:00:05.373727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435982357640718:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:05.373735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.375962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982357640717:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50724
2025-11-19T14:00:05.372680Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435984165006211:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.372693Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435984165006211:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.375168Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435984165006211:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:05.375265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435984165006197:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.375287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:05.375293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435984165006197:2110] [node 5] ICP22 created new session: [4:7574435984165006223:2110]
2025-11-19T14:00:05.375309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435984165006223:2110] [node 5] ICS09 handshake done sender: [4:7574435984165006211:2117] self: [4:7574435984165006210:0] peer: [5:7574435982357640716:0] socket: 212 qp: -1
2025-11-19T14:00:05.375316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435984165006223:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:05.375323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:05.377387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435982357640684:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.377420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435982357640684:2112] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21689) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435982357640713:2121] held: no
2025-11-19T14:00:05.377569Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435982357640713:2121] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:05.377933Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435984165006220:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.377986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435984165006129:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.378994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435984165006226:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47444
2025-11-19T14:00:05.379142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435982357640684:2112] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.379158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:05.379164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435982357640684:2112] [node 4] ICP22 created new session: [5:7574435982357640720:2112]
2025-11-19T14:00:05.379170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435982357640720:2112] [node 4] ICS09 handshake done sender: [5:7574435982357640713:2121] self: [5:7574435982357640716:0] peer: [4:7574435984165006210:0] socket: 213 qp: -1
2025-11-19T14:00:05.379176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435982357640720:2112] [node 4] ICS10 traffic start
2025-11-19T14:00:05.379188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:05.378133Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435982122152500:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.378144Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435982122152500:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.379660Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435982122152500:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:05.379904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435982122152487:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.379934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:05.379939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435982122152487:2107] [node 4] ICP22 created new session: [3:7574435982122152524:2107]
2025-11-19T14:00:05.379945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435982122152524:2107] [node 4] ICS09 handshake done sender: [3:7574435982122152500:2115] self: [3:7574435982122152495:0] peer: [4:7574435984165006225:0] socket: 185 qp: -1
2025-11-19T14:00:05.379951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435982122152524:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:05.379959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:05.380403Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435984165006220:2121] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:05.381050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435984165006129:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11911) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435984165006220:2121] held: no
2025-11-19T14:00:05.381296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435984165006129:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.381308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:05.381314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435984165006129:2050] [node 3] ICP22 created new session: [4:7574435984165006227:2050]
2025-11-19T14:00:05.381319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435984165006227:2050] [node 3] ICS09 handshake done sender: [4:7574435984165006220:2121] self: [4:7574435984165006225:0] peer: [3:7574435982122152495:0] socket: 186 qp: -1
2025-11-19T14:00:05.381325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435984165006227:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:05.381335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:05.399323Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:05.399421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435983752486690:2048] [node 1] ICS01 socket: 152 reason# EndOfStream
2025-11-19T14:00:05.399431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435983752486603:2048] [node 1] ICP30 unregister session Session# [2:7574435983752486690:2048] VirtualId# [2:7574435983752486679:0]
2025-11-19T14:00:05.399439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435983752486690:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.399481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:05.399503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:05.403507Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:05.403605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435982357640710:2048] [node 1] ICS07 socket disconnect 160 reason# EndOfStream
2025-11-19T14:00:05.403612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435982357640710:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.403678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:05.403704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435982357640710:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:05.405379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:05.405436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435982364865357:2048] [node 1] ICS01 socket: 158 reason# EndOfStream
2025-11-19T14:00:05.405443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435982364865268:2048] [node 1] ICP30 unregister session Session# [6:7574435982364865357:2048] VirtualId# [6:7574435982364865338:0]
2025-11-19T14:00:05.405451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435982364865357:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.405486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:05.405505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:05.406230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435982357640617:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1047) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:05.406244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435982357640710:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:05.406249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435982357640617:2048] [node 1] ICP30 unregister session Session# [5:7574435982357640710:2048] VirtualId# [5:7574435982357640693:0]
2025-11-19T14:00:05.406290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:05.406296Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435982357640617:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:05.406298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:05.403949Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:05.404018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574435982122152510:2048] [node 1] ICS07 socket disconnect 174 reason# EndOfStream
2025-11-19T14:00:05.404023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435982122152510:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.404056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:05.404076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574435982122152510:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:05.404633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435982122152420:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1047) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:05.404640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435982122152510:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:05.404645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435982122152420:2048] [node 1] ICP30 unregister session Session# [3:7574435982122152510:2048] VirtualId# [3:7574435982122152497:0]
2025-11-19T14:00:05.404688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:05.404693Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574435982122152420:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:05.404696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:05.404872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:05.404977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574435984165006213:2048] [node 1] ICS07 socket disconnect 192 reason# EndOfStream
2025-11-19T14:00:05.404984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435984165006213:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.405028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:05.405048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574435984165006213:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:05.405820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435984165006127:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1047) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:05.405854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435984165006213:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:05.405857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435984165006127:2048] [node 1] ICP30 unregister session Session# [4:7574435984165006213:2048] VirtualId# [4:7574435984165006204:0]
2025-11-19T14:00:05.405896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:05.405901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574435984165006127:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:05.405904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:05.413079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574435982357640617:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:05.413140Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:05.413230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435982122152517:2049] [node 2] ICS01 socket: 203 reason# EndOfStream
2025-11-19T14:00:05.413238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435982122152421:2049] [node 2] ICP30 unregister session Session# [3:7574435982122152517:2049] VirtualId# [3:7574435982122152509:0]
2025-11-19T14:00:05.413243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435982122152517:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.413279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:05.413299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:05.416802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574435984165006127:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:05.422097Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:05.422206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435984165006218:2049] [node 2] ICS01 socket: 202 reason# EndOfStream
2025-11-19T14:00:05.422216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435984165006128:2049] [node 2] ICP30 unregister session Session# [4:7574435984165006218:2049] VirtualId# [4:7574435984165006209:0]
2025-11-19T14:00:05.422224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435984165006218:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.422278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:05.422306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:05.428233Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:05.428333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435982357640705:2049] [node 2] ICS01 socket: 171 reason# EndOfStream
2025-11-19T14:00:05.428343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435982357640618:2049] [node 2] ICP30 unregister session Session# [5:7574435982357640705:2049] VirtualId# [5:7574435982357640703:0]
2025-11-19T14:00:05.428352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435982357640705:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.428397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:05.428423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:05.432414Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:05.432526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574435982364865359:2049] [node 2] ICS07 socket disconnect 166 reason# EndOfStream
2025-11-19T14:00:05.432535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435982364865359:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.432613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:05.432639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574435982364865359:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:05.434530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435982364865269:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22255) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:05.434547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435982364865359:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:05.434553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435982364865269:2049] [node 2] ICP30 unregister session Session# [6:7574435982364865359:2049] VirtualId# [6:7574435982364865350:0]
2025-11-19T14:00:05.434616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:05.434636Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574435982364865269:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22255) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:05.434640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22255) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:05.448129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574435982364865269:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:05.466494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:05.466562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435982364865366:2050] [node 3] ICS01 socket: 173 reason# EndOfStream
2025-11-19T14:00:05.466570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435982364865270:2050] [node 3] ICP30 unregister session Session# [6:7574435982364865366:2050] VirtualId# [6:7574435982364865364:0]
2025-11-19T14:00:05.466575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435982364865366:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.466608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:05.466633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:05.468073Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:05.468178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435982357640718:2050] [node 3] ICS01 socket: 175 reason# EndOfStream
2025-11-19T14:00:05.468188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435982357640619:2050] [node 3] ICP30 unregister session Session# [5:7574435982357640718:2050] VirtualId# [5:7574435982357640714:0]
2025-11-19T14:00:05.468195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435982357640718:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.468256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:05.468283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:05.497305Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:05.497417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435982364865351:2097] [node 4] ICS01 socket: 150 reason# EndOfStream
2025-11-19T14:00:05.497427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435982364865319:2097] [node 4] ICP30 unregister session Session# [6:7574435982364865351:2097] VirtualId# [6:7574435982364865320:0]
2025-11-19T14:00:05.497437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435982364865351:2097] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.497485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:05.497518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:05.591940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435983738615016:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:05.591963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435983738615017:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:05.620174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435984579361558:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:05.620187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435984579361559:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:05.655887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435985876348924:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:05.655903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435985876348925:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:05.676721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435982017061529:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:05.676741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435982017061530:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:05.676746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435982017061531:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:05.689349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435982431163029:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:05.689386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435982431163030:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:05.689392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435982431163031:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:05.708403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435985876348936:2058] [node 5] ICP01 ready to work
2025-11-19T14:00:05.708394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435983738615056:2086] [node 4] ICP01 ready to work
2025-11-19T14:00:05.708900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435984579361585:2067] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.715203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435985876348942:2060] [node 6] ICP01 ready to work
2025-11-19T14:00:05.711976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435982431163066:2070] [node 6] ICP01 ready to work
2025-11-19T14:00:05.712087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435982431163067:2072] [node 4] ICP01 ready to work
2025-11-19T14:00:05.714958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982431163068:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38683
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.724516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435983738615082:2089] [node 6] ICP01 ready to work
2025-11-19T14:00:05.724729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435983738615084:2094] [node 5] ICP01 ready to work
2025-11-19T14:00:05.725670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435983738615083:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9014
2025-11-19T14:00:05.726010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435984579361559:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17006725ns) {Configure:104(17006328ns) {ProcessConfigured:135(16745586ns) {StartInitialHandshake:157(16745284ns) {DropHandshakes:469(373ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(44ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:476(656ns) {SwitchToState:178(379ns)}}}}}
2025-11-19T14:00:05.726507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435984579361618:2076] [node 6] ICP01 ready to work
2025-11-19T14:00:05.726708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435984579361621:2078] [node 5] ICP01 ready to work
2025-11-19T14:00:05.727360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435984579361619:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40555
2025-11-19T14:00:05.727413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435984579361620:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40555
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.734175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435985876348942:2060] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(18886079ns) {Configure:104(18885629ns) {ProcessConfigured:135(17346037ns) {StartInitialHandshake:157(17345798ns) {DropHandshakes:469(1751ns) {DropIncomingHandshake:441(1083ns) DropOutgoingHandshake:456(136ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(1041ns) {SwitchToState:178(138ns)}}}}}
2025-11-19T14:00:05.734431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435985876348992:2067] [node 4] ICP01 ready to work
2025-11-19T14:00:05.735246Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435983738615083:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.709564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435986009536685:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:05.709601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435986009536686:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:05.709621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435986009536687:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:05.709870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435986009536719:2075] [node 5] ICP01 ready to work
2025-11-19T14:00:05.732997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435986009536756:2086] [node 4] ICP01 ready to work
2025-11-19T14:00:05.734071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986009536757:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23342
2025-11-19T14:00:05.734084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986009536758:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23342
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.754681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986009536764:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23342
2025-11-19T14:00:05.751980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435982017061531:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(42915217ns) {Configure:104(42913373ns) {ProcessConfigured:135(42525922ns) {StartInitialHandshake:157(42525316ns) {DropHandshakes:469(363ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(2278ns) {SwitchToState:178(1053ns)}}}}}
2025-11-19T14:00:05.752195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435982017061593:2065] [node 5] ICP01 ready to work
2025-11-19T14:00:05.752372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435982017061594:2070] [node 6] ICP01 ready to work
2025-11-19T14:00:05.753866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435982017061595:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61484
2025-11-19T14:00:05.753886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435982017061596:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61484
2025-11-19T14:00:05.753900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435982017061597:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61484
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.757197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435985876348925:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(22473206ns) {Configure:104(22472664ns) {ProcessConfigured:135(22106345ns) {StartInitialHandshake:157(22105614ns) {DropHandshakes:469(446ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(1804ns) {SwitchToState:178(1282ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.761845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435983738615096:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9014
2025-11-19T14:00:05.761579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982431163108:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38683
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.768604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435985876348990:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59637
2025-11-19T14:00:05.768670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435985876348991:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59637
2025-11-19T14:00:05.768766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435985876349003:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59637
2025-11-19T14:00:05.771973Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435982431163068:2084] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.787503Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435984579361619:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.787963Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435983738615096:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.787733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982431163110:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38683
2025-11-19T14:00:05.790737Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435985876349003:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.790784Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435985876348990:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.791180Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435982017061595:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.791219Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435982017061596:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.791293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435982017061529:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.787738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435984579361639:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40555
2025-11-19T14:00:05.788143Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435986009536757:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.788278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986009536766:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23342
2025-11-19T14:00:05.788298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435986009536719:2075] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.791489Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435982431163072:2086] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.791501Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435982431163072:2086] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.791951Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435982431163108:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.792086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435982431163067:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.792279Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435984579361620:2104] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.792319Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435986009536764:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.792519Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435982431163072:2086] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:05.792747Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435985876348991:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.792775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435985876348925:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.792153Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435983738615079:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.792172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435983738615079:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.792682Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435983738615081:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.792687Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435983738615081:2109] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.792415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435984579361558:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.795619Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435984579361639:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.793635Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435986009536758:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.793672Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435986009536766:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.793724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435986009536756:2086] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.793736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435986009536685:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.799832Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435982017061599:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.799860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435982017061599:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.800492Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435982017061597:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.800738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435982017061530:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.800785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435982017061612:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61484
2025-11-19T14:00:05.801015Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435983738615079:2108] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:05.801105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435982431163066:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.801134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:05.801141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435982431163066:2070] [node 6] ICP22 created new session: [5:7574435982431163113:2070]
2025-11-19T14:00:05.801149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435982431163113:2070] [node 6] ICS09 handshake done sender: [5:7574435982431163072:2086] self: [5:7574435982431163071:0] peer: [6:7574435986009536767:0] socket: 161 qp: -1
2025-11-19T14:00:05.801157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435982431163113:2070] [node 6] ICS10 traffic start
2025-11-19T14:00:05.801167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:05.801371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986009536768:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23342
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.805218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435985876349009:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59637
2025-11-19T14:00:05.803866Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435984579361615:2101] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.803891Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435984579361615:2101] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.804450Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435984579361617:2102] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.804454Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435984579361617:2102] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.805147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435984579361643:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40555
2025-11-19T14:00:05.805394Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435984579361615:2101] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:05.806141Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435982431163110:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.805695Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435982017061599:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:05.805821Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435983738615081:2109] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:05.812084Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435985876348991:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:05.812375Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435984579361617:2102] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:05.812616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435986009536687:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.812672Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435986009536757:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:05.813042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435983738615056:2086] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.813065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:05.813075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435983738615056:2086] [node 4] ICP22 created new session: [1:7574435983738615101:2086]
2025-11-19T14:00:05.813082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435983738615101:2086] [node 4] ICS09 handshake done sender: [1:7574435983738615079:2108] self: [1:7574435983738615064:0] peer: [4:7574435982017061611:0] socket: 164 qp: -1
2025-11-19T14:00:05.813092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435983738615101:2086] [node 4] ICS10 traffic start
2025-11-19T14:00:05.813104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:05.813433Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435983738615086:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.813437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435983738615086:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.813999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435983738615016:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.814016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:05.814024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435983738615016:2048] [node 2] ICP22 created new session: [1:7574435983738615103:2048]
2025-11-19T14:00:05.814030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435983738615103:2048] [node 2] ICS09 handshake done sender: [1:7574435983738615081:2109] self: [1:7574435983738615080:0] peer: [2:7574435984579361641:0] socket: 165 qp: -1
2025-11-19T14:00:05.814038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435983738615103:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:05.814044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:05.814622Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435982017061601:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.814635Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435982017061601:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.815070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435982017061593:2065] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.815082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:05.815088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435982017061593:2065] [node 5] ICP22 created new session: [4:7574435982017061616:2065]
2025-11-19T14:00:05.815093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435982017061616:2065] [node 5] ICS09 handshake done sender: [4:7574435982017061599:2113] self: [4:7574435982017061598:0] peer: [5:7574435982431163112:0] socket: 181 qp: -1
2025-11-19T14:00:05.815100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435982017061616:2065] [node 5] ICS10 traffic start
2025-11-19T14:00:05.815107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:05.815336Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435982017061596:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.815876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435982431163031:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.815916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982431163115:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38683
2025-11-19T14:00:05.816075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435984579361559:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.816098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:05.816104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435984579361559:2049] [node 3] ICP22 created new session: [2:7574435984579361646:2049]
2025-11-19T14:00:05.816109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435984579361646:2049] [node 3] ICS09 handshake done sender: [2:7574435984579361615:2101] self: [2:7574435984579361588:0] peer: [3:7574435985876349008:0] socket: 170 qp: -1
2025-11-19T14:00:05.816115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435984579361646:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:05.816122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:05.816521Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435985876348989:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.816533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435985876348989:2108] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.816965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435985876348925:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.816977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:05.816982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435985876348925:2049] [node 2] ICP22 created new session: [3:7574435985876349010:2049]
2025-11-19T14:00:05.816988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435985876349010:2049] [node 2] ICS09 handshake done sender: [3:7574435985876348991:2110] self: [3:7574435985876349008:0] peer: [2:7574435984579361588:0] socket: 175 qp: -1
2025-11-19T14:00:05.816994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435985876349010:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:05.817003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:05.819252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435984579361585:2067] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.819271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:05.819278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435984579361585:2067] [node 4] ICP22 created new session: [2:7574435984579361649:2067]
2025-11-19T14:00:05.819285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435984579361649:2067] [node 4] ICS09 handshake done sender: [2:7574435984579361617:2102] self: [2:7574435984579361616:0] peer: [4:7574435982017061614:0] socket: 171 qp: -1
2025-11-19T14:00:05.819291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435984579361649:2067] [node 4] ICS10 traffic start
2025-11-19T14:00:05.819301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:05.819986Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435982431163108:2116] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:05.820170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435986009536719:2075] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.820193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:05.820199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435986009536719:2075] [node 5] ICP22 created new session: [6:7574435986009536772:2075]
2025-11-19T14:00:05.820205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435986009536772:2075] [node 5] ICS09 handshake done sender: [6:7574435986009536757:2111] self: [6:7574435986009536767:0] peer: [5:7574435982431163071:0] socket: 172 qp: -1
2025-11-19T14:00:05.820212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435986009536772:2075] [node 5] ICS10 traffic start
2025-11-19T14:00:05.820219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:05.820372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435986009536719:2075] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:05.822102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435982431163067:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.822122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:05.822129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435982431163067:2072] [node 4] ICP22 created new session: [5:7574435982431163118:2072]
2025-11-19T14:00:05.822135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435982431163118:2072] [node 4] ICS09 handshake done sender: [5:7574435982431163108:2116] self: [5:7574435982431163112:0] peer: [4:7574435982017061598:0] socket: 184 qp: -1
2025-11-19T14:00:05.822141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435982431163118:2072] [node 4] ICS10 traffic start
2025-11-19T14:00:05.822147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:05.823146Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435984579361620:2104] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.823327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435982017061529:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.823348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:05.823354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435982017061529:2048] [node 1] ICP22 created new session: [4:7574435982017061619:2048]
2025-11-19T14:00:05.823360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435982017061619:2048] [node 1] ICS09 handshake done sender: [4:7574435982017061596:2111] self: [4:7574435982017061611:0] peer: [1:7574435983738615064:0] socket: 178 qp: -1
2025-11-19T14:00:05.823367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435982017061619:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:05.823374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:05.824066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435982017061615:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61484
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.836435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986009536774:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23342
2025-11-19T14:00:05.836462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986009536775:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23342
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.837469Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435985876348941:2062] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.837483Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435985876348941:2062] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.838353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435985876349013:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59637
2025-11-19T14:00:05.838373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435985876349014:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59637
2025-11-19T14:00:05.838568Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435982017061601:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:05.838892Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435982017061597:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:05.839009Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435983738615086:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:05.839160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435983738615082:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.839181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:05.839189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435983738615082:2089] [node 6] ICP22 created new session: [1:7574435983738615108:2089]
2025-11-19T14:00:05.839195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435983738615108:2089] [node 6] ICS09 handshake done sender: [1:7574435983738615086:2111] self: [1:7574435983738615085:0] peer: [6:7574435986009536770:0] socket: 166 qp: -1
2025-11-19T14:00:05.839201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435983738615108:2089] [node 6] ICS10 traffic start
2025-11-19T14:00:05.839210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.839704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435982017061594:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.839719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:05.839724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435982017061594:2070] [node 6] ICP22 created new session: [4:7574435982017061623:2070]
2025-11-19T14:00:05.839728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435982017061623:2070] [node 6] ICS09 handshake done sender: [4:7574435982017061601:2114] self: [4:7574435982017061600:0] peer: [6:7574435986009536769:0] socket: 182 qp: -1
2025-11-19T14:00:05.839735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435982017061623:2070] [node 6] ICS10 traffic start
2025-11-19T14:00:05.839760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:05.840046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435982017061530:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.840057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:05.840062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435982017061530:2049] [node 2] ICP22 created new session: [4:7574435982017061625:2049]
2025-11-19T14:00:05.840066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435982017061625:2049] [node 2] ICS09 handshake done sender: [4:7574435982017061597:2112] self: [4:7574435982017061614:0] peer: [2:7574435984579361616:0] socket: 179 qp: -1
2025-11-19T14:00:05.840070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435982017061625:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:05.840075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:05.840607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435982017061622:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61484
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.841209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982431163120:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38683
2025-11-19T14:00:05.841230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982431163121:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38683
2025-11-19T14:00:05.841241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982431163122:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38683
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.841517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435984579361558:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.841541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:05.841547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435984579361558:2048] [node 1] ICP22 created new session: [2:7574435984579361653:2048]
2025-11-19T14:00:05.841553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435984579361653:2048] [node 1] ICS09 handshake done sender: [2:7574435984579361620:2104] self: [2:7574435984579361641:0] peer: [1:7574435983738615080:0] socket: 168 qp: -1
2025-11-19T14:00:05.841560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435984579361653:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:05.841569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:05.841721Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435984579361558:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:05.841931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435984579361648:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40555
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.845577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435983738615110:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9014
2025-11-19T14:00:05.846493Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435985876348989:2108] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:05.847925Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435985876348941:2062] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:05.848217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435985876348942:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.848235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:05.848241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435985876348942:2060] [node 6] ICP22 created new session: [3:7574435985876349017:2060]
2025-11-19T14:00:05.848249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435985876349017:2060] [node 6] ICS09 handshake done sender: [3:7574435985876348989:2108] self: [3:7574435985876348960:0] peer: [6:7574435986009536771:0] socket: 187 qp: -1
2025-11-19T14:00:05.848256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435985876349017:2060] [node 6] ICS10 traffic start
2025-11-19T14:00:05.848265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:05.848474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435985876348936:2058] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.848486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:05.848489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435985876348936:2058] [node 5] ICP22 created new session: [3:7574435985876349019:2058]
2025-11-19T14:00:05.848493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435985876349019:2058] [node 5] ICS09 handshake done sender: [3:7574435985876348941:2062] self: [3:7574435985876348939:0] peer: [5:7574435982431163117:0] socket: 185 qp: -1
2025-11-19T14:00:05.848497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435985876349019:2058] [node 5] ICS10 traffic start
2025-11-19T14:00:05.848502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:05.849023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435986009536685:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435986009536758:2112] held: no
2025-11-19T14:00:05.850270Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435986009536764:2116] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:05.850940Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435984579361648:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.855961Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435986009536758:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:05.856141Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435982431163031:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435982431163110:2117] held: no
2025-11-19T14:00:05.856253Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435982431163110:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.856471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986009536777:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23342
2025-11-19T14:00:05.856527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986009536778:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23342
2025-11-19T14:00:05.856687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435982431163031:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.856707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:05.856713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435982431163031:2050] [node 3] ICP22 created new session: [5:7574435982431163124:2050]
2025-11-19T14:00:05.856720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435982431163124:2050] [node 3] ICS09 handshake done sender: [5:7574435982431163110:2117] self: [5:7574435982431163117:0] peer: [3:7574435985876348939:0] socket: 189 qp: -1
2025-11-19T14:00:05.856727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435982431163124:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:05.856736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:05.857255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435986009536756:2086] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.857271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:05.857276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435986009536756:2086] [node 4] ICP22 created new session: [6:7574435986009536779:2086]
2025-11-19T14:00:05.857281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435986009536779:2086] [node 4] ICS09 handshake done sender: [6:7574435986009536764:2116] self: [6:7574435986009536769:0] peer: [4:7574435982017061600:0] socket: 183 qp: -1
2025-11-19T14:00:05.857287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435986009536779:2086] [node 4] ICS10 traffic start
2025-11-19T14:00:05.857294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:05.857592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435986009536685:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.857597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:05.857602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435986009536685:2048] [node 1] ICP22 created new session: [6:7574435986009536781:2048]
2025-11-19T14:00:05.857606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435986009536781:2048] [node 1] ICS09 handshake done sender: [6:7574435986009536758:2112] self: [6:7574435986009536770:0] peer: [1:7574435983738615085:0] socket: 173 qp: -1
2025-11-19T14:00:05.857612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435986009536781:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:05.857617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:05.857972Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435986009536766:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:05.858519Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435985876349014:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.859599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435985876348942:2060] [node 6] ICP18 handshake (actor [3:7574435985876349014:2120]) is too old
2025-11-19T14:00:05.859886Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435982017061622:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.859919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435982017061531:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.860369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435983738615111:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9014
2025-11-19T14:00:05.860510Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435982431163120:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.860572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435982431163029:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.860669Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435983738615110:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.860697Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435983738615088:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.860701Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435983738615088:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.861350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982431163127:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38683
2025-11-19T14:00:05.861608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435986009536687:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.861623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:05.861636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435986009536687:2050] [node 3] ICP22 created new session: [6:7574435986009536784:2050]
2025-11-19T14:00:05.861642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435986009536784:2050] [node 3] ICS09 handshake done sender: [6:7574435986009536766:2118] self: [6:7574435986009536771:0] peer: [3:7574435985876348960:0] socket: 190 qp: -1
2025-11-19T14:00:05.861648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435986009536784:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:05.861655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:05.862678Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435982431163121:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.862703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435982431163030:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.867954Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435984579361626:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.867973Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435984579361626:2106] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.868835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435982431163129:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38683
2025-11-19T14:00:05.868975Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435983738615088:2112] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:05.869217Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435982431163120:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:05.869306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435982431163029:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.869327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:05.869339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435982431163029:2048] [node 1] ICP22 created new session: [5:7574435982431163130:2048]
2025-11-19T14:00:05.869347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435982431163130:2048] [node 1] ICS09 handshake done sender: [5:7574435982431163120:2119] self: [5:7574435982431163126:0] peer: [1:7574435983738615087:0] socket: 162 qp: -1
2025-11-19T14:00:05.869354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435982431163130:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:05.869364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:05.870296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435983738615084:2094] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.870318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:05.870324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435983738615084:2094] [node 5] ICP22 created new session: [1:7574435983738615114:2094]
2025-11-19T14:00:05.870329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435983738615114:2094] [node 5] ICS09 handshake done sender: [1:7574435983738615088:2112] self: [1:7574435983738615087:0] peer: [5:7574435982431163126:0] socket: 177 qp: -1
2025-11-19T14:00:05.870335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435983738615114:2094] [node 5] ICS10 traffic start
2025-11-19T14:00:05.870343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:05.871149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435985876349000:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.871162Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435985876349000:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.871677Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435985876349013:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.871876Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435984579361626:2106] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:05.872220Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435986009536777:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.872313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435986009536686:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:05.872527Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435984579361624:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.872535Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435984579361624:2105] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:05.873020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435984579361621:2078] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.873035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:05.873042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435984579361621:2078] [node 5] ICP22 created new session: [2:7574435984579361657:2078]
2025-11-19T14:00:05.873048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435984579361657:2078] [node 5] ICS09 handshake done sender: [2:7574435984579361626:2106] self: [2:7574435984579361625:0] peer: [5:7574435982431163128:0] socket: 208 qp: -1
2025-11-19T14:00:05.873054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435984579361657:2078] [node 5] ICS10 traffic start
2025-11-19T14:00:05.873062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:05.873919Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435983738615111:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.874408Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435982431163121:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:05.874585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435982431163030:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.874603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:05.874609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435982431163030:2049] [node 2] ICP22 created new session: [5:7574435982431163132:2049]
2025-11-19T14:00:05.874614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435982431163132:2049] [node 2] ICS09 handshake done sender: [5:7574435982431163121:2120] self: [5:7574435982431163128:0] peer: [2:7574435984579361625:0] socket: 215 qp: -1
2025-11-19T14:00:05.874620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435982431163132:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:05.874628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:05.875904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435985876348924:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.876178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435982017061629:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61484
2025-11-19T14:00:05.876285Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435983738615090:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:05.876296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435983738615090:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.877654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986009536787:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23342
2025-11-19T14:00:05.877792Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435985876349000:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:05.878027Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435982017061622:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:05.878108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435982017061531:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.878124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:05.878131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435982017061531:2050] [node 3] ICP22 created new session: [4:7574435982017061630:2050]
2025-11-19T14:00:05.878138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435982017061630:2050] [node 3] ICS09 handshake done sender: [4:7574435982017061622:2120] self: [4:7574435982017061628:0] peer: [3:7574435985876348999:0] socket: 214 qp: -1
2025-11-19T14:00:05.878144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435982017061630:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:05.878153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:05.879370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435985876348992:2067] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.879393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:05.879400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435985876348992:2067] [node 4] ICP22 created new session: [3:7574435985876349025:2067]
2025-11-19T14:00:05.879407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435985876349025:2067] [node 4] ICS09 handshake done sender: [3:7574435985876349000:2115] self: [3:7574435985876348999:0] peer: [4:7574435982017061628:0] socket: 213 qp: -1
2025-11-19T14:00:05.879413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435985876349025:2067] [node 4] ICS10 traffic start
2025-11-19T14:00:05.879422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:05.880348Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435983738615090:2113] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:05.880477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435983738615017:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.880498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:05.880505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435983738615017:2049] [node 3] ICP22 created new session: [1:7574435983738615118:2049]
2025-11-19T14:00:05.880511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435983738615118:2049] [node 3] ICS09 handshake done sender: [1:7574435983738615090:2113] self: [1:7574435983738615089:0] peer: [3:7574435985876349023:0] socket: 191 qp: -1
2025-11-19T14:00:05.880517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435983738615118:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:05.880527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:05.881464Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435984579361624:2105] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:05.881698Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435986009536777:2122] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:05.881786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435986009536686:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.881802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:05.881807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435986009536686:2049] [node 2] ICP22 created new session: [6:7574435986009536788:2049]
2025-11-19T14:00:05.881813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435986009536788:2049] [node 2] ICS09 handshake done sender: [6:7574435986009536777:2122] self: [6:7574435986009536786:0] peer: [2:7574435984579361623:0] socket: 169 qp: -1
2025-11-19T14:00:05.881819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435986009536788:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:05.881827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:05.882819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435984579361618:2076] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:05.882839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:05.882845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435984579361618:2076] [node 6] ICP22 created new session: [2:7574435984579361660:2076]
2025-11-19T14:00:05.882850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435984579361660:2076] [node 6] ICS09 handshake done sender: [2:7574435984579361624:2105] self: [2:7574435984579361623:0] peer: [6:7574435986009536786:0] socket: 207 qp: -1
2025-11-19T14:00:05.882856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435984579361660:2076] [node 6] ICS10 traffic start
2025-11-19T14:00:05.882864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:05.889155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435985876349024:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59637
2025-11-19T14:00:05.889319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435985876348924:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13859) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435985876349013:2119] held: no
2025-11-19T14:00:05.889480Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435985876349013:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:05.889543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435985876348924:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:05.889560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:05.889566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435985876348924:2048] [node 1] ICP22 created new session: [3:7574435985876349027:2048]
2025-11-19T14:00:05.889572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435985876349027:2048] [node 1] ICS09 handshake done sender: [3:7574435985876349013:2119] self: [3:7574435985876349023:0] peer: [1:7574435983738615089:0] socket: 210 qp: -1
2025-11-19T14:00:05.889579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435985876349027:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:05.889587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:05.894438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:05.894537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435984579361653:2048] [node 1] ICS01 socket: 168 reason# EndOfStream
2025-11-19T14:00:05.894547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435984579361558:2048] [node 1] ICP30 unregister session Session# [2:7574435984579361653:2048] VirtualId# [2:7574435984579361641:0]
2025-11-19T14:00:05.894553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435984579361653:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.894592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:05.894613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:05.895085Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:05.895153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435985876349027:2048] [node 1] ICS01 socket: 210 reason# EndOfStream
2025-11-19T14:00:05.895159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435985876348924:2048] [node 1] ICP30 unregister session Session# [3:7574435985876349027:2048] VirtualId# [3:7574435985876349023:0]
2025-11-19T14:00:05.895165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435985876349027:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.895201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:05.895220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:05.922008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:05.922117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435982017061619:2048] [node 1] ICS01 socket: 178 reason# EndOfStream
2025-11-19T14:00:05.922127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435982017061529:2048] [node 1] ICP30 unregister session Session# [4:7574435982017061619:2048] VirtualId# [4:7574435982017061611:0]
2025-11-19T14:00:05.922135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435982017061619:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.922198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:05.922226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:05.922266Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:05.922336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435986009536781:2048] [node 1] ICS01 socket: 173 reason# EndOfStream
2025-11-19T14:00:05.922341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435986009536685:2048] [node 1] ICP30 unregister session Session# [6:7574435986009536781:2048] VirtualId# [6:7574435986009536770:0]
2025-11-19T14:00:05.922346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435986009536781:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.922376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:05.922400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:05.922432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:05.922497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435982431163130:2048] [node 1] ICS01 socket: 162 reason# EndOfStream
2025-11-19T14:00:05.922502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435982431163029:2048] [node 1] ICP30 unregister session Session# [5:7574435982431163130:2048] VirtualId# [5:7574435982431163126:0]
2025-11-19T14:00:05.922507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435982431163130:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.922534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:05.922557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:05.923963Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:05.924056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435985876349010:2049] [node 2] ICS01 socket: 175 reason# EndOfStream
2025-11-19T14:00:05.924063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435985876348925:2049] [node 2] ICP30 unregister session Session# [3:7574435985876349010:2049] VirtualId# [3:7574435985876349008:0]
2025-11-19T14:00:05.924069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435985876349010:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.924112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:05.924133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:05.935153Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:05.935231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435982017061625:2049] [node 2] ICS01 socket: 179 reason# EndOfStream
2025-11-19T14:00:05.935238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435982017061530:2049] [node 2] ICP30 unregister session Session# [4:7574435982017061625:2049] VirtualId# [4:7574435982017061614:0]
2025-11-19T14:00:05.935243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435982017061625:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.935291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:05.935323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:05.935355Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:05.935407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435986009536788:2049] [node 2] ICS01 socket: 169 reason# EndOfStream
2025-11-19T14:00:05.935412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435986009536686:2049] [node 2] ICP30 unregister session Session# [6:7574435986009536788:2049] VirtualId# [6:7574435986009536786:0]
2025-11-19T14:00:05.935416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435986009536788:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.935441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:05.935456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:05.935484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:05.935561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435982431163132:2049] [node 2] ICS01 socket: 215 reason# EndOfStream
2025-11-19T14:00:05.935566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435982431163030:2049] [node 2] ICP30 unregister session Session# [5:7574435982431163132:2049] VirtualId# [5:7574435982431163128:0]
2025-11-19T14:00:05.935570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435982431163132:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.935596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:05.935611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:05.946060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:05.946229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435982431163124:2050] [node 3] ICS01 socket: 189 reason# EndOfStream
2025-11-19T14:00:05.946239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435982431163031:2050] [node 3] ICP30 unregister session Session# [5:7574435982431163124:2050] VirtualId# [5:7574435982431163117:0]
2025-11-19T14:00:05.946245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435982431163124:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.946310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:05.946334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:05.946377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:05.946438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435986009536784:2050] [node 3] ICS01 socket: 190 reason# EndOfStream
2025-11-19T14:00:05.946444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435986009536687:2050] [node 3] ICP30 unregister session Session# [6:7574435986009536784:2050] VirtualId# [6:7574435986009536771:0]
2025-11-19T14:00:05.946449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435986009536784:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.946476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:05.946495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:05.962556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:05.962649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435986009536779:2086] [node 4] ICS01 socket: 183 reason# EndOfStream
2025-11-19T14:00:05.962657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435986009536756:2086] [node 4] ICP30 unregister session Session# [6:7574435986009536779:2086] VirtualId# [6:7574435986009536769:0]
2025-11-19T14:00:05.962664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435986009536779:2086] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:05.962725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:05.962749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:06.028100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435989173648624:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:06.028115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435989173648625:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:06.063364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435990076985346:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:06.063381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435990076985347:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:06.085275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435989275270863:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:06.085289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435989275270864:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:06.101333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435987134838419:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:06.101354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435987134838420:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:06.101359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435987134838421:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:06.104399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435987898668045:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:06.104407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435987898668046:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:06.104411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435987898668047:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:06.134375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435987869597443:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:06.134417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435987869597445:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:06.148762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435987869597444:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:06.175568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435987898668070:2067] [node 4] ICP01 ready to work
2025-11-19T14:00:06.175679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435987898668071:2071] [node 6] ICP01 ready to work
2025-11-19T14:00:06.176580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435987869597463:2060] [node 4] ICP01 ready to work
2025-11-19T14:00:06.176693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435987869597465:2063] [node 5] ICP01 ready to work
2025-11-19T14:00:06.179601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435990076985408:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:06.179689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435990076985409:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:06.187967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435989275270925:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:06.188080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435989275270926:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:06.195415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435990076985418:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:06.192798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435989173648692:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:06.192969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435989173648693:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:06.196108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435989173648686:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:06.197299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435987134838486:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.203894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435990076985409:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19899236ns) {Configure:104(19897941ns) {ProcessConfigured:135(19493255ns) {StartInitialHandshake:157(19491996ns) {DropHandshakes:469(462ns) {DropIncomingHandshake:441(87ns) DropOutgoingHandshake:456(84ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:476(1647ns) {SwitchToState:178(696ns)}}}}}
2025-11-19T14:00:06.203968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435990076985414:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2603
2025-11-19T14:00:06.203987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435990076985415:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2603
2025-11-19T14:00:06.212279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435987134838481:2107] [node 6] ICP01 ready to work
2025-11-19T14:00:06.213606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435990076985418:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17748682ns) {Configure:104(17747266ns) {ProcessConfigured:135(17545028ns) {StartInitialHandshake:157(17544533ns) {DropHandshakes:469(217ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(13ns)} GenerateSessionVirtualId:988(311ns) PrepareNewSessionHandshake:476(1481ns) {SwitchToState:178(555ns)}}}}}
2025-11-19T14:00:06.215182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435987898668071:2071] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(38897877ns) {Configure:104(38896911ns) {ProcessConfigured:135(38608854ns) {StartInitialHandshake:157(38608704ns) {DropHandshakes:469(400ns) {DropIncomingHandshake:441(94ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:476(614ns) {SwitchToState:178(554ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.216194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435987869597521:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:557
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.216654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435989275270939:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:06.217559Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435990076985414:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.216464Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435989173648686:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(20165084ns) {Configure:104(20164401ns) {ProcessConfigured:135(19848637ns) {StartInitialHandshake:157(19848140ns) {DropHandshakes:469(564ns) {DropIncomingHandshake:441(96ns) DropOutgoingHandshake:456(83ns)} GenerateSessionVirtualId:988(124ns) PrepareNewSessionHandshake:476(1047ns) {SwitchToState:178(417ns)}}}}}
2025-11-19T14:00:06.216619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435989173648703:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53807
2025-11-19T14:00:06.216632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435989173648704:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53807
2025-11-19T14:00:06.216350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435987898668119:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
2025-11-19T14:00:06.216371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435987898668120:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
2025-11-19T14:00:06.216743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435987134838486:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19245954ns) {Configure:104(19244936ns) {ProcessConfigured:135(18916655ns) {StartInitialHandshake:157(18916308ns) {DropHandshakes:469(431ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:476(1272ns) {SwitchToState:178(809ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.227894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435989275270929:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6148
2025-11-19T14:00:06.228067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435989275270930:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6148
2025-11-19T14:00:06.228087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435989275270931:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6148
2025-11-19T14:00:06.228106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435989275270932:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6148
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.232575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435990076985428:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2603
2025-11-19T14:00:06.232686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435989173648705:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53807
2025-11-19T14:00:06.235891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435987898668046:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(55804585ns) {Configure:104(55803745ns) {ProcessConfigured:135(55464892ns) {StartInitialHandshake:157(55463789ns) {DropHandshakes:469(532ns) {DropIncomingHandshake:441(92ns) DropOutgoingHandshake:456(84ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:476(2452ns) {SwitchToState:178(892ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.239871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435987134838420:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(27491563ns) {Configure:104(27489473ns) {ProcessConfigured:135(27064179ns) {StartInitialHandshake:157(27063430ns) {DropHandshakes:469(1279ns) {DropIncomingHandshake:441(265ns) DropOutgoingHandshake:456(163ns)} GenerateSessionVirtualId:988(504ns) PrepareNewSessionHandshake:476(2670ns) {SwitchToState:178(1541ns)}}}}}
2025-11-19T14:00:06.240544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435987134838495:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31055
2025-11-19T14:00:06.240555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435987134838496:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31055
2025-11-19T14:00:06.240053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435987869597522:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:557
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.245898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435989173648706:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53807
2025-11-19T14:00:06.243872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435989275270925:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(55691464ns) {Configure:104(55690262ns) {ProcessConfigured:135(55306425ns) {StartInitialHandshake:157(55305608ns) {DropHandshakes:469(726ns) {DropIncomingHandshake:441(222ns) DropOutgoingHandshake:456(138ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(2295ns) {SwitchToState:178(955ns)}}}}}
2025-11-19T14:00:06.243929Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435989275270929:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.252088Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435990076985415:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.252125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435990076985346:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.252314Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989173648703:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.252345Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989173648689:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.252348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435989173648689:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.252935Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435987869597521:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.252965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435987869597443:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.253181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435990076985430:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2603
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.253254Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435987898668119:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.253447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435987898668125:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
2025-11-19T14:00:06.253629Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435989173648689:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:06.253817Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435990076985415:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:06.253885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435990076985346:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.253902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:06.253908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435990076985346:2048] [node 1] ICP22 created new session: [2:7574435990076985431:2048]
2025-11-19T14:00:06.253914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435990076985431:2048] [node 1] ICS09 handshake done sender: [2:7574435990076985415:2112] self: [2:7574435990076985429:0] peer: [1:7574435989173648684:0] socket: 171 qp: -1
2025-11-19T14:00:06.253922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435990076985431:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:06.253930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:06.254836Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989173648695:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.254844Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435989173648695:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.255276Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989173648704:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.255284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435989173648624:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.255295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:06.255301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435989173648624:2048] [node 2] ICP22 created new session: [1:7574435989173648709:2048]
2025-11-19T14:00:06.255306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435989173648709:2048] [node 2] ICS09 handshake done sender: [1:7574435989173648689:2110] self: [1:7574435989173648684:0] peer: [2:7574435990076985429:0] socket: 166 qp: -1
2025-11-19T14:00:06.255310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435989173648709:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:06.255331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.255949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435987869597524:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:557
2025-11-19T14:00:06.256157Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435989275270931:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.256258Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435989275270930:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.256341Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435989275270932:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.256374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435989275270863:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.256392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435989275270864:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.256585Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435987898668120:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.256613Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435987898668125:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.256630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435987898668046:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.256639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435987898668070:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.256768Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435987869597522:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.256790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435987869597465:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.257634Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435989275270934:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.257641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435989275270934:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.256005Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435987134838495:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.256902Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435987134838496:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.256928Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435987134838489:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.256930Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435987134838489:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.257510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435987134838421:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.258233Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435987898668114:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.258240Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435987898668114:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.258731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435987898668129:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.258869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435987869597526:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:557
2025-11-19T14:00:06.258980Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435989275270934:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:06.259075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435989275270926:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.259092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:06.259097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435989275270926:2108] [node 4] ICP22 created new session: [3:7574435989275270951:2108]
2025-11-19T14:00:06.259103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435989275270951:2108] [node 4] ICS09 handshake done sender: [3:7574435989275270934:2114] self: [3:7574435989275270933:0] peer: [4:7574435987134838501:0] socket: 187 qp: -1
2025-11-19T14:00:06.259108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435989275270951:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:06.259117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:06.259644Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435987898668114:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:06.259820Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435987869597522:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.259994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435987869597465:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.260009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:06.260013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435987869597465:2063] [node 5] ICP22 created new session: [6:7574435987869597527:2063]
2025-11-19T14:00:06.260018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435987869597527:2063] [node 5] ICS09 handshake done sender: [6:7574435987869597522:2116] self: [6:7574435987869597525:0] peer: [5:7574435987898668085:0] socket: 186 qp: -1
2025-11-19T14:00:06.260023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435987869597527:2063] [node 5] ICS10 traffic start
2025-11-19T14:00:06.260029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:06.262642Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435990076985428:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.262677Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435990076985420:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.262679Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435990076985420:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.263022Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435990076985413:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.263024Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435990076985413:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.259573Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435987134838489:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:06.262151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435987134838486:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.262165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:06.262171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435987134838486:2110] [node 5] ICP22 created new session: [4:7574435987134838503:2110]
2025-11-19T14:00:06.262176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435987134838503:2110] [node 5] ICS09 handshake done sender: [4:7574435987134838489:2111] self: [4:7574435987134838487:0] peer: [5:7574435987898668128:0] socket: 194 qp: -1
2025-11-19T14:00:06.262182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435987134838503:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:06.262215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:06.264247Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989173648706:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.264273Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989173648705:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.264294Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989173648691:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.264297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435989173648691:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.264351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435987134838502:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31055
2025-11-19T14:00:06.264616Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435987134838496:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:06.264758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435987134838421:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.264771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:06.264776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435987134838421:2050] [node 3] ICP22 created new session: [4:7574435987134838506:2050]
2025-11-19T14:00:06.264780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435987134838506:2050] [node 3] ICS09 handshake done sender: [4:7574435987134838496:2116] self: [4:7574435987134838501:0] peer: [3:7574435989275270933:0] socket: 193 qp: -1
2025-11-19T14:00:06.264785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435987134838506:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:06.264791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.266538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435987869597530:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:557
2025-11-19T14:00:06.266457Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435989173648695:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.266739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435989173648692:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.266750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:06.266754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435989173648692:2108] [node 6] ICP22 created new session: [1:7574435989173648716:2108]
2025-11-19T14:00:06.266759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435989173648716:2108] [node 6] ICS09 handshake done sender: [1:7574435989173648695:2112] self: [1:7574435989173648694:0] peer: [6:7574435987869597523:0] socket: 168 qp: -1
2025-11-19T14:00:06.266763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435989173648716:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:06.266769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:06.267029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435989173648714:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53807
2025-11-19T14:00:06.267334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435987898668071:2071] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.267356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:06.267361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435987898668071:2071] [node 6] ICP22 created new session: [5:7574435987898668131:2071]
2025-11-19T14:00:06.267367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435987898668131:2071] [node 6] ICS09 handshake done sender: [5:7574435987898668114:2111] self: [5:7574435987898668085:0] peer: [6:7574435987869597525:0] socket: 182 qp: -1
2025-11-19T14:00:06.267372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435987898668131:2071] [node 6] ICS10 traffic start
2025-11-19T14:00:06.267378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:06.268062Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435987898668125:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:06.268076Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435989173648691:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.268883Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435987869597521:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:06.268970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435987869597443:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.268982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:06.268988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435987869597443:2048] [node 1] ICP22 created new session: [6:7574435987869597531:2048]
2025-11-19T14:00:06.268994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435987869597531:2048] [node 1] ICS09 handshake done sender: [6:7574435987869597521:2115] self: [6:7574435987869597523:0] peer: [1:7574435989173648694:0] socket: 172 qp: -1
2025-11-19T14:00:06.268999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435987869597531:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:06.269005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:06.268276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435987898668070:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.268287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:06.268291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435987898668070:2067] [node 4] ICP22 created new session: [5:7574435987898668135:2067]
2025-11-19T14:00:06.268297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435987898668135:2067] [node 4] ICS09 handshake done sender: [5:7574435987898668125:2117] self: [5:7574435987898668128:0] peer: [4:7574435987134838487:0] socket: 199 qp: -1
2025-11-19T14:00:06.268301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435987898668135:2067] [node 4] ICS10 traffic start
2025-11-19T14:00:06.268305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:06.268653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435987898668133:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.269935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435989275270954:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6148
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.270100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435989275270955:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6148
2025-11-19T14:00:06.270117Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435989275270930:2111] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:06.270177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435989275270863:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.270190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:06.270195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435989275270863:2048] [node 1] ICP22 created new session: [3:7574435989275270956:2048]
2025-11-19T14:00:06.270199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435989275270956:2048] [node 1] ICS09 handshake done sender: [3:7574435989275270930:2111] self: [3:7574435989275270948:0] peer: [1:7574435989173648690:0] socket: 176 qp: -1
2025-11-19T14:00:06.270204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435989275270956:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:06.270209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.272979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435987134838508:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31055
2025-11-19T14:00:06.273191Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989173648714:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.273206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435989173648625:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.273221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:06.273227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435989173648625:2049] [node 3] ICP22 created new session: [1:7574435989173648721:2049]
2025-11-19T14:00:06.273232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435989173648721:2049] [node 3] ICS09 handshake done sender: [1:7574435989173648691:2111] self: [1:7574435989173648690:0] peer: [3:7574435989275270948:0] socket: 167 qp: -1
2025-11-19T14:00:06.273236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435989173648721:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:06.273244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.275204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435987869597533:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:557
2025-11-19T14:00:06.277740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435989173648692:2108] [node 6] ICP18 handshake (actor [1:7574435989173648714:2119]) is too old
2025-11-19T14:00:06.278169Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435987869597530:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.279420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435987869597445:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.280771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435987898668138:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
2025-11-19T14:00:06.281003Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435989275270941:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.281014Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435989275270941:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.282445Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435987898668138:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.282479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435987898668047:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.283150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435987869597536:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:557
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.283294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435990076985436:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2603
2025-11-19T14:00:06.283485Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435989275270941:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:06.283445Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435987869597533:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.283475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435987869597463:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.283648Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435987869597530:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.283676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435990076985437:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2603
2025-11-19T14:00:06.283764Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435990076985420:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:06.283664Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435987898668133:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.283771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435987898668045:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.283792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435987898668140:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
2025-11-19T14:00:06.284011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435987869597445:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.284045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:06.284051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435987869597445:2050] [node 3] ICP22 created new session: [6:7574435987869597538:2050]
2025-11-19T14:00:06.284057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435987869597538:2050] [node 3] ICS09 handshake done sender: [6:7574435987869597530:2119] self: [6:7574435987869597535:0] peer: [3:7574435989275270940:0] socket: 174 qp: -1
2025-11-19T14:00:06.284063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435987869597538:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:06.284072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:06.284636Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435990076985413:2110] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:06.284742Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435990076985437:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.284762Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435990076985436:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.284768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435990076985347:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.284780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:06.284784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435990076985347:2049] [node 3] ICP22 created new session: [2:7574435990076985439:2049]
2025-11-19T14:00:06.284789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435990076985439:2049] [node 3] ICS09 handshake done sender: [2:7574435990076985420:2114] self: [2:7574435990076985419:0] peer: [3:7574435989275270949:0] socket: 175 qp: -1
2025-11-19T14:00:06.284793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435990076985439:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:06.284800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:06.284997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435990076985408:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.285002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:06.285006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435990076985408:2107] [node 5] ICP22 created new session: [2:7574435990076985441:2107]
2025-11-19T14:00:06.285010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435990076985441:2107] [node 5] ICS09 handshake done sender: [2:7574435990076985413:2110] self: [2:7574435990076985410:0] peer: [5:7574435987898668127:0] socket: 173 qp: -1
2025-11-19T14:00:06.285014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435990076985441:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:06.285019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:06.285065Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435989275270945:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.285076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435989275270945:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.285004Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435987134838494:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.285020Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435987134838494:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.285150Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989173648697:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.285161Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435989173648697:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.285815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435989275270939:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.285831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:06.285836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435989275270939:2109] [node 6] ICP22 created new session: [3:7574435989275270961:2109]
2025-11-19T14:00:06.285841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435989275270961:2109] [node 6] ICS09 handshake done sender: [3:7574435989275270941:2117] self: [3:7574435989275270940:0] peer: [6:7574435987869597535:0] socket: 164 qp: -1
2025-11-19T14:00:06.285846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435989275270961:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:06.285852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:06.286422Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435987134838508:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.286469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435987134838510:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31055
2025-11-19T14:00:06.286509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435987134838511:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31055
2025-11-19T14:00:06.286523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435987134838419:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.286826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435989275270960:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6148
2025-11-19T14:00:06.286945Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435989275270932:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:06.287001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435989275270864:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.287012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:06.287017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435989275270864:2049] [node 2] ICP22 created new session: [3:7574435989275270963:2049]
2025-11-19T14:00:06.287021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435989275270963:2049] [node 2] ICS09 handshake done sender: [3:7574435989275270932:2113] self: [3:7574435989275270949:0] peer: [2:7574435990076985419:0] socket: 178 qp: -1
2025-11-19T14:00:06.287026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435989275270963:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:06.287031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:06.288667Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435987898668120:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.288759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435987898668142:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
2025-11-19T14:00:06.288854Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435989173648697:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:06.289017Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435987898668133:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:06.289176Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989173648702:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.289184Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435989173648702:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.290071Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435989173648702:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.290143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435987898668046:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.290162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:06.290177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435987898668046:2049] [node 2] ICP22 created new session: [5:7574435987898668143:2049]
2025-11-19T14:00:06.290182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435987898668143:2049] [node 2] ICS09 handshake done sender: [5:7574435987898668120:2115] self: [5:7574435987898668127:0] peer: [2:7574435990076985410:0] socket: 180 qp: -1
2025-11-19T14:00:06.290188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435987898668143:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:06.290195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:06.291042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435987134838513:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31055
2025-11-19T14:00:06.291280Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435987134838508:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:06.291350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435987134838419:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.291364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:06.291372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435987134838419:2048] [node 1] ICP22 created new session: [4:7574435987134838514:2048]
2025-11-19T14:00:06.291378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435987134838514:2048] [node 1] ICS09 handshake done sender: [4:7574435987134838508:2118] self: [4:7574435987134838512:0] peer: [1:7574435989173648701:0] socket: 216 qp: -1
2025-11-19T14:00:06.291383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435987134838514:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:06.291389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:06.292868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435987898668045:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.292885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:06.292893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435987898668045:2048] [node 1] ICP22 created new session: [5:7574435987898668145:2048]
2025-11-19T14:00:06.292899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435987898668145:2048] [node 1] ICS09 handshake done sender: [5:7574435987898668133:2119] self: [5:7574435987898668141:0] peer: [1:7574435989173648696:0] socket: 213 qp: -1
2025-11-19T14:00:06.292905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435987898668145:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:06.292913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:06.295380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435990076985408:2107] [node 5] ICP18 handshake (actor [2:7574435990076985437:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.296690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435987869597540:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:557
2025-11-19T14:00:06.296065Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435987134838510:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.296881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435987869597541:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:557
2025-11-19T14:00:06.297107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435989173648693:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.297128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:06.297134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435989173648693:2109] [node 5] ICP22 created new session: [1:7574435989173648725:2109]
2025-11-19T14:00:06.297141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435989173648725:2109] [node 5] ICS09 handshake done sender: [1:7574435989173648697:2113] self: [1:7574435989173648696:0] peer: [5:7574435987898668141:0] socket: 210 qp: -1
2025-11-19T14:00:06.297148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435989173648725:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:06.297158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:06.297471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435989173648686:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.297481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:06.297484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435989173648686:2107] [node 4] ICP22 created new session: [1:7574435989173648727:2107]
2025-11-19T14:00:06.297488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435989173648727:2107] [node 4] ICS09 handshake done sender: [1:7574435989173648702:2114] self: [1:7574435989173648701:0] peer: [4:7574435987134838512:0] socket: 212 qp: -1
2025-11-19T14:00:06.297491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435989173648727:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:06.297495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:06.297308Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435989275270955:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.298419Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435989275270945:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:06.298740Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435987134838494:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:06.298882Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435987869597533:2120] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:06.298552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435989275270925:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.298568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:06.298582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435989275270925:2107] [node 5] ICP22 created new session: [3:7574435989275270967:2107]
2025-11-19T14:00:06.298594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435989275270967:2107] [node 5] ICS09 handshake done sender: [3:7574435989275270945:2118] self: [3:7574435989275270927:0] peer: [5:7574435987898668139:0] socket: 218 qp: -1
2025-11-19T14:00:06.298598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435989275270967:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:06.298606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:06.298979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435987869597463:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.299001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:06.299007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435987869597463:2060] [node 4] ICP22 created new session: [6:7574435987869597542:2060]
2025-11-19T14:00:06.299013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435987869597542:2060] [node 4] ICS09 handshake done sender: [6:7574435987869597533:2120] self: [6:7574435987869597537:0] peer: [4:7574435987134838493:0] socket: 177 qp: -1
2025-11-19T14:00:06.299024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435987869597542:2060] [node 4] ICS10 traffic start
2025-11-19T14:00:06.299032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:06.299119Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435987134838511:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.299289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435987134838481:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.299299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:06.299304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435987134838481:2107] [node 6] ICP22 created new session: [4:7574435987134838518:2107]
2025-11-19T14:00:06.299309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435987134838518:2107] [node 6] ICS09 handshake done sender: [4:7574435987134838494:2114] self: [4:7574435987134838493:0] peer: [6:7574435987869597537:0] socket: 217 qp: -1
2025-11-19T14:00:06.299326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435987134838518:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:06.299332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:06.300054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435987134838420:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.300368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435987898668147:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14121
2025-11-19T14:00:06.300680Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435990076985422:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.300691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435990076985422:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.302674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435987898668047:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1048) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435987898668138:2120] held: no
2025-11-19T14:00:06.302826Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435987898668138:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:06.303066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435987898668047:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.303081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:06.303088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435987898668047:2050] [node 3] ICP22 created new session: [5:7574435987898668148:2050]
2025-11-19T14:00:06.303093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435987898668148:2050] [node 3] ICS09 handshake done sender: [5:7574435987898668138:2120] self: [5:7574435987898668139:0] peer: [3:7574435989275270927:0] socket: 219 qp: -1
2025-11-19T14:00:06.303099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435987898668148:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:06.303106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.304113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435987134838521:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31055
2025-11-19T14:00:06.304423Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435990076985422:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:06.304650Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435987134838511:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:06.304740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435987134838420:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.304757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:06.304762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435987134838420:2049] [node 2] ICP22 created new session: [4:7574435987134838522:2049]
2025-11-19T14:00:06.304768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435987134838522:2049] [node 2] ICS09 handshake done sender: [4:7574435987134838511:2120] self: [4:7574435987134838520:0] peer: [2:7574435990076985411:0] socket: 224 qp: -1
2025-11-19T14:00:06.304774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435987134838522:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:06.304783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:06.306132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435990076985409:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.306153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:06.306159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435990076985409:2108] [node 4] ICP22 created new session: [2:7574435990076985446:2108]
2025-11-19T14:00:06.306165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435990076985446:2108] [node 4] ICS09 handshake done sender: [2:7574435990076985422:2115] self: [2:7574435990076985411:0] peer: [4:7574435987134838520:0] socket: 209 qp: -1
2025-11-19T14:00:06.306170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435990076985446:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:06.306177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:06.307343Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435987869597541:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.307391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435987869597444:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.307508Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435990076985426:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.307516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435990076985426:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.308427Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435990076985426:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:06.308537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435990076985418:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.308553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:06.308561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435990076985418:2109] [node 6] ICP22 created new session: [2:7574435990076985449:2109]
2025-11-19T14:00:06.308566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435990076985449:2109] [node 6] ICS09 handshake done sender: [2:7574435990076985426:2116] self: [2:7574435990076985421:0] peer: [6:7574435987869597544:0] socket: 190 qp: -1
2025-11-19T14:00:06.308570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435990076985449:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:06.308576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.311868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435987869597444:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11018) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435987869597541:2123] held: no
2025-11-19T14:00:06.312068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435987869597545:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:557
2025-11-19T14:00:06.312237Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435987869597541:2123] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:06.312297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435987869597444:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.312310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:06.312316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435987869597444:2049] [node 2] ICP22 created new session: [6:7574435987869597546:2049]
2025-11-19T14:00:06.312322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435987869597546:2049] [node 2] ICS09 handshake done sender: [6:7574435987869597541:2123] self: [6:7574435987869597544:0] peer: [2:7574435990076985421:0] socket: 230 qp: -1
2025-11-19T14:00:06.312329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435987869597546:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:06.312336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:06.317196Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:06.317282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435990076985431:2048] [node 1] ICS01 socket: 171 reason# EndOfStream
2025-11-19T14:00:06.317290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435990076985346:2048] [node 1] ICP30 unregister session Session# [2:7574435990076985431:2048] VirtualId# [2:7574435990076985429:0]
2025-11-19T14:00:06.317296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435990076985431:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.317326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:06.317345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:06.317714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:06.317775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435989275270956:2048] [node 1] ICS01 socket: 176 reason# EndOfStream
2025-11-19T14:00:06.317780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435989275270863:2048] [node 1] ICP30 unregister session Session# [3:7574435989275270956:2048] VirtualId# [3:7574435989275270948:0]
2025-11-19T14:00:06.317785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435989275270956:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.317808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:06.317832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:06.329665Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:06.329756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435987869597531:2048] [node 1] ICS01 socket: 172 reason# EndOfStream
2025-11-19T14:00:06.329764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435987869597443:2048] [node 1] ICP30 unregister session Session# [6:7574435987869597531:2048] VirtualId# [6:7574435987869597523:0]
2025-11-19T14:00:06.329770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435987869597531:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.329830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:06.329856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:06.329895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:06.329955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435987898668145:2048] [node 1] ICS01 socket: 213 reason# EndOfStream
2025-11-19T14:00:06.329958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435987898668045:2048] [node 1] ICP30 unregister session Session# [5:7574435987898668145:2048] VirtualId# [5:7574435987898668141:0]
2025-11-19T14:00:06.329963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435987898668145:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.329986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:06.330008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:06.330120Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:06.330180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435987134838514:2048] [node 1] ICS01 socket: 216 reason# EndOfStream
2025-11-19T14:00:06.330185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435987134838419:2048] [node 1] ICP30 unregister session Session# [4:7574435987134838514:2048] VirtualId# [4:7574435987134838512:0]
2025-11-19T14:00:06.330188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435987134838514:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.330212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:06.330233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:06.340514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:06.340600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435989275270963:2049] [node 2] ICS01 socket: 178 reason# EndOfStream
2025-11-19T14:00:06.340609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435989275270864:2049] [node 2] ICP30 unregister session Session# [3:7574435989275270963:2049] VirtualId# [3:7574435989275270949:0]
2025-11-19T14:00:06.340615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435989275270963:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.340648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:06.340668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:06.347144Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:06.347231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435987134838522:2049] [node 2] ICS01 socket: 224 reason# EndOfStream
2025-11-19T14:00:06.347239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435987134838420:2049] [node 2] ICP30 unregister session Session# [4:7574435987134838522:2049] VirtualId# [4:7574435987134838520:0]
2025-11-19T14:00:06.347246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435987134838522:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.347290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:06.347312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:06.347361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:06.347421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435987869597546:2049] [node 2] ICS01 socket: 230 reason# EndOfStream
2025-11-19T14:00:06.347426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435987869597444:2049] [node 2] ICP30 unregister session Session# [6:7574435987869597546:2049] VirtualId# [6:7574435987869597544:0]
2025-11-19T14:00:06.347431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435987869597546:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.347463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:06.347483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:06.347595Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:06.347651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435987898668143:2049] [node 2] ICS01 socket: 180 reason# EndOfStream
2025-11-19T14:00:06.347655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435987898668046:2049] [node 2] ICP30 unregister session Session# [5:7574435987898668143:2049] VirtualId# [5:7574435987898668127:0]
2025-11-19T14:00:06.347659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435987898668143:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.347685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:06.347704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:06.362003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:06.362112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435987869597538:2050] [node 3] ICS01 socket: 174 reason# EndOfStream
2025-11-19T14:00:06.362121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435987869597445:2050] [node 3] ICP30 unregister session Session# [6:7574435987869597538:2050] VirtualId# [6:7574435987869597535:0]
2025-11-19T14:00:06.362128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435987869597538:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.362192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:06.362216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:06.362312Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:06.362373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435987898668148:2050] [node 3] ICS01 socket: 219 reason# EndOfStream
2025-11-19T14:00:06.362378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435987898668047:2050] [node 3] ICP30 unregister session Session# [5:7574435987898668148:2050] VirtualId# [5:7574435987898668139:0]
2025-11-19T14:00:06.362382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435987898668148:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.362407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:06.362427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:06.378142Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:06.378244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435987869597542:2060] [node 4] ICS01 socket: 177 reason# EndOfStream
2025-11-19T14:00:06.378253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435987869597463:2060] [node 4] ICP30 unregister session Session# [6:7574435987869597542:2060] VirtualId# [6:7574435987869597537:0]
2025-11-19T14:00:06.378261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435987869597542:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.378322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:06.378349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:06.503341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435989647841028:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:06.503352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435989647841029:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:06.544171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435987955330007:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:06.544186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435987955330008:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:06.576218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435988855751260:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:06.576233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435988855751261:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:06.620296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435988039304102:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:06.620318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435988039304103:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:06.620323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435988039304104:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:06.635902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435988635446473:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:06.635925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435988635446474:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:06.635970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435988635446475:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:06.688501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435986477641758:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:06.694989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435986477641759:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:06.696405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435989647841091:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:06.699397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435988635446495:2068] [node 6] ICP01 ready to work
2025-11-19T14:00:06.700970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435989647841097:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:06.700156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435986477641760:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:06.700353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435986477641788:2063] [node 4] ICP01 ready to work
2025-11-19T14:00:06.700451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435986477641789:2071] [node 5] ICP01 ready to work
2025-11-19T14:00:06.701944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435988855751321:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:06.704016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435989647841090:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.707686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986477641821:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25138
2025-11-19T14:00:06.708523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435988039304165:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:06.706127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435987955330073:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:06.707762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435987955330080:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50292
2025-11-19T14:00:06.708393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435987955330068:2107] [node 6] ICP01 ready to work
2025-11-19T14:00:06.708544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435987955330082:2113] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.712682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435988635446533:2076] [node 4] ICP01 ready to work
2025-11-19T14:00:06.713439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435988635446534:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62034
2025-11-19T14:00:06.713460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435988635446535:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62034
2025-11-19T14:00:06.713471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435988635446536:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62034
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.714983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435988855751331:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:06.715082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435988855751332:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:06.715741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435988855751328:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-19T14:00:06.715808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435988855751329:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-19T14:00:06.715825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435988855751330:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-19T14:00:06.717847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435988635446549:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62034
2025-11-19T14:00:06.723489Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435988855751330:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.723695Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435988635446549:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.723732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435988635446475:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.731357Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435986477641759:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(36263033ns) {Configure:104(36262085ns) {ProcessConfigured:135(35809684ns) {StartInitialHandshake:157(35809027ns) {DropHandshakes:469(1087ns) {DropIncomingHandshake:441(117ns) DropOutgoingHandshake:456(146ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:476(1398ns) {SwitchToState:178(1006ns)}}}}}
2025-11-19T14:00:06.738003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435987955330068:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(29392941ns) {Configure:104(29392136ns) {ProcessConfigured:135(29070515ns) {StartInitialHandshake:157(29069879ns) {DropHandshakes:469(386ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:476(1221ns) {SwitchToState:178(467ns)}}}}}
2025-11-19T14:00:06.741401Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435987955330082:2113] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(32712494ns) {Configure:104(32709965ns) {ProcessConfigured:135(32444544ns) {StartInitialHandshake:157(32443687ns) {DropHandshakes:469(304ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(147ns) PrepareNewSessionHandshake:476(882ns) {SwitchToState:178(486ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.741635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435988039304165:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(33048123ns) {Configure:104(33047559ns) {ProcessConfigured:135(32691743ns) {StartInitialHandshake:157(32691319ns) {DropHandshakes:469(524ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(96ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:476(1057ns) {SwitchToState:178(390ns)}}}}}
2025-11-19T14:00:06.742492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435988039304170:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48994
2025-11-19T14:00:06.742556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435988039304171:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48994
2025-11-19T14:00:06.743615Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435986477641821:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.743688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435986477641758:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.744886Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435988855751327:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.744905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435988855751327:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.745895Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435988039304170:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.747049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435988635446535:2104] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.747093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435988635446474:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.751877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435989647841097:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(50794084ns) {Configure:104(50792929ns) {ProcessConfigured:135(50461847ns) {StartInitialHandshake:157(50460995ns) {DropHandshakes:469(557ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(94ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:476(1302ns) {SwitchToState:178(927ns)}}}}}
2025-11-19T14:00:06.752105Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989647841093:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.752112Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435989647841093:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.755692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435988635446557:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62034
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.756046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435988855751342:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-19T14:00:06.756671Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435987955330075:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.756694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435987955330075:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.756909Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435987955330080:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.757027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435987955330007:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.757243Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435988855751328:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.758382Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435989647841093:2109] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:06.759850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435988039304168:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:06.759961Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435988039304171:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.765587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986477641838:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25138
2025-11-19T14:00:06.765665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986477641839:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25138
2025-11-19T14:00:06.765705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986477641840:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25138
2025-11-19T14:00:06.766156Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435986477641821:2103] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:06.766278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435986477641758:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.766298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:06.766307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435986477641758:2048] [node 1] ICP22 created new session: [6:7574435986477641841:2048]
2025-11-19T14:00:06.766314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435986477641841:2048] [node 1] ICS09 handshake done sender: [6:7574435986477641821:2103] self: [6:7574435986477641837:0] peer: [1:7574435989647841092:0] socket: 174 qp: -1
2025-11-19T14:00:06.766323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435986477641841:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:06.766359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:06.768003Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435986477641838:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.768114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435986477641789:2071] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.767456Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989647841099:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.767471Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435989647841099:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.767866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435989647841091:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.767877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:06.767884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435989647841091:2108] [node 6] ICP22 created new session: [1:7574435989647841110:2108]
2025-11-19T14:00:06.767890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435989647841110:2108] [node 6] ICS09 handshake done sender: [1:7574435989647841093:2109] self: [1:7574435989647841092:0] peer: [6:7574435986477641837:0] socket: 168 qp: -1
2025-11-19T14:00:06.767897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435989647841110:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:06.767917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:06.768256Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435988635446532:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.768266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435988635446532:2102] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.769184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986477641845:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25138
2025-11-19T14:00:06.769304Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435988635446532:2102] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.769493Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435986477641838:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.769515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435987955330088:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50292
2025-11-19T14:00:06.769548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435987955330089:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50292
2025-11-19T14:00:06.769588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435986477641789:2071] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.769602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:06.769607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435986477641789:2071] [node 5] ICP22 created new session: [6:7574435986477641846:2071]
2025-11-19T14:00:06.769613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435986477641846:2071] [node 5] ICS09 handshake done sender: [6:7574435986477641838:2116] self: [6:7574435986477641844:0] peer: [5:7574435988635446499:0] socket: 199 qp: -1
2025-11-19T14:00:06.769619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435986477641846:2071] [node 5] ICS10 traffic start
2025-11-19T14:00:06.769625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:06.770420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435988635446495:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.770443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:06.770449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435988635446495:2068] [node 6] ICP22 created new session: [5:7574435988635446559:2068]
2025-11-19T14:00:06.770454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435988635446559:2068] [node 6] ICS09 handshake done sender: [5:7574435988635446532:2102] self: [5:7574435988635446499:0] peer: [6:7574435986477641844:0] socket: 183 qp: -1
2025-11-19T14:00:06.770459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435988635446559:2068] [node 6] ICS10 traffic start
2025-11-19T14:00:06.770465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.772374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435989647841109:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16890
2025-11-19T14:00:06.772545Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435986477641839:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.772617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435986477641760:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.774459Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435989647841099:2111] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:06.775791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435987955330091:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50292
2025-11-19T14:00:06.775930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435987955330090:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50292
2025-11-19T14:00:06.775389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435989647841028:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.775408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:06.775415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435989647841028:2048] [node 2] ICP22 created new session: [1:7574435989647841114:2048]
2025-11-19T14:00:06.775422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435989647841114:2048] [node 2] ICS09 handshake done sender: [1:7574435989647841099:2111] self: [1:7574435989647841095:0] peer: [2:7574435987955330087:0] socket: 169 qp: -1
2025-11-19T14:00:06.775428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435989647841114:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:06.775436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:06.776634Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435988635446534:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.776673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435988635446473:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.776812Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435988855751336:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.776828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435988855751336:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.778284Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435988855751327:2110] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:06.777977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435989647841108:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16890
2025-11-19T14:00:06.778120Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989647841101:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.778124Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435989647841101:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.778734Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435988855751336:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:06.777828Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435987955330091:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.777971Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435987955330075:2110] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:06.779994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435987955330073:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.780012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:06.780021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435987955330073:2108] [node 5] ICP22 created new session: [2:7574435987955330094:2108]
2025-11-19T14:00:06.780028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435987955330094:2108] [node 5] ICS09 handshake done sender: [2:7574435987955330075:2110] self: [2:7574435987955330074:0] peer: [5:7574435988635446556:0] socket: 172 qp: -1
2025-11-19T14:00:06.780039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435987955330094:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:06.780052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:06.780074Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435987955330080:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:06.780471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435987955330007:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.780486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:06.780490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435987955330007:2048] [node 1] ICP22 created new session: [2:7574435987955330096:2048]
2025-11-19T14:00:06.780494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435987955330096:2048] [node 1] ICS09 handshake done sender: [2:7574435987955330080:2112] self: [2:7574435987955330087:0] peer: [1:7574435989647841095:0] socket: 175 qp: -1
2025-11-19T14:00:06.780499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435987955330096:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:06.780505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:06.781572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435988039304168:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21482153ns) {Configure:104(21480737ns) {ProcessConfigured:135(21134816ns) {StartInitialHandshake:157(21133276ns) {DropHandshakes:469(441ns) {DropIncomingHandshake:441(43ns) DropOutgoingHandshake:456(138ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(1437ns) {SwitchToState:178(75ns)}}}}}
2025-11-19T14:00:06.781306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435988855751261:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(52791345ns) {Configure:104(52790233ns) {ProcessConfigured:135(52434529ns) {StartInitialHandshake:157(52433956ns) {DropHandshakes:469(512ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(81ns)} GenerateSessionVirtualId:988(280ns) PrepareNewSessionHandshake:476(1293ns) {SwitchToState:178(605ns)}}}}}
2025-11-19T14:00:06.781340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435988855751321:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.781357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:06.781365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435988855751321:2107] [node 5] ICP22 created new session: [3:7574435988855751346:2107]
2025-11-19T14:00:06.781373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435988855751346:2107] [node 5] ICS09 handshake done sender: [3:7574435988855751327:2110] self: [3:7574435988855751322:0] peer: [5:7574435988635446551:0] socket: 189 qp: -1
2025-11-19T14:00:06.781383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435988855751346:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:06.781407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:06.781611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435988855751261:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.782795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435988635446475:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21013) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435988635446549:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.783133Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989647841108:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.783600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435989647841028:2048] [node 2] ICP18 handshake (actor [1:7574435989647841108:2115]) is too old
2025-11-19T14:00:06.783629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435989647841117:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16890
2025-11-19T14:00:06.783644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435989647841118:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16890
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.781804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435988855751332:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.781816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:06.781821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435988855751332:2109] [node 6] ICP22 created new session: [3:7574435988855751349:2109]
2025-11-19T14:00:06.781826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435988855751349:2109] [node 6] ICS09 handshake done sender: [3:7574435988855751336:2115] self: [3:7574435988855751335:0] peer: [6:7574435986477641849:0] socket: 191 qp: -1
2025-11-19T14:00:06.781831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435988855751349:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:06.781837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.784746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986477641850:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25138
2025-11-19T14:00:06.784176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435988635446564:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62034
2025-11-19T14:00:06.784577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435988635446565:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62034
2025-11-19T14:00:06.785120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435988039304102:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(63665679ns) {Configure:104(63663901ns) {ProcessConfigured:135(63298494ns) {StartInitialHandshake:157(63298054ns) {DropHandshakes:469(517ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(148ns)} GenerateSessionVirtualId:988(359ns) PrepareNewSessionHandshake:476(1065ns) {SwitchToState:178(755ns)}}}}}
2025-11-19T14:00:06.785171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435988039304104:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.785928Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435988635446549:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.787344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986477641851:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25138
2025-11-19T14:00:06.787547Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435987955330072:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.787563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435987955330072:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.788401Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435986477641839:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.788834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435988635446475:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.788853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:06.788871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435988635446475:2050] [node 3] ICP22 created new session: [5:7574435988635446567:2050]
2025-11-19T14:00:06.788876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435988635446567:2050] [node 3] ICS09 handshake done sender: [5:7574435988635446549:2113] self: [5:7574435988635446551:0] peer: [3:7574435988855751322:0] socket: 192 qp: -1
2025-11-19T14:00:06.788882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435988635446567:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:06.788890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:06.788988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435988635446566:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62034
2025-11-19T14:00:06.785305Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435989647841101:2112] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:06.785438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435989647841090:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.785450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:06.785456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435989647841090:2107] [node 5] ICP22 created new session: [1:7574435989647841120:2107]
2025-11-19T14:00:06.785463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435989647841120:2107] [node 5] ICS09 handshake done sender: [1:7574435989647841101:2112] self: [1:7574435989647841100:0] peer: [5:7574435988635446562:0] socket: 170 qp: -1
2025-11-19T14:00:06.785469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435989647841120:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:06.785477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:06.788682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435986477641760:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.788700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:06.788707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435986477641760:2050] [node 3] ICP22 created new session: [6:7574435986477641852:2050]
2025-11-19T14:00:06.788714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435986477641852:2050] [node 3] ICS09 handshake done sender: [6:7574435986477641839:2117] self: [6:7574435986477641849:0] peer: [3:7574435988855751335:0] socket: 203 qp: -1
2025-11-19T14:00:06.788721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435986477641852:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:06.788728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:06.786033Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989647841118:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.786058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435989647841090:2107] [node 5] ICP18 handshake (actor [1:7574435989647841118:2118]) is too old
2025-11-19T14:00:06.789160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435988635446473:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64065) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435988635446534:2103] held: no
2025-11-19T14:00:06.789786Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435988635446474:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29892) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435988635446535:2104] held: no
2025-11-19T14:00:06.789960Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435988635446535:2104] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:06.790105Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435988635446534:2103] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:06.790158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435988635446474:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.790171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:06.790176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435988635446474:2049] [node 2] ICP22 created new session: [5:7574435988635446569:2049]
2025-11-19T14:00:06.790182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435988635446569:2049] [node 2] ICS09 handshake done sender: [5:7574435988635446535:2104] self: [5:7574435988635446556:0] peer: [2:7574435987955330074:0] socket: 181 qp: -1
2025-11-19T14:00:06.790187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435988635446569:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:06.790206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:06.790361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435988635446473:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.790366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:06.790370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435988635446473:2048] [node 1] ICP22 created new session: [5:7574435988635446571:2048]
2025-11-19T14:00:06.790374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435988635446571:2048] [node 1] ICS09 handshake done sender: [5:7574435988635446534:2103] self: [5:7574435988635446562:0] peer: [1:7574435989647841100:0] socket: 180 qp: -1
2025-11-19T14:00:06.790378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435988635446571:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:06.790383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.787823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435988039304184:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48994
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.791465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435988855751353:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-19T14:00:06.791560Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435988855751334:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.791564Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435988855751334:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.799452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435987955330101:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50292
2025-11-19T14:00:06.799826Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435987955330072:2109] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:06.799949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435987955330008:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.799968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:06.799975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435987955330008:2049] [node 3] ICP22 created new session: [2:7574435987955330102:2049]
2025-11-19T14:00:06.799982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435987955330102:2049] [node 3] ICS09 handshake done sender: [2:7574435987955330072:2109] self: [2:7574435987955330070:0] peer: [3:7574435988855751348:0] socket: 171 qp: -1
2025-11-19T14:00:06.799988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435987955330102:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:06.799995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:06.801323Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435986477641851:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.801368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435986477641759:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.800700Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435988855751334:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:06.792370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435988039304185:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48994
2025-11-19T14:00:06.792413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435988039304186:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48994
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.802739Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989647841117:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.802042Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435988855751353:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.802183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435988855751331:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.802193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:06.802200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435988855751331:2108] [node 4] ICP22 created new session: [3:7574435988855751356:2108]
2025-11-19T14:00:06.802205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435988855751356:2108] [node 4] ICS09 handshake done sender: [3:7574435988855751334:2114] self: [3:7574435988855751333:0] peer: [4:7574435988039304183:0] socket: 190 qp: -1
2025-11-19T14:00:06.802212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435988855751356:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:06.802219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:06.802466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435988855751355:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.803001Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435988039304185:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.803155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435988039304187:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48994
2025-11-19T14:00:06.803108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435988855751260:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.801948Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435987955330090:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.802519Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435987955330101:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.802540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435987955330008:2049] [node 3] ICP18 handshake (actor [2:7574435987955330101:2120]) is too old
2025-11-19T14:00:06.803948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435988855751261:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29892) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435988855751328:2111] held: no
2025-11-19T14:00:06.804091Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435988855751328:2111] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:06.804083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435988039304102:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.804187Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435988039304184:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.804214Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435988039304186:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.804280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435988039304103:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.804548Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435988039304171:2112] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:06.804209Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989647841103:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.804232Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435989647841103:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.805098Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435989647841103:2113] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:06.805302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435989647841029:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.805324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:06.805332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435989647841029:2049] [node 3] ICP22 created new session: [1:7574435989647841124:2049]
2025-11-19T14:00:06.805338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435989647841124:2049] [node 3] ICS09 handshake done sender: [1:7574435989647841103:2113] self: [1:7574435989647841102:0] peer: [3:7574435988855751358:0] socket: 176 qp: -1
2025-11-19T14:00:06.805344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435989647841124:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:06.805351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:06.806381Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435987955330085:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.806398Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435987955330085:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.806434Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989647841106:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.806442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435989647841106:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.809681Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435987955330086:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.809706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435987955330086:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.811995Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435988855751342:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.812017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435988855751261:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.812031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:06.812039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435988855751261:2049] [node 2] ICP22 created new session: [3:7574435988855751359:2049]
2025-11-19T14:00:06.812045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435988855751359:2049] [node 2] ICS09 handshake done sender: [3:7574435988855751328:2111] self: [3:7574435988855751348:0] peer: [2:7574435987955330070:0] socket: 186 qp: -1
2025-11-19T14:00:06.812051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435988855751359:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:06.812059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:06.813185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574435988855751260:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64065) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574435988855751353:2119] held: no
2025-11-19T14:00:06.813348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574435988855751331:2108] [node 4] ICP18 handshake (actor [3:7574435988855751342:2117]) is too old
2025-11-19T14:00:06.813375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435988855751361:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5458
2025-11-19T14:00:06.810527Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435987955330086:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:06.810670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435987955330082:2113] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.810684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:06.810694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435987955330082:2113] [node 4] ICP22 created new session: [2:7574435987955330106:2113]
2025-11-19T14:00:06.810700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435987955330106:2113] [node 4] ICS09 handshake done sender: [2:7574435987955330086:2115] self: [2:7574435987955330084:0] peer: [4:7574435988039304190:0] socket: 220 qp: -1
2025-11-19T14:00:06.810707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435987955330106:2113] [node 4] ICS10 traffic start
2025-11-19T14:00:06.810718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:06.814383Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435988635446557:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.814432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435988635446533:2076] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.813773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435988039304104:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.813793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:06.813801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435988039304104:2050] [node 3] ICP22 created new session: [4:7574435988039304191:2050]
2025-11-19T14:00:06.813808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435988039304191:2050] [node 3] ICS09 handshake done sender: [4:7574435988039304171:2112] self: [4:7574435988039304183:0] peer: [3:7574435988855751333:0] socket: 194 qp: -1
2025-11-19T14:00:06.813813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435988039304191:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:06.813819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:06.815031Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435988039304169:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.815036Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435988039304169:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:06.815481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435988039304193:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48994
2025-11-19T14:00:06.816798Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435989647841106:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.816947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435989647841127:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16890
2025-11-19T14:00:06.815697Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435988855751353:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:06.817079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435989647841097:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.817092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:06.817098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435989647841097:2110] [node 4] ICP22 created new session: [1:7574435989647841128:2110]
2025-11-19T14:00:06.817104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435989647841128:2110] [node 4] ICS09 handshake done sender: [1:7574435989647841106:2114] self: [1:7574435989647841098:0] peer: [4:7574435988039304188:0] socket: 217 qp: -1
2025-11-19T14:00:06.817110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435989647841128:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:06.817118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:06.816666Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435988039304186:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:06.817743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435988855751260:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.817755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:06.817761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435988855751260:2048] [node 1] ICP22 created new session: [3:7574435988855751362:2048]
2025-11-19T14:00:06.817765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435988855751362:2048] [node 1] ICS09 handshake done sender: [3:7574435988855751353:2119] self: [3:7574435988855751358:0] peer: [1:7574435989647841102:0] socket: 212 qp: -1
2025-11-19T14:00:06.817770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435988855751362:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:06.817779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:06.818423Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435987955330088:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.818539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574435987955330082:2113] [node 4] ICP18 handshake (actor [2:7574435987955330088:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.818818Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435988039304169:2110] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:06.819009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435988039304196:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48994
2025-11-19T14:00:06.819387Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435987955330085:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:06.819484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435987955330068:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.819501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:06.819506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435987955330068:2107] [node 6] ICP22 created new session: [2:7574435987955330109:2107]
2025-11-19T14:00:06.819511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435987955330109:2107] [node 6] ICS09 handshake done sender: [2:7574435987955330085:2114] self: [2:7574435987955330083:0] peer: [6:7574435986477641854:0] socket: 219 qp: -1
2025-11-19T14:00:06.819517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435987955330109:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:06.819526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:06.820030Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435988039304185:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:06.819107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986477641855:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25138
2025-11-19T14:00:06.821264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435988039304103:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.821285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:06.821292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435988039304103:2049] [node 2] ICP22 created new session: [4:7574435988039304197:2049]
2025-11-19T14:00:06.821300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435988039304197:2049] [node 2] ICS09 handshake done sender: [4:7574435988039304186:2119] self: [4:7574435988039304190:0] peer: [2:7574435987955330084:0] socket: 177 qp: -1
2025-11-19T14:00:06.821306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435988039304197:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:06.821314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.822326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435988635446574:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62034
2025-11-19T14:00:06.822578Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435988635446557:2119] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:06.822614Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435986477641851:2121] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:06.822908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435988039304165:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.822924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:06.822931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435988039304165:2108] [node 5] ICP22 created new session: [4:7574435988039304199:2108]
2025-11-19T14:00:06.822935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435988039304199:2108] [node 5] ICS09 handshake done sender: [4:7574435988039304169:2110] self: [4:7574435988039304166:0] peer: [5:7574435988635446573:0] socket: 195 qp: -1
2025-11-19T14:00:06.822940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435988039304199:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:06.822945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:06.822700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435988635446533:2076] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.822714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:06.822719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435988635446533:2076] [node 4] ICP22 created new session: [5:7574435988635446575:2076]
2025-11-19T14:00:06.822725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435988635446575:2076] [node 4] ICS09 handshake done sender: [5:7574435988635446557:2119] self: [5:7574435988635446573:0] peer: [4:7574435988039304166:0] socket: 198 qp: -1
2025-11-19T14:00:06.822730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435988635446575:2076] [node 4] ICS10 traffic start
2025-11-19T14:00:06.822737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:06.823938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435989647841127:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.824164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435989647841097:2110] [node 4] ICP18 handshake (actor [1:7574435989647841127:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.825169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986477641856:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25138
2025-11-19T14:00:06.823703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435988039304102:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.823716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:06.823721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435988039304102:2048] [node 1] ICP22 created new session: [4:7574435988039304202:2048]
2025-11-19T14:00:06.823725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435988039304202:2048] [node 1] ICS09 handshake done sender: [4:7574435988039304185:2118] self: [4:7574435988039304188:0] peer: [1:7574435989647841098:0] socket: 179 qp: -1
2025-11-19T14:00:06.823730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435988039304202:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:06.823735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:06.825354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435986477641759:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.825371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:06.825376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435986477641759:2049] [node 2] ICP22 created new session: [6:7574435986477641857:2049]
2025-11-19T14:00:06.825381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435986477641857:2049] [node 2] ICS09 handshake done sender: [6:7574435986477641851:2121] self: [6:7574435986477641854:0] peer: [2:7574435987955330083:0] socket: 226 qp: -1
2025-11-19T14:00:06.825386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435986477641857:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:06.825394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:06.828723Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435986477641856:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.828767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435986477641788:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:06.828942Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435988039304182:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:06.828951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435988039304182:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:06.829606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435986477641860:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25138
2025-11-19T14:00:06.829793Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435988039304182:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:06.829992Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435986477641856:2123] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:06.830113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435986477641788:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:06.830134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:06.830140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435986477641788:2063] [node 4] ICP22 created new session: [6:7574435986477641861:2063]
2025-11-19T14:00:06.830146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435986477641861:2063] [node 4] ICS09 handshake done sender: [6:7574435986477641856:2123] self: [6:7574435986477641859:0] peer: [4:7574435988039304180:0] socket: 185 qp: -1
2025-11-19T14:00:06.830155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435986477641861:2063] [node 4] ICS10 traffic start
2025-11-19T14:00:06.830165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:06.831335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435988039304168:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:06.831359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:06.831367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435988039304168:2109] [node 6] ICP22 created new session: [4:7574435988039304205:2109]
2025-11-19T14:00:06.831373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435988039304205:2109] [node 6] ICS09 handshake done sender: [4:7574435988039304182:2116] self: [4:7574435988039304180:0] peer: [6:7574435986477641859:0] socket: 234 qp: -1
2025-11-19T14:00:06.831379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435988039304205:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:06.831388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:06.843587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:06.843678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435987955330096:2048] [node 1] ICS01 socket: 175 reason# EndOfStream
2025-11-19T14:00:06.843688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435987955330007:2048] [node 1] ICP30 unregister session Session# [2:7574435987955330096:2048] VirtualId# [2:7574435987955330087:0]
2025-11-19T14:00:06.843696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435987955330096:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.843730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:06.843788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:06.844181Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:06.844259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435988855751362:2048] [node 1] ICS01 socket: 212 reason# EndOfStream
2025-11-19T14:00:06.844266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435988855751260:2048] [node 1] ICP30 unregister session Session# [3:7574435988855751362:2048] VirtualId# [3:7574435988855751358:0]
2025-11-19T14:00:06.844272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435988855751362:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.844305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:06.844323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:06.855962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:06.856058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435986477641841:2048] [node 1] ICS01 socket: 174 reason# EndOfStream
2025-11-19T14:00:06.856070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435986477641758:2048] [node 1] ICP30 unregister session Session# [6:7574435986477641841:2048] VirtualId# [6:7574435986477641837:0]
2025-11-19T14:00:06.856078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435986477641841:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.856124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:06.856150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:06.856184Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:06.856242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435988039304202:2048] [node 1] ICS01 socket: 179 reason# EndOfStream
2025-11-19T14:00:06.856247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435988039304102:2048] [node 1] ICP30 unregister session Session# [4:7574435988039304202:2048] VirtualId# [4:7574435988039304188:0]
2025-11-19T14:00:06.856252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435988039304202:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.856279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:06.856298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:06.858362Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:06.858455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435988635446571:2048] [node 1] ICS01 socket: 180 reason# EndOfStream
2025-11-19T14:00:06.858462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435988635446473:2048] [node 1] ICP30 unregister session Session# [5:7574435988635446571:2048] VirtualId# [5:7574435988635446562:0]
2025-11-19T14:00:06.858468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435988635446571:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.858521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:06.858546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:06.871137Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:06.871223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435988855751359:2049] [node 2] ICS01 socket: 186 reason# EndOfStream
2025-11-19T14:00:06.871234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435988855751261:2049] [node 2] ICP30 unregister session Session# [3:7574435988855751359:2049] VirtualId# [3:7574435988855751348:0]
2025-11-19T14:00:06.871240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435988855751359:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.871274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:06.871292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:06.872994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:06.873109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435988635446569:2049] [node 2] ICS01 socket: 181 reason# EndOfStream
2025-11-19T14:00:06.873118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435988635446474:2049] [node 2] ICP30 unregister session Session# [5:7574435988635446569:2049] VirtualId# [5:7574435988635446556:0]
2025-11-19T14:00:06.873126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435988635446569:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.873182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:06.873206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:06.879880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:06.879981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435988039304197:2049] [node 2] ICS01 socket: 177 reason# EndOfStream
2025-11-19T14:00:06.879990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435988039304103:2049] [node 2] ICP30 unregister session Session# [4:7574435988039304197:2049] VirtualId# [4:7574435988039304190:0]
2025-11-19T14:00:06.879998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435988039304197:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.880053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:06.880079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:06.880125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:06.880210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435986477641857:2049] [node 2] ICS01 socket: 226 reason# EndOfStream
2025-11-19T14:00:06.880215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435986477641759:2049] [node 2] ICP30 unregister session Session# [6:7574435986477641857:2049] VirtualId# [6:7574435986477641854:0]
2025-11-19T14:00:06.880220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435986477641857:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.880247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:06.880268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:06.892562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:06.892658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435986477641852:2050] [node 3] ICS01 socket: 203 reason# EndOfStream
2025-11-19T14:00:06.892667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435986477641760:2050] [node 3] ICP30 unregister session Session# [6:7574435986477641852:2050] VirtualId# [6:7574435986477641849:0]
2025-11-19T14:00:06.892674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435986477641852:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.892731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:06.892753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:06.895847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:06.895970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435988635446567:2050] [node 3] ICS01 socket: 192 reason# EndOfStream
2025-11-19T14:00:06.895978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435988635446475:2050] [node 3] ICP30 unregister session Session# [5:7574435988635446567:2050] VirtualId# [5:7574435988635446551:0]
2025-11-19T14:00:06.895985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435988635446567:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.896035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:06.896059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:06.911468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:06.911581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435986477641861:2063] [node 4] ICS01 socket: 185 reason# EndOfStream
2025-11-19T14:00:06.911590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435986477641788:2063] [node 4] ICP30 unregister session Session# [6:7574435986477641861:2063] VirtualId# [6:7574435986477641859:0]
2025-11-19T14:00:06.911597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435986477641861:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.911653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:06.911675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:06.920227Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T14:00:06.920315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435986477641846:2071] [node 5] ICS01 socket: 199 reason# EndOfStream
2025-11-19T14:00:06.920324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435986477641789:2071] [node 5] ICP30 unregister session Session# [6:7574435986477641846:2071] VirtualId# [6:7574435986477641844:0]
2025-11-19T14:00:06.920331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435986477641846:2071] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:06.920377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T14:00:06.920401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:06.972306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435988590740803:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:06.972321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435988590740804:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:06.990791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435986956732336:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:06.990807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435986956732337:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:07.004313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435991484825995:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:07.004323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435991484825996:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:07.024212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435991178765151:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:07.024224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435991178765152:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:07.024229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435991178765153:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:07.040939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435994232167618:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:07.040950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435994232167619:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:07.040954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435994232167620:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:07.068172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435992201118188:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:07.068183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435992201118189:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:07.068188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435992201118190:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:07.125079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435994232167649:2076] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.126406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435992201118221:2077] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.124882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435991178765192:2087] [node 5] ICP01 ready to work
2025-11-19T14:00:07.125552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435991178765200:2091] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.127907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435991251699692:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19544
2025-11-19T14:00:07.127944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435991251699693:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19544
2025-11-19T14:00:07.126260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435992885708165:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:07.126358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435992885708167:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:07.126459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435992885708168:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:07.126347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435991484826055:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7001
2025-11-19T14:00:07.127207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435991484826056:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7001
2025-11-19T14:00:07.127390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435991484826057:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7001
2025-11-19T14:00:07.131792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435992201118222:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7194
2025-11-19T14:00:07.135738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435991251699698:2110] [node 5] ICP01 ready to work
2025-11-19T14:00:07.135256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435991484826059:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:07.137299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435994232167661:2083] [node 6] ICP01 ready to work
2025-11-19T14:00:07.136570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435991484826062:2111] [node 5] ICP01 ready to work
2025-11-19T14:00:07.136664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435991484826063:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.143037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435991484826075:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7001
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.144243Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435994232167619:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16355610ns) {Configure:104(16355268ns) {ProcessConfigured:135(16097769ns) {StartInitialHandshake:157(16096927ns) {DropHandshakes:469(313ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(829ns) {SwitchToState:178(159ns)}}}}}
2025-11-19T14:00:07.145582Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435991484826055:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.143589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435992201118262:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7194
2025-11-19T14:00:07.144630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435991178765229:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52069
2025-11-19T14:00:07.144651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435991178765230:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52069
2025-11-19T14:00:07.144661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435991178765231:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52069
2025-11-19T14:00:07.144672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435991178765232:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52069
2025-11-19T14:00:07.149398Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435992201118222:2078] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.149444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435992201118221:2077] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.150191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435992201118264:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7194
2025-11-19T14:00:07.150477Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435992201118222:2078] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:07.150979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435992201118221:2077] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.150994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:07.150998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435992201118221:2077] [node 4] ICP22 created new session: [6:7574435992201118265:2077]
2025-11-19T14:00:07.151003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435992201118265:2077] [node 4] ICS09 handshake done sender: [6:7574435992201118222:2078] self: [6:7574435992201118263:0] peer: [4:7574435991178765204:0] socket: 177 qp: -1
2025-11-19T14:00:07.151008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435992201118265:2077] [node 4] ICS10 traffic start
2025-11-19T14:00:07.151022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:07.149571Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435991178765205:2095] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.149582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435991178765205:2095] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.150295Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435991178765205:2095] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:07.150570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435991178765200:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.150583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:07.150589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435991178765200:2091] [node 6] ICP22 created new session: [4:7574435991178765234:2091]
2025-11-19T14:00:07.150595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435991178765234:2091] [node 6] ICS09 handshake done sender: [4:7574435991178765205:2095] self: [4:7574435991178765204:0] peer: [6:7574435992201118263:0] socket: 175 qp: -1
2025-11-19T14:00:07.150601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435991178765234:2091] [node 6] ICS10 traffic start
2025-11-19T14:00:07.150622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.153393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435992885708178:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42027
2025-11-19T14:00:07.153404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435992885708179:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42027
2025-11-19T14:00:07.153417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435992885708180:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42027
2025-11-19T14:00:07.153724Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435992885708180:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.156273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435992201118188:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(30269603ns) {Configure:104(30268854ns) {ProcessConfigured:135(30003075ns) {StartInitialHandshake:157(30002340ns) {DropHandshakes:469(273ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(154ns) PrepareNewSessionHandshake:476(1290ns) {SwitchToState:178(582ns)}}}}}
2025-11-19T14:00:07.161816Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435992201118262:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.161861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435992201118190:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.162215Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435991484826075:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.161357Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435991178765231:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.161406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435991178765153:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.164197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435992201118236:2086] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.167507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435991484825995:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(32143924ns) {Configure:104(32142051ns) {ProcessConfigured:135(30351898ns) {StartInitialHandshake:157(30350643ns) {DropHandshakes:469(336ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:476(1376ns) {SwitchToState:178(1099ns)}}}}}
2025-11-19T14:00:07.167710Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435991484825996:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(30998637ns) {Configure:104(30998126ns) {ProcessConfigured:135(30341518ns) {StartInitialHandshake:157(30340748ns) {DropHandshakes:469(233ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(42ns) PrepareNewSessionHandshake:476(698ns) {SwitchToState:178(393ns)}}}}}
2025-11-19T14:00:07.167771Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435991484826056:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.167360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435994232167695:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:113
2025-11-19T14:00:07.167386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435994232167696:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:113
2025-11-19T14:00:07.167998Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435994232167696:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.168019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435994232167620:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.168111Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435991178765230:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.168163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435991178765151:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.168221Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435991484826057:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.168256Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435991484826069:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.168259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435991484826069:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.168613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435991484825995:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.169032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435991251699702:2112] [node 6] ICP01 ready to work
2025-11-19T14:00:07.168120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435994232167661:2083] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(30654962ns) {Configure:104(30654256ns) {ProcessConfigured:135(30289022ns) {StartInitialHandshake:157(30288681ns) {DropHandshakes:469(670ns) {DropIncomingHandshake:441(109ns) DropOutgoingHandshake:456(115ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(622ns) {SwitchToState:178(304ns)}}}}}
2025-11-19T14:00:07.168917Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435994232167695:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.168939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435994232167649:2076] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.171413Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435992885708170:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.171428Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435992885708170:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.172086Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435991484826071:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.172100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435991484826071:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.172777Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435991484826067:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.172796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435991484826067:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.173797Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435992885708178:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.175923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435992885708183:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42027
2025-11-19T14:00:07.177364Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435992885708160:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.177370Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435992885708160:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.177924Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435992885708160:2107] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:07.178005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435988590740803:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.178019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:07.178025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435988590740803:2048] [node 2] ICP22 created new session: [1:7574435992885708185:2048]
2025-11-19T14:00:07.178032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435992885708185:2048] [node 2] ICS09 handshake done sender: [1:7574435992885708160:2107] self: [1:7574435992885708159:0] peer: [2:7574435991251699707:0] socket: 182 qp: -1
2025-11-19T14:00:07.178038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435992885708185:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:07.178056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:07.176732Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435991251699693:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.176763Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435991251699692:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.176807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435986956732336:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.181007Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435992885708179:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.181043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435988590740803:2048] [node 2] ICP18 handshake (actor [1:7574435992885708179:2116]) is too old
2025-11-19T14:00:07.181324Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435992885708183:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.181632Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435992885708172:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.181637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435992885708172:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.182447Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435992885708172:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:07.180237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435992201118273:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7194
2025-11-19T14:00:07.180949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435994232167700:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:113
2025-11-19T14:00:07.181429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435994232167700:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.181461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435994232167618:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.182324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435994232167702:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:113
2025-11-19T14:00:07.182636Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435994232167700:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:07.182747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435994232167618:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.182764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:07.182770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435994232167618:2048] [node 1] ICP22 created new session: [5:7574435994232167703:2048]
2025-11-19T14:00:07.182776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435994232167703:2048] [node 1] ICS09 handshake done sender: [5:7574435994232167700:2117] self: [5:7574435994232167701:0] peer: [1:7574435992885708171:0] socket: 214 qp: -1
2025-11-19T14:00:07.182782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435994232167703:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:07.182789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:07.185244Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435991178765232:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.185505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435992885708167:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.185528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:07.185534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435992885708167:2109] [node 5] ICP22 created new session: [1:7574435992885708190:2109]
2025-11-19T14:00:07.185541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435992885708190:2109] [node 5] ICS09 handshake done sender: [1:7574435992885708172:2113] self: [1:7574435992885708171:0] peer: [5:7574435994232167701:0] socket: 212 qp: -1
2025-11-19T14:00:07.185547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435992885708190:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:07.185554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.186659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435992201118274:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7194
2025-11-19T14:00:07.188637Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435992201118274:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.188673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435992201118188:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.191780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435991251699703:2111] [node 4] ICP01 ready to work
2025-11-19T14:00:07.191886Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435992885708164:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.191903Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435992885708164:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.192005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435991251699698:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(56180523ns) {Configure:104(56179132ns) {ProcessConfigured:135(55860050ns) {StartInitialHandshake:157(55859305ns) {DropHandshakes:469(887ns) {DropIncomingHandshake:441(132ns) DropOutgoingHandshake:456(171ns)} GenerateSessionVirtualId:988(167ns) PrepareNewSessionHandshake:476(573ns) {SwitchToState:178(312ns)}}}}}
2025-11-19T14:00:07.193618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435986956732336:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11172) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435991251699693:2108] held: no
2025-11-19T14:00:07.191806Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435991178765214:2103] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.191833Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435991178765214:2103] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.196431Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435992201118273:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.196468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435992201118189:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.196168Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435992885708176:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.196186Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435992885708176:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.196365Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435992885708170:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:07.199391Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435991484826071:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:07.199558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435991484826063:2112] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.199577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:07.199584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435991484826063:2112] [node 4] ICP22 created new session: [3:7574435991484826083:2112]
2025-11-19T14:00:07.199590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435991484826083:2112] [node 4] ICS09 handshake done sender: [3:7574435991484826071:2115] self: [3:7574435991484826070:0] peer: [4:7574435991178765237:0] socket: 198 qp: -1
2025-11-19T14:00:07.199596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435991484826083:2112] [node 4] ICS10 traffic start
2025-11-19T14:00:07.199604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:07.205570Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435991178765153:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435991178765231:2117] held: no
2025-11-19T14:00:07.205934Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435991484826069:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.206142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435992885708165:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.206166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:07.206173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435992885708165:2108] [node 4] ICP22 created new session: [1:7574435992885708194:2108]
2025-11-19T14:00:07.206179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435992885708194:2108] [node 4] ICS09 handshake done sender: [1:7574435992885708170:2112] self: [1:7574435992885708169:0] peer: [4:7574435991178765238:0] socket: 186 qp: -1
2025-11-19T14:00:07.206185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435992885708194:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:07.206195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:07.206020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435992201118236:2086] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(41676167ns) {Configure:104(41675091ns) {ProcessConfigured:135(41279267ns) {StartInitialHandshake:157(41278748ns) {DropHandshakes:469(927ns) {DropIncomingHandshake:441(197ns) DropOutgoingHandshake:456(151ns)} GenerateSessionVirtualId:988(288ns) PrepareNewSessionHandshake:476(1110ns) {SwitchToState:178(763ns)}}}}}
2025-11-19T14:00:07.206709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435991251699715:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19544
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.212067Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435991178765214:2103] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.213010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435991178765192:2087] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.213036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:07.213043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435991178765192:2087] [node 5] ICP22 created new session: [4:7574435991178765242:2087]
2025-11-19T14:00:07.213051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435991178765242:2087] [node 5] ICS09 handshake done sender: [4:7574435991178765214:2103] self: [4:7574435991178765197:0] peer: [5:7574435994232167699:0] socket: 194 qp: -1
2025-11-19T14:00:07.213058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435991178765242:2087] [node 5] ICS10 traffic start
2025-11-19T14:00:07.213081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.214670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435992201118278:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7194
2025-11-19T14:00:07.214688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435992201118279:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7194
2025-11-19T14:00:07.211807Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435991251699706:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.211830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435991251699706:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.212242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435991251699717:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19544
2025-11-19T14:00:07.212270Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435991251699693:2108] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:07.214444Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435986956732337:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(45217098ns) {Configure:104(45216374ns) {ProcessConfigured:135(44885008ns) {StartInitialHandshake:157(44884213ns) {DropHandshakes:469(554ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(188ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(839ns) {SwitchToState:178(261ns)}}}}}
2025-11-19T14:00:07.215050Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435991251699706:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:07.215228Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435992201118273:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.215819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435992201118280:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7194
2025-11-19T14:00:07.215394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435991251699702:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.215413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:07.215418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435991251699702:2112] [node 6] ICP22 created new session: [2:7574435991251699720:2112]
2025-11-19T14:00:07.215423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435991251699720:2112] [node 6] ICS09 handshake done sender: [2:7574435991251699706:2113] self: [2:7574435991251699705:0] peer: [6:7574435992201118276:0] socket: 210 qp: -1
2025-11-19T14:00:07.215429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435991251699720:2112] [node 6] ICS10 traffic start
2025-11-19T14:00:07.215446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:07.214943Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435991178765151:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11172) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435991178765230:2116] held: no
2025-11-19T14:00:07.215990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435992201118189:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.216006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:07.216011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435992201118189:2049] [node 2] ICP22 created new session: [6:7574435992201118281:2049]
2025-11-19T14:00:07.216016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435992201118281:2049] [node 2] ICS09 handshake done sender: [6:7574435992201118273:2117] self: [6:7574435992201118276:0] peer: [2:7574435991251699705:0] socket: 213 qp: -1
2025-11-19T14:00:07.216021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435992201118281:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:07.216027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.217390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435991178765244:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52069
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.217570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435991178765245:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52069
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.217693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435991178765246:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52069
2025-11-19T14:00:07.217754Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435991178765230:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:07.217823Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435991178765231:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:07.217891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435991178765151:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.217905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:07.217911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435991178765151:2048] [node 1] ICP22 created new session: [4:7574435991178765247:2048]
2025-11-19T14:00:07.217930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435991178765247:2048] [node 1] ICS09 handshake done sender: [4:7574435991178765230:2116] self: [4:7574435991178765238:0] peer: [1:7574435992885708169:0] socket: 201 qp: -1
2025-11-19T14:00:07.217936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435991178765247:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:07.217942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:07.218114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435991178765153:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.218121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:07.218125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435991178765153:2050] [node 3] ICP22 created new session: [4:7574435991178765249:2050]
2025-11-19T14:00:07.218129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435991178765249:2050] [node 3] ICS09 handshake done sender: [4:7574435991178765231:2117] self: [4:7574435991178765237:0] peer: [3:7574435991484826070:0] socket: 202 qp: -1
2025-11-19T14:00:07.218132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435991178765249:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:07.218137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:07.213550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435994232167707:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:113
2025-11-19T14:00:07.213605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435994232167708:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:113
2025-11-19T14:00:07.213951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435991484826062:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.213974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:07.213980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435991484826062:2111] [node 5] ICP22 created new session: [3:7574435991484826087:2111]
2025-11-19T14:00:07.213987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435991484826087:2111] [node 5] ICS09 handshake done sender: [3:7574435991484826069:2114] self: [3:7574435991484826068:0] peer: [5:7574435994232167697:0] socket: 197 qp: -1
2025-11-19T14:00:07.213993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435991484826087:2111] [node 5] ICS10 traffic start
2025-11-19T14:00:07.214025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:07.219155Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435991178765246:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.219180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435991178765152:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.218743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435986956732336:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.218758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:07.218762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435986956732336:2048] [node 1] ICP22 created new session: [2:7574435991251699722:2048]
2025-11-19T14:00:07.218767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435991251699722:2048] [node 1] ICS09 handshake done sender: [2:7574435991251699693:2108] self: [2:7574435991251699707:0] peer: [1:7574435992885708159:0] socket: 187 qp: -1
2025-11-19T14:00:07.218771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435991251699722:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:07.218776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:07.219715Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435992201118274:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:07.219319Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435992885708176:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:07.219888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435992201118188:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.219903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:07.219908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435992201118188:2048] [node 1] ICP22 created new session: [6:7574435992201118284:2048]
2025-11-19T14:00:07.219914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435992201118284:2048] [node 1] ICS09 handshake done sender: [6:7574435992201118274:2118] self: [6:7574435992201118275:0] peer: [1:7574435992885708175:0] socket: 219 qp: -1
2025-11-19T14:00:07.219919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435992201118284:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:07.219925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:07.221418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435992885708196:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42027
2025-11-19T14:00:07.221965Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435992885708164:2110] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:07.222157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435992885708168:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.222171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:07.222178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435992885708168:2111] [node 6] ICP22 created new session: [1:7574435992885708199:2111]
2025-11-19T14:00:07.222183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435992885708199:2111] [node 6] ICS09 handshake done sender: [1:7574435992885708176:2114] self: [1:7574435992885708175:0] peer: [6:7574435992201118275:0] socket: 218 qp: -1
2025-11-19T14:00:07.222188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435992885708199:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:07.222205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:07.220587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435991251699716:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19544
2025-11-19T14:00:07.225170Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435991251699712:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.225178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435991251699712:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.224881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435994232167706:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:113
2025-11-19T14:00:07.224964Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435994232167649:2076] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26059) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435994232167695:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.225682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435991484826091:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7001
2025-11-19T14:00:07.225702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435991484826092:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7001
2025-11-19T14:00:07.226263Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435991484826067:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.227694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435991251699725:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19544
2025-11-19T14:00:07.227960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435991484826059:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.227983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:07.227989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435991484826059:2110] [node 6] ICP22 created new session: [3:7574435991484826093:2110]
2025-11-19T14:00:07.227996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435991484826093:2110] [node 6] ICS09 handshake done sender: [3:7574435991484826067:2113] self: [3:7574435991484826066:0] peer: [6:7574435992201118269:0] socket: 196 qp: -1
2025-11-19T14:00:07.228003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435991484826093:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:07.228012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:07.228228Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435991484826057:2109] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:07.228334Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435991251699725:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.228490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435991484825995:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.228504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:07.228512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435991484825995:2048] [node 1] ICP22 created new session: [3:7574435991484826095:2048]
2025-11-19T14:00:07.228517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435991484826095:2048] [node 1] ICS09 handshake done sender: [3:7574435991484826057:2109] self: [3:7574435991484826081:0] peer: [1:7574435992885708163:0] socket: 184 qp: -1
2025-11-19T14:00:07.228522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435991484826095:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:07.228530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:07.233780Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435991251699717:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.236077Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435994232167620:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574435994232167696:2115] held: no
2025-11-19T14:00:07.236283Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435994232167695:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:07.236564Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435992201118262:2112] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:07.236821Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435991484826063:2112] [node 4] ICP35 event processing took too much time PendingConnection:278(37164347ns) {HandleHandshakeStatus:329(37163166ns) {DropIncomingHandshake:441(4647ns) DropOutgoingHandshake:456(192ns) SwitchToState:178(234ns) ProcessPendingSessionEvents:495(25128240ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25108702ns max# 25038341ns}}}
2025-11-19T14:00:07.237538Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435991251699716:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.237235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435988590740804:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.237260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:07.237268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435988590740804:2049] [node 3] ICP22 created new session: [1:7574435992885708201:2049]
2025-11-19T14:00:07.237274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435992885708201:2049] [node 3] ICS09 handshake done sender: [1:7574435992885708164:2110] self: [1:7574435992885708163:0] peer: [3:7574435991484826081:0] socket: 183 qp: -1
2025-11-19T14:00:07.237281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435992885708201:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:07.237288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:07.237792Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435994232167706:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.237827Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435994232167696:2115] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:07.238459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435994232167709:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:113
2025-11-19T14:00:07.238494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435994232167619:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.240139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435994232167649:2076] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.240153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:07.240160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435994232167649:2076] [node 4] ICP22 created new session: [5:7574435994232167712:2076]
2025-11-19T14:00:07.240165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435994232167712:2076] [node 4] ICS09 handshake done sender: [5:7574435994232167695:2114] self: [5:7574435994232167699:0] peer: [4:7574435991178765197:0] socket: 195 qp: -1
2025-11-19T14:00:07.240172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435994232167712:2076] [node 4] ICS10 traffic start
2025-11-19T14:00:07.240180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:07.240190Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435991178765192:2087] [node 5] ICP35 event processing took too much time PendingConnection:278(27083538ns) {HandleHandshakeStatus:329(27082833ns) {DropIncomingHandshake:441(4565ns) DropOutgoingHandshake:456(280ns) SwitchToState:178(217ns) ProcessPendingSessionEvents:495(18584881ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18561910ns max# 18499600ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.243083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435991178765252:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52069
2025-11-19T14:00:07.243208Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435991251699710:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.243218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435991251699710:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.243633Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435991251699712:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.243942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435992201118190:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.243961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:07.243970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435992201118190:2050] [node 3] ICP22 created new session: [6:7574435992201118287:2050]
2025-11-19T14:00:07.243976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435992201118287:2050] [node 3] ICS09 handshake done sender: [6:7574435992201118262:2112] self: [6:7574435992201118269:0] peer: [3:7574435991484826066:0] socket: 199 qp: -1
2025-11-19T14:00:07.243983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435992201118287:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:07.243990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:07.244129Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435992201118190:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.244984Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435994232167709:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.245001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435994232167620:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.245011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:07.245018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435994232167620:2050] [node 3] ICP22 created new session: [5:7574435994232167715:2050]
2025-11-19T14:00:07.245023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435994232167715:2050] [node 3] ICS09 handshake done sender: [5:7574435994232167696:2115] self: [5:7574435994232167697:0] peer: [3:7574435991484826068:0] socket: 209 qp: -1
2025-11-19T14:00:07.245029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435994232167715:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:07.245034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:07.245258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435994232167714:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:113
2025-11-19T14:00:07.246476Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435991178765246:2121] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:07.246724Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435991251699710:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:07.246982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435991251699698:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.246999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:07.247005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435991251699698:2110] [node 5] ICP22 created new session: [2:7574435991251699731:2110]
2025-11-19T14:00:07.247010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435991251699731:2110] [node 5] ICS09 handshake done sender: [2:7574435991251699710:2114] self: [2:7574435991251699699:0] peer: [5:7574435994232167711:0] socket: 188 qp: -1
2025-11-19T14:00:07.247014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435991251699731:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:07.247020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:07.246043Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435992885708168:2111] [node 6] ICP35 event processing took too much time PendingConnection:278(23787822ns) {HandleHandshakeStatus:329(23787172ns) {DropIncomingHandshake:441(3615ns) DropOutgoingHandshake:456(88ns) SwitchToState:178(73ns) ProcessPendingSessionEvents:495(8723912ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8703440ns max# 8643716ns}}}
2025-11-19T14:00:07.247381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435991178765152:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.247397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:07.247403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435991178765152:2049] [node 2] ICP22 created new session: [4:7574435991178765253:2049]
2025-11-19T14:00:07.247408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435991178765253:2049] [node 2] ICS09 handshake done sender: [4:7574435991178765246:2121] self: [4:7574435991178765251:0] peer: [2:7574435991251699711:0] socket: 233 qp: -1
2025-11-19T14:00:07.247412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435991178765253:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:07.247419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:07.248353Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435994232167706:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:07.248452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435994232167619:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.248472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:07.248480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435994232167619:2049] [node 2] ICP22 created new session: [5:7574435994232167718:2049]
2025-11-19T14:00:07.248485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435994232167718:2049] [node 2] ICS09 handshake done sender: [5:7574435994232167706:2119] self: [5:7574435994232167711:0] peer: [2:7574435991251699699:0] socket: 221 qp: -1
2025-11-19T14:00:07.248490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435994232167718:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:07.248496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:07.248931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435991484826091:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.248979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435991484825996:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.249125Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435991251699719:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.249134Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435991251699719:2119] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.249416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435992201118286:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7194
2025-11-19T14:00:07.249794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435991251699703:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.249814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:07.249819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435991251699703:2111] [node 4] ICP22 created new session: [2:7574435991251699733:2111]
2025-11-19T14:00:07.249824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435991251699733:2111] [node 4] ICS09 handshake done sender: [2:7574435991251699712:2115] self: [2:7574435991251699711:0] peer: [4:7574435991178765251:0] socket: 189 qp: -1
2025-11-19T14:00:07.249829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435991251699733:2111] [node 4] ICS10 traffic start
2025-11-19T14:00:07.249836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.253808Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435992201118286:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.253861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435992201118236:2086] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.254310Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435988590740804:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:07.254341Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435994232167698:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.254356Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435994232167698:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.254667Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435991251699719:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:07.254776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435986956732337:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.254792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:07.254799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435986956732337:2049] [node 3] ICP22 created new session: [2:7574435991251699736:2049]
2025-11-19T14:00:07.254804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435991251699736:2049] [node 3] ICS09 handshake done sender: [2:7574435991251699719:2119] self: [2:7574435991251699704:0] peer: [3:7574435991484826097:0] socket: 236 qp: -1
2025-11-19T14:00:07.254808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435991251699736:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:07.254815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:07.254847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435991484826098:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7001
2025-11-19T14:00:07.255741Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435991484826091:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.256111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435992201118290:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7194
2025-11-19T14:00:07.256351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435991484825996:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.256373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:07.256378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435991484825996:2049] [node 2] ICP22 created new session: [3:7574435991484826099:2049]
2025-11-19T14:00:07.256384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435991484826099:2049] [node 2] ICS09 handshake done sender: [3:7574435991484826091:2119] self: [3:7574435991484826097:0] peer: [2:7574435991251699704:0] socket: 185 qp: -1
2025-11-19T14:00:07.256390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435991484826099:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:07.256402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:07.257526Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435994232167698:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:07.257714Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435992201118286:2123] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:07.257807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435992201118236:2086] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.257819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:07.257825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435992201118236:2086] [node 5] ICP22 created new session: [6:7574435992201118291:2086]
2025-11-19T14:00:07.257830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435992201118291:2086] [node 5] ICS09 handshake done sender: [6:7574435992201118286:2123] self: [6:7574435992201118289:0] peer: [5:7574435994232167689:0] socket: 243 qp: -1
2025-11-19T14:00:07.257835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435992201118291:2086] [node 5] ICS10 traffic start
2025-11-19T14:00:07.257843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:07.258925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435994232167661:2083] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.258951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:07.258958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435994232167661:2083] [node 6] ICP22 created new session: [5:7574435994232167721:2083]
2025-11-19T14:00:07.258965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435994232167721:2083] [node 6] ICS09 handshake done sender: [5:7574435994232167698:2116] self: [5:7574435994232167689:0] peer: [6:7574435992201118289:0] socket: 239 qp: -1
2025-11-19T14:00:07.258971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435994232167721:2083] [node 6] ICS10 traffic start
2025-11-19T14:00:07.258983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:07.275614Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:07.275703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435991251699722:2048] [node 1] ICS01 socket: 187 reason# EndOfStream
2025-11-19T14:00:07.275713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435986956732336:2048] [node 1] ICP30 unregister session Session# [2:7574435991251699722:2048] VirtualId# [2:7574435991251699707:0]
2025-11-19T14:00:07.275720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435991251699722:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.275767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:07.275788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:07.276191Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:07.276254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435991484826095:2048] [node 1] ICS01 socket: 184 reason# EndOfStream
2025-11-19T14:00:07.276260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435991484825995:2048] [node 1] ICP30 unregister session Session# [3:7574435991484826095:2048] VirtualId# [3:7574435991484826081:0]
2025-11-19T14:00:07.276264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435991484826095:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.276288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:07.276304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:07.295985Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:07.296085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435994232167703:2048] [node 1] ICS01 socket: 214 reason# EndOfStream
2025-11-19T14:00:07.296095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435994232167618:2048] [node 1] ICP30 unregister session Session# [5:7574435994232167703:2048] VirtualId# [5:7574435994232167701:0]
2025-11-19T14:00:07.296104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435994232167703:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.296163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:07.296188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:07.298425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:07.298513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435991178765247:2048] [node 1] ICS01 socket: 201 reason# EndOfStream
2025-11-19T14:00:07.298522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435991178765151:2048] [node 1] ICP30 unregister session Session# [4:7574435991178765247:2048] VirtualId# [4:7574435991178765238:0]
2025-11-19T14:00:07.298528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435991178765247:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.298582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:07.298608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:07.298647Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:07.298713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435992201118284:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-11-19T14:00:07.298717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435992201118188:2048] [node 1] ICP30 unregister session Session# [6:7574435992201118284:2048] VirtualId# [6:7574435992201118275:0]
2025-11-19T14:00:07.298721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435992201118284:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.298752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:07.298772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:07.304564Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:07.304653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435991484826099:2049] [node 2] ICS01 socket: 185 reason# EndOfStream
2025-11-19T14:00:07.304662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435991484825996:2049] [node 2] ICP30 unregister session Session# [3:7574435991484826099:2049] VirtualId# [3:7574435991484826097:0]
2025-11-19T14:00:07.304669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435991484826099:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.304714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:07.304734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:07.314865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:07.314956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435994232167718:2049] [node 2] ICS01 socket: 221 reason# EndOfStream
2025-11-19T14:00:07.314964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435994232167619:2049] [node 2] ICP30 unregister session Session# [5:7574435994232167718:2049] VirtualId# [5:7574435994232167711:0]
2025-11-19T14:00:07.314971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435994232167718:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.315025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:07.315050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:07.315089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:07.315162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435991178765253:2049] [node 2] ICS01 socket: 233 reason# EndOfStream
2025-11-19T14:00:07.315167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435991178765152:2049] [node 2] ICP30 unregister session Session# [4:7574435991178765253:2049] VirtualId# [4:7574435991178765251:0]
2025-11-19T14:00:07.315171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435991178765253:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.315193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:07.315212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:07.315244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:07.315302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435992201118281:2049] [node 2] ICS01 socket: 213 reason# EndOfStream
2025-11-19T14:00:07.315306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435992201118189:2049] [node 2] ICP30 unregister session Session# [6:7574435992201118281:2049] VirtualId# [6:7574435992201118276:0]
2025-11-19T14:00:07.315310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435992201118281:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.315344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:07.315364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:07.328429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:07.328518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435992201118287:2050] [node 3] ICS01 socket: 199 reason# EndOfStream
2025-11-19T14:00:07.328528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435992201118190:2050] [node 3] ICP30 unregister session Session# [6:7574435992201118287:2050] VirtualId# [6:7574435992201118269:0]
2025-11-19T14:00:07.328534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435992201118287:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.328596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:07.328619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:07.328657Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:07.328731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435994232167715:2050] [node 3] ICS01 socket: 209 reason# EndOfStream
2025-11-19T14:00:07.328737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435994232167620:2050] [node 3] ICP30 unregister session Session# [5:7574435994232167715:2050] VirtualId# [5:7574435994232167697:0]
2025-11-19T14:00:07.328741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435994232167715:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.328772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:07.328796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:07.351411Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:07.351509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435992201118265:2077] [node 4] ICS01 socket: 177 reason# EndOfStream
2025-11-19T14:00:07.351519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435992201118221:2077] [node 4] ICP30 unregister session Session# [6:7574435992201118265:2077] VirtualId# [6:7574435992201118263:0]
2025-11-19T14:00:07.351527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435992201118265:2077] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.351602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:07.351632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:07.484484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435991316529650:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:07.488146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435991316529651:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:07.513352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435994419951799:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:07.513372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435994419951800:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:07.536198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435993547531171:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:07.536213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435993547531172:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:07.556035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435992419067146:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:07.556046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435992419067147:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:07.556052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435992419067148:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:07.588038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435992237843827:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:07.588052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435992237843828:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:07.588056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435992237843829:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:07.608176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435994151271889:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:07.608192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435994151271890:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:07.608198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435994151271891:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:07.615857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435992237843843:2062] [node 6] ICP01 ready to work
2025-11-19T14:00:07.616919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435991316529695:2091] [node 6] ICP01 ready to work
2025-11-19T14:00:07.617399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435994419951842:2088] [node 4] ICP01 ready to work
2025-11-19T14:00:07.617483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435994419951843:2089] [node 5] ICP01 ready to work
2025-11-19T14:00:07.620555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435991316529708:2097] [node 4] ICP01 ready to work
2025-11-19T14:00:07.620094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435992237843852:2067] [node 4] ICP01 ready to work
2025-11-19T14:00:07.620596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435994151271917:2067] [node 4] ICP01 ready to work
2025-11-19T14:00:07.629093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435992419067199:2098] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.634520Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435994419951800:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(16985010ns) {Configure:104(16984415ns) {ProcessConfigured:135(16610159ns) {StartInitialHandshake:157(16609687ns) {DropHandshakes:469(533ns) {DropIncomingHandshake:441(87ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(88ns) PrepareNewSessionHandshake:476(1078ns) {SwitchToState:178(563ns)}}}}}
2025-11-19T14:00:07.634661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435994419951857:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20059
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.636278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435993547531228:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24903
2025-11-19T14:00:07.636301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435993547531229:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24903
2025-11-19T14:00:07.634888Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435991316529651:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18911378ns) {Configure:104(18910948ns) {ProcessConfigured:135(16734210ns) {StartInitialHandshake:157(16733937ns) {DropHandshakes:469(513ns) {DropIncomingHandshake:441(122ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(329ns) PrepareNewSessionHandshake:476(1036ns) {SwitchToState:178(849ns)}}}}}
2025-11-19T14:00:07.635074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435991316529714:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56378
2025-11-19T14:00:07.635129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435991316529715:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56378
2025-11-19T14:00:07.635365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435991316529650:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19153074ns) {Configure:104(19152011ns) {ProcessConfigured:135(17103864ns) {StartInitialHandshake:157(17103327ns) {DropHandshakes:469(408ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(140ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(405ns) {SwitchToState:178(69ns)}}}}}
2025-11-19T14:00:07.636816Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435991316529714:2105] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.640361Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435993547531229:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.640393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435993547531171:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.640461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435994151271904:2061] [node 5] ICP01 ready to work
2025-11-19T14:00:07.644189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435992419067179:2078] [node 6] ICP01 ready to work
2025-11-19T14:00:07.646023Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435994419951842:2088] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(28081115ns) {Configure:104(26063368ns) {ProcessConfigured:135(25588042ns) {StartInitialHandshake:157(25586859ns) {DropHandshakes:469(487ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(64ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:476(1071ns) {SwitchToState:178(568ns)}}}}}
2025-11-19T14:00:07.648202Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435992237843852:2067] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(28058072ns) {Configure:104(28057436ns) {ProcessConfigured:135(27702589ns) {StartInitialHandshake:157(27702115ns) {DropHandshakes:469(464ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(173ns) PrepareNewSessionHandshake:476(1329ns) {SwitchToState:178(575ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.648996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435991316529708:2097] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(28389187ns) {Configure:104(28388944ns) {ProcessConfigured:135(28115957ns) {StartInitialHandshake:157(28115794ns) {DropHandshakes:469(283ns) {DropIncomingHandshake:441(27ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(41ns) PrepareNewSessionHandshake:476(970ns) {SwitchToState:178(54ns)}}}}}
2025-11-19T14:00:07.649372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435993547531244:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24903
2025-11-19T14:00:07.649225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435994151271917:2067] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(28570045ns) {Configure:104(28569553ns) {ProcessConfigured:135(28234324ns) {StartInitialHandshake:157(28234078ns) {DropHandshakes:469(260ns) {DropIncomingHandshake:441(36ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(600ns) {SwitchToState:178(189ns)}}}}}
2025-11-19T14:00:07.650921Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435991316529713:2104] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.650945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435991316529713:2104] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.652089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435994419951850:2081] [node 6] ICP01 ready to work
2025-11-19T14:00:07.656341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435994419951843:2089] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(36372365ns) {Configure:104(17797259ns) {ProcessConfigured:135(17440486ns) {StartInitialHandshake:157(17439692ns) {DropHandshakes:469(690ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(209ns)} GenerateSessionVirtualId:988(337ns) PrepareNewSessionHandshake:476(1413ns) {SwitchToState:178(657ns)}}}}}
2025-11-19T14:00:07.659889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435992237843828:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20767549ns) {Configure:104(20766828ns) {ProcessConfigured:135(20383191ns) {StartInitialHandshake:157(20382614ns) {DropHandshakes:469(442ns) {DropIncomingHandshake:441(77ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:476(1344ns) {SwitchToState:178(1102ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.660086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435993547531172:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(20975159ns) {Configure:104(20974645ns) {ProcessConfigured:135(20571163ns) {StartInitialHandshake:157(20570592ns) {DropHandshakes:469(472ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:476(951ns) {SwitchToState:178(282ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.660703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994151271966:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61245
2025-11-19T14:00:07.660446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435992237843904:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8529
2025-11-19T14:00:07.660501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435992237843905:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8529
2025-11-19T14:00:07.663986Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435992237843905:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.664036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435992237843828:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.668404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435994151271891:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(52280601ns) {Configure:104(52279802ns) {ProcessConfigured:135(51643277ns) {StartInitialHandshake:157(51642662ns) {DropHandshakes:469(237ns) {DropIncomingHandshake:441(14ns) DropOutgoingHandshake:456(39ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(1467ns) {SwitchToState:178(995ns)}}}}}
2025-11-19T14:00:07.669778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435994419951799:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(28650272ns) {Configure:104(28649658ns) {ProcessConfigured:135(28344764ns) {StartInitialHandshake:157(28344215ns) {DropHandshakes:469(479ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(254ns) PrepareNewSessionHandshake:476(1448ns) {SwitchToState:178(702ns)}}}}}
2025-11-19T14:00:07.670697Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435992419067146:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(26743869ns) {Configure:104(26742977ns) {ProcessConfigured:135(26482440ns) {StartInitialHandshake:157(26481985ns) {DropHandshakes:469(392ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(52ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:476(1277ns) {SwitchToState:178(426ns)}}}}}
2025-11-19T14:00:07.671109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435992419067179:2078] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26812293ns) {Configure:104(26812100ns) {ProcessConfigured:135(26596060ns) {StartInitialHandshake:157(26595752ns) {DropHandshakes:469(181ns) {DropIncomingHandshake:441(41ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:476(895ns) {SwitchToState:178(77ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.674796Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435994419951857:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.671638Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435992237843904:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.675924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435992237843852:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.676646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435993547531239:2099] [node 5] ICP01 ready to work
2025-11-19T14:00:07.676730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435993547531232:2086] [node 6] ICP01 ready to work
2025-11-19T14:00:07.677297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435993547531231:2084] [node 4] ICP01 ready to work
2025-11-19T14:00:07.679137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994151271968:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61245
2025-11-19T14:00:07.679215Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435994151271966:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.679258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435994151271904:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.676226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435991316529723:2103] [node 5] ICP01 ready to work
2025-11-19T14:00:07.683882Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435994419951875:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.683908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435994419951875:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.684537Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435991316529715:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.685414Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435991316529713:2104] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:07.685241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435993547531250:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24903
2025-11-19T14:00:07.685545Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435993547531229:2103] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.687836Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435991316529695:2091] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(70848804ns) {Configure:104(70847988ns) {ProcessConfigured:135(69451270ns) {StartInitialHandshake:157(69450109ns) {DropHandshakes:469(970ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(142ns) PrepareNewSessionHandshake:476(1402ns) {SwitchToState:178(521ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.685378Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435992419067201:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.685392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435992419067201:2099] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.688219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435992419067224:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19724
2025-11-19T14:00:07.688254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435992419067225:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19724
2025-11-19T14:00:07.688282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435992419067226:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19724
2025-11-19T14:00:07.688816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435992237843908:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8529
2025-11-19T14:00:07.689070Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435992419067201:2099] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:07.689167Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435992237843904:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:07.689020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574435994419951850:2081] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(36778858ns) {Configure:104(36778342ns) {ProcessConfigured:135(36445642ns) {StartInitialHandshake:157(36445108ns) {DropHandshakes:469(497ns) {DropIncomingHandshake:441(46ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(592ns) {SwitchToState:178(344ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.689378Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435992237843845:2063] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.689400Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435992237843845:2063] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.689444Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435992419067225:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.689474Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435992419067226:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.689514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435992419067199:2098] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.689525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:07.689531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435992419067199:2098] [node 5] ICP22 created new session: [4:7574435992419067227:2098]
2025-11-19T14:00:07.689538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435992419067227:2098] [node 5] ICS09 handshake done sender: [4:7574435992419067201:2099] self: [4:7574435992419067200:0] peer: [5:7574435992237843907:0] socket: 183 qp: -1
2025-11-19T14:00:07.689545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435992419067227:2098] [node 5] ICS10 traffic start
2025-11-19T14:00:07.689553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:07.689586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435991316529651:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.689609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:07.689614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435991316529651:2049] [node 3] ICP22 created new session: [1:7574435991316529733:2049]
2025-11-19T14:00:07.689620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435991316529733:2049] [node 3] ICS09 handshake done sender: [1:7574435991316529713:2104] self: [1:7574435991316529696:0] peer: [3:7574435993547531240:0] socket: 191 qp: -1
2025-11-19T14:00:07.689625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435991316529733:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:07.689632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:07.689705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574435992237843827:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(69602246ns) {Configure:104(69600737ns) {ProcessConfigured:135(68409232ns) {StartInitialHandshake:157(68408894ns) {DropHandshakes:469(524ns) {DropIncomingHandshake:441(24ns) DropOutgoingHandshake:456(42ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:476(754ns) {SwitchToState:178(143ns)}}}}}
2025-11-19T14:00:07.689718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435992237843852:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.689726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:07.689730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435992237843852:2067] [node 4] ICP22 created new session: [5:7574435992237843910:2067]
2025-11-19T14:00:07.689734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435992237843910:2067] [node 4] ICS09 handshake done sender: [5:7574435992237843904:2114] self: [5:7574435992237843907:0] peer: [4:7574435992419067200:0] socket: 184 qp: -1
2025-11-19T14:00:07.689738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435992237843910:2067] [node 4] ICS10 traffic start
2025-11-19T14:00:07.689751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:07.689824Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435994151271968:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.690431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994151271971:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61245
2025-11-19T14:00:07.691734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435993547531171:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.691799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:07.691805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435993547531171:2048] [node 1] ICP22 created new session: [3:7574435993547531252:2048]
2025-11-19T14:00:07.691813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435993547531252:2048] [node 1] ICS09 handshake done sender: [3:7574435993547531229:2103] self: [3:7574435993547531240:0] peer: [1:7574435991316529696:0] socket: 193 qp: -1
2025-11-19T14:00:07.691821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435993547531252:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:07.691837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:07.693081Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435994151271971:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.693119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435994151271889:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.693282Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435991316529731:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.693292Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435991316529731:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.693994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994151271973:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61245
2025-11-19T14:00:07.694185Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435991316529731:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:07.694353Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435994151271971:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:07.694454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435994151271889:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.694470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:07.694475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435994151271889:2048] [node 1] ICP22 created new session: [6:7574435994151271974:2048]
2025-11-19T14:00:07.694481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435994151271974:2048] [node 1] ICS09 handshake done sender: [6:7574435994151271971:2117] self: [6:7574435994151271972:0] peer: [1:7574435991316529698:0] socket: 218 qp: -1
2025-11-19T14:00:07.694485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435994151271974:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:07.694502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:07.694938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435991316529695:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.694951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:07.694956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435991316529695:2091] [node 6] ICP22 created new session: [1:7574435991316529738:2091]
2025-11-19T14:00:07.694961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435991316529738:2091] [node 6] ICS09 handshake done sender: [1:7574435991316529731:2115] self: [1:7574435991316529698:0] peer: [6:7574435994151271972:0] socket: 189 qp: -1
2025-11-19T14:00:07.694965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435991316529738:2091] [node 6] ICS10 traffic start
2025-11-19T14:00:07.694979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:07.695971Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435993547531232:2086] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19106612ns) {Configure:104(19105941ns) {ProcessConfigured:135(18812269ns) {StartInitialHandshake:157(18811827ns) {DropHandshakes:469(505ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:476(804ns) {SwitchToState:178(563ns)}}}}}
2025-11-19T14:00:07.696049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435993547531228:2102] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.696190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435993547531172:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.696588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435993547531231:2084] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19187073ns) {Configure:104(19186600ns) {ProcessConfigured:135(18974606ns) {StartInitialHandshake:157(18974240ns) {DropHandshakes:469(496ns) {DropIncomingHandshake:441(33ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:476(571ns) {SwitchToState:178(145ns)}}}}}
2025-11-19T14:00:07.697654Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435992419067148:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(58915089ns) {Configure:104(58914184ns) {ProcessConfigured:135(58600961ns) {StartInitialHandshake:157(58600167ns) {DropHandshakes:469(691ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(274ns) PrepareNewSessionHandshake:476(740ns) {SwitchToState:178(194ns)}}}}}
2025-11-19T14:00:07.697718Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435992419067224:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.697751Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435992419067220:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.697754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435992419067220:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.698358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435992419067146:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.698847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435994419951880:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20059
2025-11-19T14:00:07.699093Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435994419951875:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:07.698936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435992419067223:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19724
2025-11-19T14:00:07.699180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435994419951843:2089] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.699192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:07.699199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435994419951843:2089] [node 5] ICP22 created new session: [2:7574435994419951882:2089]
2025-11-19T14:00:07.699205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435994419951882:2089] [node 5] ICS09 handshake done sender: [2:7574435994419951875:2113] self: [2:7574435994419951865:0] peer: [5:7574435992237843906:0] socket: 203 qp: -1
2025-11-19T14:00:07.699209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435994419951882:2089] [node 5] ICS10 traffic start
2025-11-19T14:00:07.699224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:07.699248Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435992419067220:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:07.700332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435992419067179:2078] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.700353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:07.700359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435992419067179:2078] [node 6] ICP22 created new session: [4:7574435992419067234:2078]
2025-11-19T14:00:07.700366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435992419067234:2078] [node 6] ICS09 handshake done sender: [4:7574435992419067220:2112] self: [4:7574435992419067215:0] peer: [6:7574435994151271970:0] socket: 207 qp: -1
2025-11-19T14:00:07.700371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435992419067234:2078] [node 6] ICS10 traffic start
2025-11-19T14:00:07.700389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.704121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994151271977:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61245
2025-11-19T14:00:07.704174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994151271978:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61245
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.704804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435992237843913:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8529
2025-11-19T14:00:07.704824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435992237843914:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8529
2025-11-19T14:00:07.704834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435992237843915:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8529
2025-11-19T14:00:07.704929Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574435992419067147:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(59320177ns) {Configure:104(59319467ns) {ProcessConfigured:135(59055228ns) {StartInitialHandshake:157(59054765ns) {DropHandshakes:469(447ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(187ns) PrepareNewSessionHandshake:476(946ns) {SwitchToState:178(127ns)}}}}}
2025-11-19T14:00:07.706307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435992419067147:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.706837Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435992237843845:2063] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.708120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994151271979:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61245
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.708465Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435991316529724:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.708477Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435991316529724:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.709075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435991316529740:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56378
2025-11-19T14:00:07.709107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435991316529741:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56378
2025-11-19T14:00:07.709470Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435994151271966:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:07.709681Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435992237843905:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.709804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435992237843843:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.709817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:07.709822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435992237843843:2062] [node 6] ICP22 created new session: [5:7574435992237843917:2062]
2025-11-19T14:00:07.709829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435992237843917:2062] [node 6] ICS09 handshake done sender: [5:7574435992237843845:2063] self: [5:7574435992237843844:0] peer: [6:7574435994151271969:0] socket: 180 qp: -1
2025-11-19T14:00:07.709836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435992237843917:2062] [node 6] ICS10 traffic start
2025-11-19T14:00:07.709853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:07.710083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435992237843916:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8529
2025-11-19T14:00:07.710324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435994151271904:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.710337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:07.710343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435994151271904:2061] [node 5] ICP22 created new session: [6:7574435994151271980:2061]
2025-11-19T14:00:07.710348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435994151271980:2061] [node 5] ICS09 handshake done sender: [6:7574435994151271966:2114] self: [6:7574435994151271969:0] peer: [5:7574435992237843844:0] socket: 181 qp: -1
2025-11-19T14:00:07.710352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435994151271980:2061] [node 5] ICS10 traffic start
2025-11-19T14:00:07.710361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:07.710984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435992237843828:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.710996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:07.711001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435992237843828:2049] [node 2] ICP22 created new session: [5:7574435992237843920:2049]
2025-11-19T14:00:07.711005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435992237843920:2049] [node 2] ICS09 handshake done sender: [5:7574435992237843905:2115] self: [5:7574435992237843906:0] peer: [2:7574435994419951865:0] socket: 205 qp: -1
2025-11-19T14:00:07.711010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435992237843920:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:07.711014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.712019Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435991316529724:2111] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:07.713964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435991316529744:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56378
2025-11-19T14:00:07.714293Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435992237843916:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.714325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435992237843827:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.714419Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435991316529735:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.714425Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435991316529735:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.714954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435992237843924:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8529
2025-11-19T14:00:07.715041Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435991316529735:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:07.715201Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435992237843916:2121] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:07.715297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435991316529723:2103] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.715310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:07.715317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435991316529723:2103] [node 5] ICP22 created new session: [1:7574435991316529746:2103]
2025-11-19T14:00:07.715322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435991316529746:2103] [node 5] ICS09 handshake done sender: [1:7574435991316529735:2116] self: [1:7574435991316529729:0] peer: [5:7574435992237843923:0] socket: 237 qp: -1
2025-11-19T14:00:07.715327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435991316529746:2103] [node 5] ICS10 traffic start
2025-11-19T14:00:07.715349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:07.719871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435994419951885:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20059
2025-11-19T14:00:07.719884Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435994419951856:2099] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.719901Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435994419951856:2099] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.719970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435994151271917:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.720016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435994151271917:2067] [node 4] ICP35 event processing took too much time PendingConnection:278(30122082ns) {IncomingHandshake:261(30121078ns) {DropIncomingHandshake:441(122ns) PrepareNewSessionHandshake:476(206ns) {SwitchToState:178(57ns)} GenerateSessionVirtualId:988(220ns) IssueIncomingHandshakeReply:193(30117850ns) {SwitchToState:178(488ns)}}}
2025-11-19T14:00:07.720062Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435994151271968:2116] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:07.716935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435991316529708:2097] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.716956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:07.716965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435991316529708:2097] [node 4] ICP22 created new session: [1:7574435991316529748:2097]
2025-11-19T14:00:07.716971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435991316529748:2097] [node 4] ICS09 handshake done sender: [1:7574435991316529724:2111] self: [1:7574435991316529709:0] peer: [4:7574435992419067232:0] socket: 198 qp: -1
2025-11-19T14:00:07.716978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435991316529748:2097] [node 4] ICS10 traffic start
2025-11-19T14:00:07.717005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:07.720716Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435993547531244:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.720733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435994151271917:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.720750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:07.720755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435994151271917:2067] [node 4] ICP22 created new session: [6:7574435994151271982:2067]
2025-11-19T14:00:07.720761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435994151271982:2067] [node 4] ICS09 handshake done sender: [6:7574435994151271968:2116] self: [6:7574435994151271970:0] peer: [4:7574435992419067215:0] socket: 208 qp: -1
2025-11-19T14:00:07.720767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435994151271982:2067] [node 4] ICS10 traffic start
2025-11-19T14:00:07.720776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.720491Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435992419067179:2078] [node 6] ICP35 event processing took too much time PendingConnection:278(20058217ns) {HandleHandshakeStatus:329(20057489ns) {DropIncomingHandshake:441(4357ns) DropOutgoingHandshake:456(73ns) SwitchToState:178(218ns) ProcessPendingSessionEvents:495(133951ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 111893ns max# 37753ns}}}
2025-11-19T14:00:07.722581Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574435992419067199:2098] [node 5] ICP35 event processing took too much time PendingConnection:278(32963326ns) {HandleHandshakeStatus:329(32962406ns) {DropIncomingHandshake:441(115ns) DropOutgoingHandshake:456(253ns) SwitchToState:178(213ns) ProcessPendingSessionEvents:495(23447898ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23424662ns max# 23360167ns}}}
2025-11-19T14:00:07.723007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435991316529651:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(33331914ns) {HandleHandshakeStatus:329(33331070ns) {DropIncomingHandshake:441(4910ns) DropOutgoingHandshake:456(120ns) SwitchToState:178(83ns) ProcessPendingSessionEvents:495(23854729ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23835579ns max# 23774580ns}}}
2025-11-19T14:00:07.723556Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435991316529740:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.723593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435991316529708:2097] [node 4] ICP18 handshake (actor [1:7574435991316529740:2117]) is too old
2025-11-19T14:00:07.723527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574435992237843852:2067] [node 4] ICP35 event processing took too much time PendingConnection:278(33722656ns) {HandleHandshakeStatus:329(33722191ns) {DropIncomingHandshake:441(100ns) DropOutgoingHandshake:456(1553ns) SwitchToState:178(28ns) ProcessPendingSessionEvents:495(24347077ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24326887ns max# 24269318ns}}}
2025-11-19T14:00:07.723801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435992237843827:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.723813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:07.723820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435992237843827:2048] [node 1] ICP22 created new session: [5:7574435992237843925:2048]
2025-11-19T14:00:07.723825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435992237843925:2048] [node 1] ICS09 handshake done sender: [5:7574435992237843916:2121] self: [5:7574435992237843923:0] peer: [1:7574435991316529729:0] socket: 238 qp: -1
2025-11-19T14:00:07.723832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435992237843925:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:07.723840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-19T14:00:07.724176Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435994419951880:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.724204Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435994419951885:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.724269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435994419951886:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20059
2025-11-19T14:00:07.724323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435994419951887:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20059
2025-11-19T14:00:07.724339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435993547531260:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24903
::ffff:127.1.0.0
2025-11-19T14:00:07.724667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435993547531261:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24903
2025-11-19T14:00:07.724637Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435992419067146:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15068) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435992419067224:2114] held: no
2025-11-19T14:00:07.724914Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435993547531260:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.725466Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435993547531261:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.725582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435994419951799:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.725623Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435994419951886:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.729908Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435991316529741:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.732075Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435994419951870:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.732095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435994419951870:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.732161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435992419067239:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19724
2025-11-19T14:00:07.732332Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435992237843914:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.732378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435992237843829:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.732088Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574435991316529695:2091] [node 6] ICP35 event processing took too much time PendingConnection:278(37048953ns) {HandleHandshakeStatus:329(37048496ns) {DropIncomingHandshake:441(3045ns) DropOutgoingHandshake:456(79ns) SwitchToState:178(60ns) ProcessPendingSessionEvents:495(33066834ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 33043807ns max# 32967615ns}}}
2025-11-19T14:00:07.732408Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435991316529717:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.732414Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435991316529717:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.733146Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435993547531251:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.733158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435993547531251:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.733728Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435992419067224:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:07.733846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435992419067146:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.733861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:07.733868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435992419067146:2048] [node 1] ICP22 created new session: [4:7574435992419067240:2048]
2025-11-19T14:00:07.733875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435992419067240:2048] [node 1] ICS09 handshake done sender: [4:7574435992419067224:2114] self: [4:7574435992419067232:0] peer: [1:7574435991316529709:0] socket: 213 qp: -1
2025-11-19T14:00:07.733883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435992419067240:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:07.733897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:07.734600Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435994151271977:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.734647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435994151271891:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.734222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435994419951843:2089] [node 5] ICP35 event processing took too much time PendingConnection:278(34950717ns) {HandleHandshakeStatus:329(34950114ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(136ns) SwitchToState:178(68ns) ProcessPendingSessionEvents:495(14014729ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13996294ns max# 13934844ns}}}
2025-11-19T14:00:07.734948Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435993547531254:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.734958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435993547531254:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.734793Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435991316529717:2107] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:07.735828Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435994419951856:2099] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:07.737602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435994419951800:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.737620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:07.737625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435994419951800:2049] [node 3] ICP22 created new session: [2:7574435994419951892:2049]
2025-11-19T14:00:07.737630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435994419951892:2049] [node 3] ICS09 handshake done sender: [2:7574435994419951856:2099] self: [2:7574435994419951844:0] peer: [3:7574435993547531256:0] socket: 188 qp: -1
2025-11-19T14:00:07.737635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435994419951892:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:07.737642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:07.735668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435993547531264:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24903
2025-11-19T14:00:07.736344Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435993547531254:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:07.736370Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435993547531228:2102] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:07.736440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435993547531172:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.736454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:07.736459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435993547531172:2049] [node 2] ICP22 created new session: [3:7574435993547531266:2049]
2025-11-19T14:00:07.736464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435993547531266:2049] [node 2] ICS09 handshake done sender: [3:7574435993547531228:2102] self: [3:7574435993547531256:0] peer: [2:7574435994419951844:0] socket: 192 qp: -1
2025-11-19T14:00:07.736471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435993547531266:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:07.736504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:07.736161Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574435992419067199:2098] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.738893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574435992237843843:2062] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.739131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994151271986:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61245
2025-11-19T14:00:07.739312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994151271987:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61245
2025-11-19T14:00:07.739193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435991316529650:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.739211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:07.739217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435991316529650:2048] [node 2] ICP22 created new session: [1:7574435991316529752:2048]
2025-11-19T14:00:07.739222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435991316529752:2048] [node 2] ICS09 handshake done sender: [1:7574435991316529717:2107] self: [1:7574435991316529697:0] peer: [2:7574435994419951888:0] socket: 194 qp: -1
2025-11-19T14:00:07.739227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435991316529752:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:07.739234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:07.739010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435992419067242:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19724
2025-11-19T14:00:07.739934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435993547531232:2086] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.739952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:07.739957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435993547531232:2086] [node 6] ICP22 created new session: [3:7574435993547531268:2086]
2025-11-19T14:00:07.739962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435993547531268:2086] [node 6] ICS09 handshake done sender: [3:7574435993547531254:2117] self: [3:7574435993547531248:0] peer: [6:7574435994151271985:0] socket: 223 qp: -1
2025-11-19T14:00:07.739968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435993547531268:2086] [node 6] ICS10 traffic start
2025-11-19T14:00:07.739973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:07.740035Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435994151271977:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:07.740068Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435994419951870:2110] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:07.740248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435994419951842:2088] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.740259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:07.740263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435994419951842:2088] [node 4] ICP22 created new session: [2:7574435994419951896:2088]
2025-11-19T14:00:07.740267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435994419951896:2088] [node 4] ICS09 handshake done sender: [2:7574435994419951870:2110] self: [2:7574435994419951853:0] peer: [4:7574435992419067237:0] socket: 202 qp: -1
2025-11-19T14:00:07.740271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435994419951896:2088] [node 4] ICS10 traffic start
2025-11-19T14:00:07.740275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.741416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435992237843928:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8529
2025-11-19T14:00:07.747994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435994151271891:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.748033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:07.748045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435994151271891:2050] [node 3] ICP22 created new session: [6:7574435994151271988:2050]
2025-11-19T14:00:07.748055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435994151271988:2050] [node 3] ICS09 handshake done sender: [6:7574435994151271977:2119] self: [6:7574435994151271985:0] peer: [3:7574435993547531248:0] socket: 224 qp: -1
2025-11-19T14:00:07.748067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435994151271988:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:07.748096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:07.748308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574435994151271891:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:07.748810Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435992419067226:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:07.748545Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435993547531251:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.753479Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574435991316529723:2103] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:07.752026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435992419067243:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19724
2025-11-19T14:00:07.755041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435993547531239:2099] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.755073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:07.755080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435993547531239:2099] [node 5] ICP22 created new session: [3:7574435993547531272:2099]
2025-11-19T14:00:07.755084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435993547531272:2099] [node 5] ICS09 handshake done sender: [3:7574435993547531251:2116] self: [3:7574435993547531247:0] peer: [5:7574435992237843927:0] socket: 222 qp: -1
2025-11-19T14:00:07.755091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435993547531272:2099] [node 5] ICS10 traffic start
2025-11-19T14:00:07.755099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:07.755542Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435992419067243:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.755581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435992419067148:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.754294Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435994419951799:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15068) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574435994419951880:2116] held: no
2025-11-19T14:00:07.755912Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435992237843914:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:07.756062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435992237843829:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.756087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:07.756125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435992237843829:2050] [node 3] ICP22 created new session: [5:7574435992237843929:2050]
2025-11-19T14:00:07.756132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435992237843929:2050] [node 3] ICS09 handshake done sender: [5:7574435992237843914:2119] self: [5:7574435992237843927:0] peer: [3:7574435993547531247:0] socket: 232 qp: -1
2025-11-19T14:00:07.756138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435992237843929:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:07.756159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:07.757008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574435993547531232:2086] [node 6] ICP35 event processing took too much time PendingConnection:278(16971075ns) {HandleHandshakeStatus:329(16970469ns) {DropIncomingHandshake:441(2900ns) DropOutgoingHandshake:456(80ns) SwitchToState:178(67ns) ProcessPendingSessionEvents:495(12601116ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12578865ns max# 12503511ns}}}
2025-11-19T14:00:07.756866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435992419067147:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.756889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:07.756894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435992419067147:2049] [node 2] ICP22 created new session: [4:7574435992419067245:2049]
2025-11-19T14:00:07.756900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435992419067245:2049] [node 2] ICS09 handshake done sender: [4:7574435992419067226:2116] self: [4:7574435992419067237:0] peer: [2:7574435994419951853:0] socket: 215 qp: -1
2025-11-19T14:00:07.756906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435992419067245:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:07.756923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:07.757996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574435994419951842:2088] [node 4] ICP35 event processing took too much time PendingConnection:278(17649239ns) {HandleHandshakeStatus:329(17648552ns) {DropIncomingHandshake:441(2217ns) DropOutgoingHandshake:456(85ns) SwitchToState:178(67ns) ProcessPendingSessionEvents:495(13439908ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13425924ns max# 13371209ns}}}
2025-11-19T14:00:07.759209Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435994151271987:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.759254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435994151271890:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:07.759440Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435994419951878:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.759451Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435994419951878:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.761770Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435994419951878:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.761845Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435993547531258:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:07.761858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435993547531258:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:07.762647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435994419951899:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20059
2025-11-19T14:00:07.763446Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435993547531258:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:07.763570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435994419951850:2081] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.763599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:07.763606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435994419951850:2081] [node 6] ICP22 created new session: [2:7574435994419951900:2081]
2025-11-19T14:00:07.763612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435994419951900:2081] [node 6] ICS09 handshake done sender: [2:7574435994419951878:2115] self: [2:7574435994419951874:0] peer: [6:7574435994151271990:0] socket: 247 qp: -1
2025-11-19T14:00:07.763617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435994419951900:2081] [node 6] ICS10 traffic start
2025-11-19T14:00:07.763625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:07.764136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435993547531231:2084] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:07.764153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:07.764158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435993547531231:2084] [node 4] ICP22 created new session: [3:7574435993547531275:2084]
2025-11-19T14:00:07.764163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435993547531275:2084] [node 4] ICS09 handshake done sender: [3:7574435993547531258:2118] self: [3:7574435993547531249:0] peer: [4:7574435992419067244:0] socket: 201 qp: -1
2025-11-19T14:00:07.764168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435993547531275:2084] [node 4] ICS10 traffic start
2025-11-19T14:00:07.764173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:07.764692Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435994419951880:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:07.764879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435994419951799:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.764891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:07.764896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435994419951799:2048] [node 1] ICP22 created new session: [2:7574435994419951902:2048]
2025-11-19T14:00:07.764901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435994419951902:2048] [node 1] ICS09 handshake done sender: [2:7574435994419951880:2116] self: [2:7574435994419951888:0] peer: [1:7574435991316529697:0] socket: 195 qp: -1
2025-11-19T14:00:07.764906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435994419951902:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:07.764915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.765338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435992419067247:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19724
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:07.765487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994151271991:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61245
2025-11-19T14:00:07.765679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435992419067148:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18273) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435992419067243:2121] held: no
2025-11-19T14:00:07.765782Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435992419067243:2121] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:07.766451Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435994151271987:2123] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:07.766715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435994151271890:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23374) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435994151271987:2123] held: no
2025-11-19T14:00:07.766799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435994151271890:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.766809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:07.766814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435994151271890:2049] [node 2] ICP22 created new session: [6:7574435994151271992:2049]
2025-11-19T14:00:07.766819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435994151271992:2049] [node 2] ICS09 handshake done sender: [6:7574435994151271987:2123] self: [6:7574435994151271990:0] peer: [2:7574435994419951874:0] socket: 250 qp: -1
2025-11-19T14:00:07.766826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435994151271992:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:07.766834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:07.767570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435992419067148:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:07.767587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:07.767592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435992419067148:2050] [node 3] ICP22 created new session: [4:7574435992419067248:2050]
2025-11-19T14:00:07.767599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435992419067248:2050] [node 3] ICS09 handshake done sender: [4:7574435992419067243:2121] self: [4:7574435992419067244:0] peer: [3:7574435993547531249:0] socket: 217 qp: -1
2025-11-19T14:00:07.767605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435992419067248:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:07.767616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:07.785846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:07.785947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435993547531252:2048] [node 1] ICS01 socket: 193 reason# EndOfStream
2025-11-19T14:00:07.785958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435993547531171:2048] [node 1] ICP30 unregister session Session# [3:7574435993547531252:2048] VirtualId# [3:7574435993547531240:0]
2025-11-19T14:00:07.785965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435993547531252:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.786007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:07.786031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:07.787991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:07.788082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435994419951902:2048] [node 1] ICS01 socket: 195 reason# EndOfStream
2025-11-19T14:00:07.788091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435994419951799:2048] [node 1] ICP30 unregister session Session# [2:7574435994419951902:2048] VirtualId# [2:7574435994419951888:0]
2025-11-19T14:00:07.788098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435994419951902:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.788157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:07.788190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:07.792075Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:07.792155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435994151271974:2048] [node 1] ICS01 socket: 218 reason# EndOfStream
2025-11-19T14:00:07.792162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435994151271889:2048] [node 1] ICP30 unregister session Session# [6:7574435994151271974:2048] VirtualId# [6:7574435994151271972:0]
2025-11-19T14:00:07.792168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435994151271974:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.792213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:07.792235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:07.792268Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:07.792327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435992419067240:2048] [node 1] ICS01 socket: 213 reason# EndOfStream
2025-11-19T14:00:07.792331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435992419067146:2048] [node 1] ICP30 unregister session Session# [4:7574435992419067240:2048] VirtualId# [4:7574435992419067232:0]
2025-11-19T14:00:07.792335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435992419067240:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.792365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:07.792384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:07.795821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:07.795909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435992237843925:2048] [node 1] ICS01 socket: 238 reason# EndOfStream
2025-11-19T14:00:07.795917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435992237843827:2048] [node 1] ICP30 unregister session Session# [5:7574435992237843925:2048] VirtualId# [5:7574435992237843923:0]
2025-11-19T14:00:07.795923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435992237843925:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.795976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:07.796000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:07.815883Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:07.815982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435993547531266:2049] [node 2] ICS01 socket: 192 reason# EndOfStream
2025-11-19T14:00:07.815994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435993547531172:2049] [node 2] ICP30 unregister session Session# [3:7574435993547531266:2049] VirtualId# [3:7574435993547531256:0]
2025-11-19T14:00:07.816002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435993547531266:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.816056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:07.816088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:07.823194Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:07.823285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435992419067245:2049] [node 2] ICS01 socket: 215 reason# EndOfStream
2025-11-19T14:00:07.823293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435992419067147:2049] [node 2] ICP30 unregister session Session# [4:7574435992419067245:2049] VirtualId# [4:7574435992419067237:0]
2025-11-19T14:00:07.823299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435992419067245:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.823358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:07.823385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:07.823431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:07.823493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435992237843920:2049] [node 2] ICS01 socket: 205 reason# EndOfStream
2025-11-19T14:00:07.823497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435992237843828:2049] [node 2] ICP30 unregister session Session# [5:7574435992237843920:2049] VirtualId# [5:7574435992237843906:0]
2025-11-19T14:00:07.823502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435992237843920:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.823527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:07.823547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:07.823963Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:07.824030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435994151271992:2049] [node 2] ICS01 socket: 250 reason# EndOfStream
2025-11-19T14:00:07.824034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435994151271890:2049] [node 2] ICP30 unregister session Session# [6:7574435994151271992:2049] VirtualId# [6:7574435994151271990:0]
2025-11-19T14:00:07.824039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435994151271992:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.824069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:07.824090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:07.838081Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:07.838212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435994151271988:2050] [node 3] ICS01 socket: 224 reason# EndOfStream
2025-11-19T14:00:07.838220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435994151271891:2050] [node 3] ICP30 unregister session Session# [6:7574435994151271988:2050] VirtualId# [6:7574435994151271985:0]
2025-11-19T14:00:07.838227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435994151271988:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.838272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:07.838296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:07.838328Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:07.838387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574435992237843929:2050] [node 3] ICS07 socket disconnect 232 reason# EndOfStream
2025-11-19T14:00:07.838391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435992237843929:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.838414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:07.838434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574435992237843929:2050] [node 3] ICS15 start handshake
2025-11-19T14:00:07.840389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574435992237843829:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18273) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:07.840405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435992237843929:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:07.840409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435992237843829:2050] [node 3] ICP30 unregister session Session# [5:7574435992237843929:2050] VirtualId# [5:7574435992237843927:0]
2025-11-19T14:00:07.840446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:07.840450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574435992237843829:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18273) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:07.840453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18273) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:07.846390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:07.846470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435994151271982:2067] [node 4] ICS01 socket: 208 reason# EndOfStream
2025-11-19T14:00:07.846477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435994151271917:2067] [node 4] ICP30 unregister session Session# [6:7574435994151271982:2067] VirtualId# [6:7574435994151271970:0]
2025-11-19T14:00:07.846484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435994151271982:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:07.846528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:07.846550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:08.041008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435992290944451:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:08.041026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435992290944452:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:08.107899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435996844419590:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:08.107920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435996844419591:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:08.219843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435995349609304:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:08.219858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435995349609305:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:08.315981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435998343233223:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:08.316000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435998343233224:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:08.316006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435998343233225:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:08.472073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435996475812284:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:08.472090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435996475812285:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:08.472094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435996475812286:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:08.619918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435994945459105:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:08.619941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435994945459106:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:08.619947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435994945459107:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.651272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435998343233252:2071] [node 5] ICP01 ready to work
2025-11-19T14:00:08.651390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574435998343233253:2074] [node 6] ICP01 ready to work
2025-11-19T14:00:08.651400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435996585911809:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:08.651501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435996585911810:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:08.654116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435994945459131:2063] [node 4] ICP01 ready to work
2025-11-19T14:00:08.655671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435996475812313:2070] [node 4] ICP01 ready to work
2025-11-19T14:00:08.652496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435995349609355:2095] [node 4] ICP01 ready to work
2025-11-19T14:00:08.652937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435995349609356:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58913
2025-11-19T14:00:08.656265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435996844419654:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:08.656494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435995349609364:2100] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.657575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435998343233276:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41593
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.657787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435998343233284:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41593
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.687908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574435995349609364:2100] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(31282947ns) {Configure:104(31281892ns) {ProcessConfigured:135(31027020ns) {StartInitialHandshake:157(31026200ns) {DropHandshakes:469(463ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:476(2233ns) {SwitchToState:178(1163ns)}}}}}
2025-11-19T14:00:08.688167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574435995349609371:2103] [node 5] ICP01 ready to work
2025-11-19T14:00:08.688344Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435995349609356:2097] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.689091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435995349609370:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58913
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.705548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435996844419664:2111] [node 5] ICP01 ready to work
2025-11-19T14:00:08.705708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574435996844419665:2112] [node 6] ICP01 ready to work
2025-11-19T14:00:08.706413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435996844419663:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19267
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.707841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574435994945459105:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(53643658ns) {Configure:104(53643119ns) {ProcessConfigured:135(53271898ns) {StartInitialHandshake:157(53271429ns) {DropHandshakes:469(480ns) {DropIncomingHandshake:441(87ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:476(877ns) {SwitchToState:178(407ns)}}}}}
2025-11-19T14:00:08.708918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435994945459177:2079] [node 5] ICP01 ready to work
2025-11-19T14:00:08.709378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994945459178:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12620
2025-11-19T14:00:08.709390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994945459179:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12620
2025-11-19T14:00:08.709781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574435996475812359:2080] [node 6] ICP01 ready to work
2025-11-19T14:00:08.710391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435996475812358:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.721395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435996585911824:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:08.721928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435996585911816:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45324
2025-11-19T14:00:08.721972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435996585911817:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45324
2025-11-19T14:00:08.722002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435996585911818:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45324
2025-11-19T14:00:08.722034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435996585911819:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45324
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.732246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435996475812363:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.736040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435996844419670:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19267
2025-11-19T14:00:08.736385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435995349609386:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58913
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.735931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435998343233276:2095] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.739484Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435998343233284:2098] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.739616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435998343233225:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:08.739647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435998343233303:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41593
2025-11-19T14:00:08.739660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435998343233304:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41593
2025-11-19T14:00:08.739701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435998343233305:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41593
2025-11-19T14:00:08.740082Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435996585911817:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.740117Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435996585911816:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.740133Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435996585911818:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.743867Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435995349609370:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.744008Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435996844419663:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.744242Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435995349609358:2098] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.744249Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435995349609358:2098] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:08.744748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435995349609305:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:08.744895Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435996844419656:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.744901Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435996844419656:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:08.745394Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435996844419670:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.745468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574435996844419590:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:08.745667Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435995349609358:2098] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.745811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435995349609355:2095] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.745826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:08.745832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435995349609355:2095] [node 4] ICP22 created new session: [3:7574435995349609390:2095]
2025-11-19T14:00:08.745838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435995349609390:2095] [node 4] ICS09 handshake done sender: [3:7574435995349609358:2098] self: [3:7574435995349609357:0] peer: [4:7574435998343233306:0] socket: 192 qp: -1
2025-11-19T14:00:08.745846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435995349609390:2095] [node 4] ICS10 traffic start
2025-11-19T14:00:08.745856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:08.746154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435995349609389:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58913
2025-11-19T14:00:08.747776Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435995349609373:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.747783Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435995349609373:2107] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.746303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435996475812364:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-19T14:00:08.747645Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435996475812364:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.747668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435996475812286:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:08.748294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435996475812366:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-19T14:00:08.750450Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435996844419656:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.750607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435996844419591:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.750627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:08.750634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435996844419591:2049] [node 3] ICP22 created new session: [2:7574435996844419674:2049]
2025-11-19T14:00:08.750640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435996844419674:2049] [node 3] ICS09 handshake done sender: [2:7574435996844419656:2109] self: [2:7574435996844419655:0] peer: [3:7574435995349609388:0] socket: 193 qp: -1
2025-11-19T14:00:08.750647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435996844419674:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:08.750656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:08.751106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435996475812367:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-19T14:00:08.752081Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435995349609373:2107] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:08.753348Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435996475812364:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:08.753439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435996475812286:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.753458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:08.753464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435996475812286:2050] [node 3] ICP22 created new session: [5:7574435996475812368:2050]
2025-11-19T14:00:08.753470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435996475812368:2050] [node 3] ICS09 handshake done sender: [5:7574435996475812364:2117] self: [5:7574435996475812365:0] peer: [3:7574435995349609372:0] socket: 226 qp: -1
2025-11-19T14:00:08.753477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435996475812368:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:08.753486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:08.756080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435995349609371:2103] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.756103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:08.756108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435995349609371:2103] [node 5] ICP22 created new session: [3:7574435995349609394:2103]
2025-11-19T14:00:08.756114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435995349609394:2103] [node 5] ICS09 handshake done sender: [3:7574435995349609373:2107] self: [3:7574435995349609372:0] peer: [5:7574435996475812365:0] socket: 225 qp: -1
2025-11-19T14:00:08.756119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435995349609394:2103] [node 5] ICS10 traffic start
2025-11-19T14:00:08.756129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:08.757341Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435995349609370:2106] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:08.758532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435995349609305:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.758548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:08.758553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435995349609305:2049] [node 2] ICP22 created new session: [3:7574435995349609397:2049]
2025-11-19T14:00:08.758558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435995349609397:2049] [node 2] ICS09 handshake done sender: [3:7574435995349609370:2106] self: [3:7574435995349609388:0] peer: [2:7574435996844419655:0] socket: 201 qp: -1
2025-11-19T14:00:08.758563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435995349609397:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:08.758570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.759448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435996844419677:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19267
2025-11-19T14:00:08.759484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435996844419678:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19267
2025-11-19T14:00:08.764395Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435994945459179:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.764505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435994945459107:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:08.765277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435995349609386:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.765311Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435995349609369:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.765314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574435995349609369:2105] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:08.766090Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435996475812367:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.766156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435996475812285:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:08.766259Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435996844419677:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.766288Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435996844419667:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.766291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435996844419667:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.767222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994945459186:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12620
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.767328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435996475812372:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.767409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435998343233224:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:08.767444Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435998343233304:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.767695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435998343233308:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41593
2025-11-19T14:00:08.767930Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435994945459178:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.767989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435994945459131:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:08.768087Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435996475812358:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.768110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435996475812313:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:08.769249Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435995349609369:2105] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:08.769490Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435994945459179:2115] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:08.769589Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435996844419667:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:08.769763Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435996475812367:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:08.769841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435996475812285:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.769874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:08.769881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435996475812285:2049] [node 2] ICP22 created new session: [5:7574435996475812374:2049]
2025-11-19T14:00:08.769888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435996475812374:2049] [node 2] ICS09 handshake done sender: [5:7574435996475812367:2119] self: [5:7574435996475812371:0] peer: [2:7574435996844419666:0] socket: 230 qp: -1
2025-11-19T14:00:08.769895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435996475812374:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:08.769903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:08.771104Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435996844419658:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.771114Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435996844419658:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:08.771547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435996844419664:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.771561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:08.771567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435996844419664:2111] [node 5] ICP22 created new session: [2:7574435996844419681:2111]
2025-11-19T14:00:08.771573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435996844419681:2111] [node 5] ICS09 handshake done sender: [2:7574435996844419667:2114] self: [2:7574435996844419666:0] peer: [5:7574435996475812371:0] socket: 229 qp: -1
2025-11-19T14:00:08.771580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435996844419681:2111] [node 5] ICS10 traffic start
2025-11-19T14:00:08.771589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:08.772246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574435995349609364:2100] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.772268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:08.772273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435995349609364:2100] [node 6] ICP22 created new session: [3:7574435995349609401:2100]
2025-11-19T14:00:08.772278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435995349609401:2100] [node 6] ICS09 handshake done sender: [3:7574435995349609369:2105] self: [3:7574435995349609365:0] peer: [6:7574435994945459185:0] socket: 202 qp: -1
2025-11-19T14:00:08.772284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435995349609401:2100] [node 6] ICS10 traffic start
2025-11-19T14:00:08.772292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:08.772716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435994945459107:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.772734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:08.772740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435994945459107:2050] [node 3] ICP22 created new session: [6:7574435994945459188:2050]
2025-11-19T14:00:08.772746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435994945459188:2050] [node 3] ICS09 handshake done sender: [6:7574435994945459179:2115] self: [6:7574435994945459185:0] peer: [3:7574435995349609365:0] socket: 206 qp: -1
2025-11-19T14:00:08.772753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435994945459188:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:08.772759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:08.773794Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435998343233303:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.773838Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435998343233275:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.773842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435998343233275:2094] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:08.774294Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435998343233255:2076] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.774297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574435998343233255:2076] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:08.774879Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435998343233276:2095] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:08.775363Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435996585911812:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.775375Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435996585911812:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:08.779962Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435996475812363:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.780631Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574435998343233305:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.780647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435998343233225:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.780663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:08.780670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435998343233225:2050] [node 3] ICP22 created new session: [4:7574435998343233312:2050]
2025-11-19T14:00:08.780676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435998343233312:2050] [node 3] ICS09 handshake done sender: [4:7574435998343233276:2095] self: [4:7574435998343233306:0] peer: [3:7574435995349609357:0] socket: 194 qp: -1
2025-11-19T14:00:08.780683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435998343233312:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:08.780690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:08.781626Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435996585911812:2109] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:08.781737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435992290944451:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.781755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:08.781761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435992290944451:2048] [node 2] ICP22 created new session: [1:7574435996585911834:2048]
2025-11-19T14:00:08.781766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435996585911834:2048] [node 2] ICS09 handshake done sender: [1:7574435996585911812:2109] self: [1:7574435996585911811:0] peer: [2:7574435996844419673:0] socket: 213 qp: -1
2025-11-19T14:00:08.781770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435996585911834:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:08.781776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.782292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574435996475812284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:08.782367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435996475812377:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.782786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994945459190:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12620
2025-11-19T14:00:08.782805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994945459191:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12620
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.783257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435996844419684:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19267
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.784516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994945459193:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12620
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.784705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574435996844419685:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19267
2025-11-19T14:00:08.784781Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435996844419658:2110] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:08.785916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435996844419654:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.785936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:08.785943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435996844419654:2108] [node 4] ICP22 created new session: [2:7574435996844419687:2108]
2025-11-19T14:00:08.785949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435996844419687:2108] [node 4] ICS09 handshake done sender: [2:7574435996844419658:2110] self: [2:7574435996844419657:0] peer: [4:7574435998343233307:0] socket: 195 qp: -1
2025-11-19T14:00:08.785955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435996844419687:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:08.785962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:08.786148Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435996844419670:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:08.786339Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435996585911821:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.786349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435996585911821:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:08.786755Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435996585911819:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.788339Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435998343233275:2094] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.788566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435995349609403:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58913
2025-11-19T14:00:08.788746Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435998343233255:2076] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:08.788936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435998343233253:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.788953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:08.788958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435998343233253:2074] [node 6] ICP22 created new session: [4:7574435998343233316:2074]
2025-11-19T14:00:08.788962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435998343233316:2074] [node 6] ICS09 handshake done sender: [4:7574435998343233275:2094] self: [4:7574435998343233260:0] peer: [6:7574435994945459187:0] socket: 199 qp: -1
2025-11-19T14:00:08.788968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435998343233316:2074] [node 6] ICS10 traffic start
2025-11-19T14:00:08.788974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:08.789149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574435998343233252:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.789157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:08.789160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435998343233252:2071] [node 5] ICP22 created new session: [4:7574435998343233318:2071]
2025-11-19T14:00:08.789164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435998343233318:2071] [node 5] ICS09 handshake done sender: [4:7574435998343233255:2076] self: [4:7574435998343233254:0] peer: [5:7574435996475812373:0] socket: 197 qp: -1
2025-11-19T14:00:08.789167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435998343233318:2071] [node 5] ICS10 traffic start
2025-11-19T14:00:08.789171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:08.789334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574435998343233223:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:08.789557Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435996585911821:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:08.789748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435996585911810:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.789762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:08.789766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435996585911810:2108] [node 5] ICP22 created new session: [1:7574435996585911839:2108]
2025-11-19T14:00:08.789771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435996585911839:2108] [node 5] ICS09 handshake done sender: [1:7574435996585911821:2116] self: [1:7574435996585911820:0] peer: [5:7574435996475812378:0] socket: 215 qp: -1
2025-11-19T14:00:08.789775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435996585911839:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:08.789782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:08.792558Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435996475812358:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:08.792723Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435994945459191:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.792745Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435994945459178:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.792838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435996475812379:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-19T14:00:08.792952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435996475812313:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.792965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:08.792970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435996475812313:2070] [node 4] ICP22 created new session: [5:7574435996475812380:2070]
2025-11-19T14:00:08.792975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435996475812380:2070] [node 4] ICS09 handshake done sender: [5:7574435996475812358:2114] self: [5:7574435996475812373:0] peer: [4:7574435998343233254:0] socket: 200 qp: -1
2025-11-19T14:00:08.792980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435996475812380:2070] [node 4] ICS10 traffic start
2025-11-19T14:00:08.792986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:08.793838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435994945459131:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.793854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:08.793859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435994945459131:2063] [node 4] ICP22 created new session: [6:7574435994945459194:2063]
2025-11-19T14:00:08.793864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435994945459194:2063] [node 4] ICS09 handshake done sender: [6:7574435994945459178:2114] self: [6:7574435994945459187:0] peer: [4:7574435998343233260:0] socket: 205 qp: -1
2025-11-19T14:00:08.793869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435994945459194:2063] [node 4] ICS10 traffic start
2025-11-19T14:00:08.793875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:08.794173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435994945459177:2079] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.796044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435998343233315:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41593
2025-11-19T14:00:08.804964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574435996844419590:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.805003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:08.805012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435996844419590:2048] [node 1] ICP22 created new session: [2:7574435996844419689:2048]
2025-11-19T14:00:08.805021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435996844419689:2048] [node 1] ICS09 handshake done sender: [2:7574435996844419670:2116] self: [2:7574435996844419673:0] peer: [1:7574435996585911811:0] socket: 216 qp: -1
2025-11-19T14:00:08.805030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435996844419689:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:08.805040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:08.805203Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574435996844419590:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:08.805459Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435996844419685:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.806710Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435996475812363:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:08.807029Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435994945459193:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.807239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435994945459106:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:08.807547Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435996475812362:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.807565Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574435996475812362:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:08.809199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574435996475812284:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.809219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:08.809226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435996475812284:2048] [node 1] ICP22 created new session: [5:7574435996475812383:2048]
2025-11-19T14:00:08.809233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435996475812383:2048] [node 1] ICS09 handshake done sender: [5:7574435996475812363:2116] self: [5:7574435996475812378:0] peer: [1:7574435996585911820:0] socket: 217 qp: -1
2025-11-19T14:00:08.809239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435996475812383:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:08.809248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:08.810548Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574435998343233224:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17227) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574435998343233284:2098] held: no
2025-11-19T14:00:08.810725Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435998343233284:2098] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:08.810854Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574435996844419669:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.810865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574435996844419669:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:08.813379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435998343233224:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.813401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:08.813408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435998343233224:2049] [node 2] ICP22 created new session: [4:7574435998343233322:2049]
2025-11-19T14:00:08.813415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435998343233322:2049] [node 2] ICS09 handshake done sender: [4:7574435998343233284:2098] self: [4:7574435998343233307:0] peer: [2:7574435996844419657:0] socket: 196 qp: -1
2025-11-19T14:00:08.813421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435998343233322:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:08.813428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:08.815275Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435996585911814:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.815289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435996585911814:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:08.815938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574435995349609403:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.815979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574435995349609304:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.816477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994945459199:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12620
2025-11-19T14:00:08.816613Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574435996844419669:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:08.816837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574435996844419665:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.816856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:08.816863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574435996844419665:2112] [node 6] ICP22 created new session: [2:7574435996844419693:2112]
2025-11-19T14:00:08.816869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574435996844419693:2112] [node 6] ICS09 handshake done sender: [2:7574435996844419669:2115] self: [2:7574435996844419668:0] peer: [6:7574435994945459198:0] socket: 243 qp: -1
2025-11-19T14:00:08.816875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574435996844419693:2112] [node 6] ICS10 traffic start
2025-11-19T14:00:08.816884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:08.817449Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435994945459193:2120] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.817855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574435996475812382:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20533
2025-11-19T14:00:08.818134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435994945459106:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.818156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:08.818161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435994945459106:2049] [node 2] ICP22 created new session: [6:7574435994945459201:2049]
2025-11-19T14:00:08.818167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435994945459201:2049] [node 2] ICS09 handshake done sender: [6:7574435994945459193:2120] self: [6:7574435994945459198:0] peer: [2:7574435996844419668:0] socket: 244 qp: -1
2025-11-19T14:00:08.818172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435994945459201:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:08.818179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.824914Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435996585911823:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.824932Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435996585911823:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:08.826047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435996585911841:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45324
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.826305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574435995349609405:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58913
2025-11-19T14:00:08.826428Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574435996475812362:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:08.826778Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574435996475812382:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.826795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574435996475812359:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.826808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:08.826814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574435996475812359:2080] [node 6] ICP22 created new session: [5:7574435996475812386:2080]
2025-11-19T14:00:08.826821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574435996475812386:2080] [node 6] ICS09 handshake done sender: [5:7574435996475812362:2115] self: [5:7574435996475812361:0] peer: [6:7574435994945459196:0] socket: 231 qp: -1
2025-11-19T14:00:08.826827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574435996475812386:2080] [node 6] ICS10 traffic start
2025-11-19T14:00:08.826837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:08.827063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574435996475812359:2080] [node 6] ICP18 handshake (actor [5:7574435996475812382:2123]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.827672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574435998343233324:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41593
2025-11-19T14:00:08.828705Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435996585911814:2110] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:08.828884Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574435998343233305:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:08.828945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574435998343233223:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.828957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:08.828962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574435998343233223:2048] [node 1] ICP22 created new session: [4:7574435998343233325:2048]
2025-11-19T14:00:08.828967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574435998343233325:2048] [node 1] ICS09 handshake done sender: [4:7574435998343233305:2119] self: [4:7574435998343233320:0] peer: [1:7574435996585911813:0] socket: 220 qp: -1
2025-11-19T14:00:08.828973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574435998343233325:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:08.828979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:08.829956Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435996585911823:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:08.830126Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574435995349609403:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:08.830190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574435995349609304:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.830203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:08.830208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574435995349609304:2048] [node 1] ICP22 created new session: [3:7574435995349609406:2048]
2025-11-19T14:00:08.830214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574435995349609406:2048] [node 1] ICS09 handshake done sender: [3:7574435995349609403:2119] self: [3:7574435995349609404:0] peer: [1:7574435996585911822:0] socket: 246 qp: -1
2025-11-19T14:00:08.830219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574435995349609406:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:08.830226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:08.831451Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435996585911841:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.831472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435996585911809:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.831483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:08.831489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435996585911809:2107] [node 4] ICP22 created new session: [1:7574435996585911845:2107]
2025-11-19T14:00:08.831494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435996585911845:2107] [node 4] ICS09 handshake done sender: [1:7574435996585911814:2110] self: [1:7574435996585911813:0] peer: [4:7574435998343233320:0] socket: 214 qp: -1
2025-11-19T14:00:08.831500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435996585911845:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:08.831507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:08.831875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435992290944452:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.831889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:08.831894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435992290944452:2049] [node 3] ICP22 created new session: [1:7574435996585911847:2049]
2025-11-19T14:00:08.831898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435996585911847:2049] [node 3] ICS09 handshake done sender: [1:7574435996585911823:2117] self: [1:7574435996585911822:0] peer: [3:7574435995349609404:0] socket: 189 qp: -1
2025-11-19T14:00:08.831902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435996585911847:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:08.831908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:08.844763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574435996585911809:2107] [node 4] ICP18 handshake (actor [1:7574435996585911841:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.845433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994945459200:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12620
2025-11-19T14:00:08.845693Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435994945459177:2079] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435994945459191:2119] held: no
2025-11-19T14:00:08.845845Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435994945459191:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.846003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994945459203:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12620
2025-11-19T14:00:08.846043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435994945459177:2079] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.846053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:08.846059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435994945459177:2079] [node 5] ICP22 created new session: [6:7574435994945459204:2079]
2025-11-19T14:00:08.846065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435994945459204:2079] [node 5] ICS09 handshake done sender: [6:7574435994945459191:2119] self: [6:7574435994945459196:0] peer: [5:7574435996475812361:0] socket: 238 qp: -1
2025-11-19T14:00:08.846071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435994945459204:2079] [node 5] ICS10 traffic start
2025-11-19T14:00:08.846078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:08.846553Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435994945459203:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.846576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435994945459105:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:08.846662Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435996585911828:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:08.846670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435996585911828:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:08.847374Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435996585911828:2118] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:08.847463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435996585911824:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:08.847483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:08.847490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435996585911824:2111] [node 6] ICP22 created new session: [1:7574435996585911850:2111]
2025-11-19T14:00:08.847497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435996585911850:2111] [node 6] ICS09 handshake done sender: [1:7574435996585911828:2118] self: [1:7574435996585911827:0] peer: [6:7574435994945459206:0] socket: 219 qp: -1
2025-11-19T14:00:08.847502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435996585911850:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:08.847510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:08.847995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435994945459207:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12620
2025-11-19T14:00:08.848167Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435994945459203:2123] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:08.851822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435994945459105:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:08.851841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:08.851848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435994945459105:2048] [node 1] ICP22 created new session: [6:7574435994945459208:2048]
2025-11-19T14:00:08.851853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435994945459208:2048] [node 1] ICS09 handshake done sender: [6:7574435994945459203:2123] self: [6:7574435994945459206:0] peer: [1:7574435996585911827:0] socket: 191 qp: -1
2025-11-19T14:00:08.851860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435994945459208:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:08.851867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:08.867395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:08.867471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435995349609406:2048] [node 1] ICS01 socket: 246 reason# EndOfStream
2025-11-19T14:00:08.867478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435995349609304:2048] [node 1] ICP30 unregister session Session# [3:7574435995349609406:2048] VirtualId# [3:7574435995349609404:0]
2025-11-19T14:00:08.867486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435995349609406:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:08.867525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:08.867547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:08.865456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:08.865554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574435996844419689:2048] [node 1] ICS07 socket disconnect 216 reason# EndOfStream
2025-11-19T14:00:08.865561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574435996844419689:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:08.865593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:08.865609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435996844419689:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:08.868490Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435996844419590:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3249) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T14:00:08.868504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574435996844419689:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:08.868787Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574435996844419590:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3249) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:08.868796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574435996844419689:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:08.868801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574435996844419590:2048] [node 1] ICP30 unregister session Session# [2:7574435996844419689:2048] VirtualId# [2:7574435996844419673:0]
2025-11-19T14:00:08.868820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:08.868828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574435996844419590:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3249) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:08.868831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3249) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:08.875904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:08.875991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435998343233325:2048] [node 1] ICS01 socket: 220 reason# EndOfStream
2025-11-19T14:00:08.876000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435998343233223:2048] [node 1] ICP30 unregister session Session# [4:7574435998343233325:2048] VirtualId# [4:7574435998343233320:0]
2025-11-19T14:00:08.876006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435998343233325:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:08.876047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:08.876069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:08.876100Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:08.876161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435994945459208:2048] [node 1] ICS01 socket: 191 reason# EndOfStream
2025-11-19T14:00:08.876166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435994945459105:2048] [node 1] ICP30 unregister session Session# [6:7574435994945459208:2048] VirtualId# [6:7574435994945459206:0]
2025-11-19T14:00:08.876171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435994945459208:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:08.876197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:08.876216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:08.876695Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:08.876759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435996475812383:2048] [node 1] ICS01 socket: 217 reason# EndOfStream
2025-11-19T14:00:08.876764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435996475812284:2048] [node 1] ICP30 unregister session Session# [5:7574435996475812383:2048] VirtualId# [5:7574435996475812378:0]
2025-11-19T14:00:08.876769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435996475812383:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:08.876802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:08.876822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:08.888483Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:08.888582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574435995349609397:2049] [node 2] ICS01 socket: 201 reason# EndOfStream
2025-11-19T14:00:08.888591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574435995349609305:2049] [node 2] ICP30 unregister session Session# [3:7574435995349609397:2049] VirtualId# [3:7574435995349609388:0]
2025-11-19T14:00:08.888598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574435995349609397:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:08.888633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:08.888652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:08.909307Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:08.909438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574435998343233322:2049] [node 2] ICS01 socket: 196 reason# EndOfStream
2025-11-19T14:00:08.909450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574435998343233224:2049] [node 2] ICP30 unregister session Session# [4:7574435998343233322:2049] VirtualId# [4:7574435998343233307:0]
2025-11-19T14:00:08.909459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574435998343233322:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:08.909511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:08.909538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:08.909579Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:08.909696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435996475812374:2049] [node 2] ICS01 socket: 230 reason# EndOfStream
2025-11-19T14:00:08.909702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435996475812285:2049] [node 2] ICP30 unregister session Session# [5:7574435996475812374:2049] VirtualId# [5:7574435996475812371:0]
2025-11-19T14:00:08.909706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435996475812374:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:08.909737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:08.909759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:08.909788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:08.909852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435994945459201:2049] [node 2] ICS01 socket: 244 reason# EndOfStream
2025-11-19T14:00:08.909856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435994945459106:2049] [node 2] ICP30 unregister session Session# [6:7574435994945459201:2049] VirtualId# [6:7574435994945459198:0]
2025-11-19T14:00:08.909860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435994945459201:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:08.909885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:08.909905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:08.957803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:08.957908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435994945459188:2050] [node 3] ICS01 socket: 206 reason# EndOfStream
2025-11-19T14:00:08.957919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435994945459107:2050] [node 3] ICP30 unregister session Session# [6:7574435994945459188:2050] VirtualId# [6:7574435994945459185:0]
2025-11-19T14:00:08.957926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435994945459188:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:08.957975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:08.958003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:08.958041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:08.958100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574435996475812368:2050] [node 3] ICS01 socket: 226 reason# EndOfStream
2025-11-19T14:00:08.958104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574435996475812286:2050] [node 3] ICP30 unregister session Session# [5:7574435996475812368:2050] VirtualId# [5:7574435996475812365:0]
2025-11-19T14:00:08.958108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574435996475812368:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:08.958132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:08.958151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:08.976006Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:08.976109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435994945459194:2063] [node 4] ICS01 socket: 205 reason# EndOfStream
2025-11-19T14:00:08.976121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435994945459131:2063] [node 4] ICP30 unregister session Session# [6:7574435994945459194:2063] VirtualId# [6:7574435994945459187:0]
2025-11-19T14:00:08.976129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435994945459194:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:08.976176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:08.976204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:09.076258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435999360775377:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:09.076271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435999360775378:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:09.127891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436003355473303:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:09.128134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436003355473302:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:09.163862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436000325207937:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:09.163878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436000325207938:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:09.203901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436002571392435:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:09.203917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436002571392436:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:09.203922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436002571392437:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:09.286217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436002550440188:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:09.286244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436002550440189:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:09.286250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436002550440190:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:09.307977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435999511695238:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:09.307989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435999511695239:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:09.307993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435999511695240:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:09.358596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436000325207984:2093] [node 5] ICP01 ready to work
2025-11-19T14:00:09.359226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436002550440227:2085] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.359914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436003355473362:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52318
2025-11-19T14:00:09.359938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436002550440232:2087] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.360710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435999511695255:2063] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.361671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435999360775440:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:09.361728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436002571392481:2087] [node 6] ICP01 ready to work
2025-11-19T14:00:09.366612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435999360775436:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18719
2025-11-19T14:00:09.368826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435999511695269:2069] [node 5] ICP01 ready to work
2025-11-19T14:00:09.367979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436003355473365:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:09.368069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436003355473366:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:09.369261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436003355473375:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:09.368953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574435999511695270:2073] [node 4] ICP01 ready to work
2025-11-19T14:00:09.372440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436002571392503:2094] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.373060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436002571392507:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31578
2025-11-19T14:00:09.373079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436002571392508:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31578
2025-11-19T14:00:09.374020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436000325207993:2095] [node 6] ICP01 ready to work
2025-11-19T14:00:09.374501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436000325208001:2102] [node 4] ICP01 ready to work
2025-11-19T14:00:09.385256Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003355473362:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.385447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436003355473381:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52318
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.384285Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435999360775440:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(17603550ns) {Configure:104(17602526ns) {ProcessConfigured:135(17280694ns) {StartInitialHandshake:157(17279845ns) {DropHandshakes:469(616ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(142ns) PrepareNewSessionHandshake:476(1425ns) {SwitchToState:178(768ns)}}}}}
2025-11-19T14:00:09.384842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435999360775447:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:09.384978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574435999360775448:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:09.385662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435999360775449:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18719
2025-11-19T14:00:09.385674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435999360775450:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18719
2025-11-19T14:00:09.392614Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003355473381:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.394781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436003355473302:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(26629415ns) {Configure:104(26628849ns) {ProcessConfigured:135(26259813ns) {StartInitialHandshake:157(26259221ns) {DropHandshakes:469(481ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:476(959ns) {SwitchToState:178(516ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.395047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436000325208017:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53510
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.395658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436002571392515:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31578
2025-11-19T14:00:09.396046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574435999360775377:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(34969142ns) {Configure:104(34968345ns) {ProcessConfigured:135(34633141ns) {StartInitialHandshake:157(34632513ns) {DropHandshakes:469(509ns) {DropIncomingHandshake:441(88ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(48ns) PrepareNewSessionHandshake:476(1152ns) {SwitchToState:178(558ns)}}}}}
2025-11-19T14:00:09.400173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436002550440264:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22255
2025-11-19T14:00:09.400595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436000325208018:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53510
2025-11-19T14:00:09.400698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436000325208016:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53510
2025-11-19T14:00:09.401060Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436000325208016:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.401089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436000325207938:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.409108Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436002550440264:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.409228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436002550440266:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22255
2025-11-19T14:00:09.409279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436002550440267:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22255
2025-11-19T14:00:09.409291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436002550440268:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22255
2025-11-19T14:00:09.409336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436002550440190:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:09.411840Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436000325208018:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.411889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436000325207937:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:09.411872Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435999360775449:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.412004Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435999360775446:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.412009Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435999360775446:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.412673Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003355473372:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.412689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436003355473372:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.412931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436002571392507:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.413234Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436000325208017:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.413716Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436002550440268:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.413743Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436002550440266:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.413762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436002550440188:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:09.413521Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436002571392515:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.413549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436002571392435:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.419375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435999511695317:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.419821Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435999360775450:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.419576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436002550440189:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19306503ns) {Configure:104(19305263ns) {ProcessConfigured:135(18964376ns) {StartInitialHandshake:157(18964013ns) {DropHandshakes:469(1223ns) {DropIncomingHandshake:441(188ns) DropOutgoingHandshake:456(188ns)} GenerateSessionVirtualId:988(311ns) PrepareNewSessionHandshake:476(760ns) {SwitchToState:178(176ns)}}}}}
2025-11-19T14:00:09.419690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436002550440189:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:09.420065Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436002571392508:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.420052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436000325207991:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.420068Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436000325207991:2099] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.420086Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003355473370:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.420099Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436003355473370:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.420129Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435999360775444:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.420141Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435999360775444:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.420210Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436002550440267:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.419434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435999511695318:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-19T14:00:09.419960Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435999511695255:2063] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.420006Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435999511695317:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.420029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435999511695269:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:09.420313Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435999511695318:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.420346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435999511695240:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:09.423795Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435999360775452:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.423811Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435999360775452:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.424545Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436002550440234:2089] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.424560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436002550440234:2089] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.428495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436002550440274:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22255
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.428574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435999511695322:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-19T14:00:09.428882Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435999511695318:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:09.428923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435999511695240:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.428938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:09.428944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435999511695240:2050] [node 3] ICP22 created new session: [6:7574435999511695323:2050]
2025-11-19T14:00:09.428950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435999511695323:2050] [node 3] ICS09 handshake done sender: [6:7574435999511695318:2117] self: [6:7574435999511695321:0] peer: [3:7574436000325208006:0] socket: 227 qp: -1
2025-11-19T14:00:09.428957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435999511695323:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:09.428966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:09.430015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436002571392519:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31578
2025-11-19T14:00:09.431818Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435999360775436:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.431898Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435999360775446:2112] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:09.427776Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436000325208007:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.427792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436000325208007:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.428696Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436000325208007:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:09.429448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436000325207993:2095] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.429457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:09.429461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436000325207993:2095] [node 6] ICP22 created new session: [3:7574436000325208024:2095]
2025-11-19T14:00:09.429465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436000325208024:2095] [node 6] ICS09 handshake done sender: [3:7574436000325208007:2111] self: [3:7574436000325208006:0] peer: [6:7574435999511695321:0] socket: 214 qp: -1
2025-11-19T14:00:09.429469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436000325208024:2095] [node 6] ICS10 traffic start
2025-11-19T14:00:09.429474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:09.429844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436000325208023:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53510
2025-11-19T14:00:09.432533Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435999360775452:2115] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:09.432710Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436002550440268:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.433374Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003355473374:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.433390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436003355473374:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.434204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436003355473385:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52318
2025-11-19T14:00:09.434862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435999360775378:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.434883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:09.434890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435999360775378:2049] [node 3] ICP22 created new session: [1:7574435999360775464:2049]
2025-11-19T14:00:09.434896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435999360775464:2049] [node 3] ICS09 handshake done sender: [1:7574435999360775446:2112] self: [1:7574435999360775445:0] peer: [3:7574436000325208020:0] socket: 220 qp: -1
2025-11-19T14:00:09.434904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435999360775464:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:09.434911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:09.435195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435999360775447:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.435208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:09.435214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435999360775447:2109] [node 5] ICP22 created new session: [1:7574435999360775466:2109]
2025-11-19T14:00:09.435218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435999360775466:2109] [node 5] ICS09 handshake done sender: [1:7574435999360775452:2115] self: [1:7574435999360775451:0] peer: [5:7574436002550440270:0] socket: 221 qp: -1
2025-11-19T14:00:09.435223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435999360775466:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:09.435228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:09.436791Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436003355473370:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.437090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436002550440188:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.437108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:09.437114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436002550440188:2048] [node 1] ICP22 created new session: [5:7574436002550440275:2048]
2025-11-19T14:00:09.437120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436002550440275:2048] [node 1] ICS09 handshake done sender: [5:7574436002550440268:2117] self: [5:7574436002550440270:0] peer: [1:7574435999360775451:0] socket: 226 qp: -1
2025-11-19T14:00:09.437128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436002550440275:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:09.437135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:09.436529Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436000325208018:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:09.436381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435999511695239:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.436423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574435999511695239:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16367854ns) {IncomingHandshake:261(16367454ns) {DropIncomingHandshake:441(289ns) PrepareNewSessionHandshake:476(429ns) {SwitchToState:178(240ns)} GenerateSessionVirtualId:988(216ns) IssueIncomingHandshakeReply:193(16361881ns) {SwitchToState:178(287ns)}}}
2025-11-19T14:00:09.438567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436000325207937:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.438579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:09.438585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436000325207937:2048] [node 1] ICP22 created new session: [3:7574436000325208029:2048]
2025-11-19T14:00:09.438591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436000325208029:2048] [node 1] ICS09 handshake done sender: [3:7574436000325208018:2117] self: [3:7574436000325208020:0] peer: [1:7574435999360775445:0] socket: 223 qp: -1
2025-11-19T14:00:09.438596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436000325208029:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:09.438603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:09.439551Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435999360775444:2111] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:09.439054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436000325208028:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53510
2025-11-19T14:00:09.440042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435999360775440:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.440059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:09.440065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435999360775440:2108] [node 4] ICP22 created new session: [1:7574435999360775471:2108]
2025-11-19T14:00:09.440070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435999360775471:2108] [node 4] ICS09 handshake done sender: [1:7574435999360775444:2111] self: [1:7574435999360775441:0] peer: [4:7574436002571392517:0] socket: 219 qp: -1
2025-11-19T14:00:09.440074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435999360775471:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:09.440081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:09.439737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574436002550440188:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:09.439896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436002550440277:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22255
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.442575Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436003355473374:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:09.442054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436002571392435:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8009) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436002571392515:2117] held: no
2025-11-19T14:00:09.442210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436002571392520:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31578
2025-11-19T14:00:09.443320Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436002571392519:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.443338Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436002571392515:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:09.443832Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436003355473372:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.444505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436002550440278:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22255
2025-11-19T14:00:09.445083Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436002550440234:2089] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:09.445453Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003355473385:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.445469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436003355473365:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.445479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:09.445486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436003355473365:2108] [node 5] ICP22 created new session: [2:7574436003355473389:2108]
2025-11-19T14:00:09.445491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436003355473389:2108] [node 5] ICS09 handshake done sender: [2:7574436003355473370:2111] self: [2:7574436003355473369:0] peer: [5:7574436002550440272:0] socket: 202 qp: -1
2025-11-19T14:00:09.445497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436003355473389:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:09.445504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:09.446939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436002550440232:2087] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.446956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:09.446962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436002550440232:2087] [node 6] ICP22 created new session: [5:7574436002550440280:2087]
2025-11-19T14:00:09.446967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436002550440280:2087] [node 6] ICS09 handshake done sender: [5:7574436002550440234:2089] self: [5:7574436002550440233:0] peer: [6:7574435999511695319:0] socket: 195 qp: -1
2025-11-19T14:00:09.446972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436002550440280:2087] [node 6] ICS10 traffic start
2025-11-19T14:00:09.446978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:09.447287Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436002550440266:2115] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:09.447415Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436002550440264:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:09.442334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435999511695326:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-19T14:00:09.444595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436002571392435:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.444611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:09.444616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436002571392435:2048] [node 1] ICP22 created new session: [4:7574436002571392521:2048]
2025-11-19T14:00:09.442351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435999511695327:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-19T14:00:09.444621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436002571392521:2048] [node 1] ICS09 handshake done sender: [4:7574436002571392515:2117] self: [4:7574436002571392517:0] peer: [1:7574435999360775441:0] socket: 224 qp: -1
2025-11-19T14:00:09.442360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435999511695328:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-19T14:00:09.444626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436002571392521:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:09.443168Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435999511695317:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:09.444633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:09.443402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435999511695239:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.444816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436002571392437:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:09.443413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:09.443417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435999511695239:2049] [node 2] ICP22 created new session: [6:7574435999511695329:2049]
2025-11-19T14:00:09.443422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435999511695329:2049] [node 2] ICS09 handshake done sender: [6:7574435999511695317:2116] self: [6:7574435999511695320:0] peer: [2:7574436003355473373:0] socket: 210 qp: -1
2025-11-19T14:00:09.443427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435999511695329:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:09.443433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:09.445253Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435999511695255:2063] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:09.445316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435999511695269:2069] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.445324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:09.445328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435999511695269:2069] [node 5] ICP22 created new session: [6:7574435999511695331:2069]
2025-11-19T14:00:09.445333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435999511695331:2069] [node 5] ICS09 handshake done sender: [6:7574435999511695255:2063] self: [6:7574435999511695319:0] peer: [5:7574436002550440233:0] socket: 196 qp: -1
2025-11-19T14:00:09.445338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435999511695331:2069] [node 5] ICS10 traffic start
2025-11-19T14:00:09.445344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:09.442466Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436000325207991:2099] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:09.442798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436000325207984:2093] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.442806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:09.442812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436000325207984:2093] [node 5] ICP22 created new session: [3:7574436000325208031:2093]
2025-11-19T14:00:09.442817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436000325208031:2093] [node 5] ICS09 handshake done sender: [3:7574436000325207991:2099] self: [3:7574436000325207985:0] peer: [5:7574436002550440269:0] socket: 200 qp: -1
2025-11-19T14:00:09.442821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436000325208031:2093] [node 5] ICS10 traffic start
2025-11-19T14:00:09.442825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:09.447375Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436000325208016:2115] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:09.447668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436002550440189:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.447678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:09.447683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436002550440189:2049] [node 2] ICP22 created new session: [5:7574436002550440282:2049]
2025-11-19T14:00:09.447688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436002550440282:2049] [node 2] ICS09 handshake done sender: [5:7574436002550440266:2115] self: [5:7574436002550440272:0] peer: [2:7574436003355473369:0] socket: 213 qp: -1
2025-11-19T14:00:09.447692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436002550440282:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:09.447696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:09.447985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436002550440190:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.447991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:09.447997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436002550440190:2050] [node 3] ICP22 created new session: [5:7574436002550440284:2050]
2025-11-19T14:00:09.448002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436002550440284:2050] [node 3] ICS09 handshake done sender: [5:7574436002550440264:2114] self: [5:7574436002550440269:0] peer: [3:7574436000325207985:0] socket: 201 qp: -1
2025-11-19T14:00:09.448005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436002550440284:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:09.448012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:09.447876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436003355473366:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.447890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:09.447896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436003355473366:2109] [node 6] ICP22 created new session: [2:7574436003355473391:2109]
2025-11-19T14:00:09.447901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436003355473391:2109] [node 6] ICS09 handshake done sender: [2:7574436003355473374:2113] self: [2:7574436003355473373:0] peer: [6:7574435999511695320:0] socket: 204 qp: -1
2025-11-19T14:00:09.447906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436003355473391:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:09.447913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:09.448254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436003355473303:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.448264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:09.448267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436003355473303:2049] [node 3] ICP22 created new session: [2:7574436003355473393:2049]
2025-11-19T14:00:09.448271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436003355473393:2049] [node 3] ICS09 handshake done sender: [2:7574436003355473372:2112] self: [2:7574436003355473371:0] peer: [3:7574436000325208019:0] socket: 203 qp: -1
2025-11-19T14:00:09.448274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436003355473393:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:09.448279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:09.448659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436003355473366:2109] [node 6] ICP18 handshake (actor [2:7574436003355473385:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.448724Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436000325208011:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.448736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436000325208011:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.449458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436000325207938:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.449467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:09.449472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436000325207938:2049] [node 2] ICP22 created new session: [3:7574436000325208033:2049]
2025-11-19T14:00:09.449476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436000325208033:2049] [node 2] ICS09 handshake done sender: [3:7574436000325208016:2115] self: [3:7574436000325208019:0] peer: [2:7574436003355473371:0] socket: 211 qp: -1
2025-11-19T14:00:09.449481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436000325208033:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:09.449487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.452333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435999511695333:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.459984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436002571392525:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31578
2025-11-19T14:00:09.460034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436002571392526:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31578
2025-11-19T14:00:09.453087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436003355473397:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52318
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.460254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574435999360775473:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18719
2025-11-19T14:00:09.460657Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003355473397:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.460724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436003355473302:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:09.460852Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435999360775473:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.460885Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435999360775458:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.460889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435999360775458:2117] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.461796Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435999360775458:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:09.461890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435999360775377:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.461914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:09.461920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435999360775377:2048] [node 2] ICP22 created new session: [1:7574435999360775476:2048]
2025-11-19T14:00:09.461927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435999360775476:2048] [node 2] ICS09 handshake done sender: [1:7574435999360775458:2117] self: [1:7574435999360775439:0] peer: [2:7574436003355473398:0] socket: 249 qp: -1
2025-11-19T14:00:09.461933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435999360775476:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:09.461941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.462429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003355473377:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.462438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436003355473377:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.463078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436003355473399:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52318
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.463449Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435999511695333:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.463489Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574435999511695328:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.463515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435999511695270:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:09.463527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574435999511695238:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:09.461622Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436002571392525:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.461656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436002571392436:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:09.463250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436002571392528:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31578
2025-11-19T14:00:09.463691Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436002571392501:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.463695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436002571392501:2107] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.466667Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574435999360775454:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.466677Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574435999360775454:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.467373Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574435999360775454:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:09.464622Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436002571392501:2107] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:09.464733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436002571392481:2087] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.464751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:09.464757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436002571392481:2087] [node 6] ICP22 created new session: [4:7574436002571392530:2087]
2025-11-19T14:00:09.464765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436002571392530:2087] [node 6] ICS09 handshake done sender: [4:7574436002571392501:2107] self: [4:7574436002571392500:0] peer: [6:7574435999511695334:0] socket: 207 qp: -1
2025-11-19T14:00:09.464772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436002571392530:2087] [node 6] ICS10 traffic start
2025-11-19T14:00:09.464780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:09.467948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435999511695270:2073] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23163) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435999511695333:2122] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.468376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436002550440287:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22255
2025-11-19T14:00:09.470068Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436002550440287:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.470104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436002550440227:2085] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:09.470289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574435999360775448:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.470308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:09.470315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574435999360775448:2110] [node 6] ICP22 created new session: [1:7574435999360775479:2110]
2025-11-19T14:00:09.470322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574435999360775479:2110] [node 6] ICS09 handshake done sender: [1:7574435999360775454:2116] self: [1:7574435999360775453:0] peer: [6:7574435999511695335:0] socket: 248 qp: -1
2025-11-19T14:00:09.470329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574435999360775479:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:09.470337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:09.471161Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436002571392505:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:09.471175Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436002571392505:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:09.471722Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436002571392519:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:09.469912Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436000325208011:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:09.471888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436000325208001:2102] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.471903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:09.471909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436000325208001:2102] [node 4] ICP22 created new session: [3:7574436000325208036:2102]
2025-11-19T14:00:09.471914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436000325208036:2102] [node 4] ICS09 handshake done sender: [3:7574436000325208011:2113] self: [3:7574436000325208010:0] peer: [4:7574436002571392523:0] socket: 235 qp: -1
2025-11-19T14:00:09.471919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436000325208036:2102] [node 4] ICS10 traffic start
2025-11-19T14:00:09.471928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.473545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436002550440289:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22255
2025-11-19T14:00:09.473765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436003355473302:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8009) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436003355473397:2118] held: no
2025-11-19T14:00:09.473913Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436003355473377:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:09.474928Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436003355473397:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:09.475339Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436002550440287:2122] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:09.474038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436002571392437:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1745) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436002571392519:2118] held: no
2025-11-19T14:00:09.474125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436002571392437:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.474134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:09.474140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436002571392437:2050] [node 3] ICP22 created new session: [4:7574436002571392534:2050]
2025-11-19T14:00:09.474149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436002571392534:2050] [node 3] ICS09 handshake done sender: [4:7574436002571392519:2118] self: [4:7574436002571392523:0] peer: [3:7574436000325208010:0] socket: 236 qp: -1
2025-11-19T14:00:09.474156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436002571392534:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:09.474163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:09.475444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436002550440227:2085] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.475461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:09.475466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436002550440227:2085] [node 4] ICP22 created new session: [5:7574436002550440290:2085]
2025-11-19T14:00:09.475473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436002550440290:2085] [node 4] ICS09 handshake done sender: [5:7574436002550440287:2122] self: [5:7574436002550440288:0] peer: [4:7574436002571392504:0] socket: 260 qp: -1
2025-11-19T14:00:09.475479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436002550440290:2085] [node 4] ICS10 traffic start
2025-11-19T14:00:09.475486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:09.476747Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436002571392505:2108] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:09.488219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435999511695336:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-19T14:00:09.488376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574435999511695337:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46872
2025-11-19T14:00:09.488522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436003355473375:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.488553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:09.488563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436003355473375:2110] [node 4] ICP22 created new session: [2:7574436003355473401:2110]
2025-11-19T14:00:09.488570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436003355473401:2110] [node 4] ICS09 handshake done sender: [2:7574436003355473377:2114] self: [2:7574436003355473376:0] peer: [4:7574436002571392527:0] socket: 205 qp: -1
2025-11-19T14:00:09.488578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436003355473401:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:09.488589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:09.488826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436003355473302:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.488832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:09.488835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436003355473302:2048] [node 1] ICP22 created new session: [2:7574436003355473403:2048]
2025-11-19T14:00:09.488839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436003355473403:2048] [node 1] ICS09 handshake done sender: [2:7574436003355473397:2118] self: [2:7574436003355473398:0] peer: [1:7574435999360775439:0] socket: 251 qp: -1
2025-11-19T14:00:09.488842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436003355473403:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:09.488848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:09.492194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436002571392503:2094] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:09.492220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:09.492227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436002571392503:2094] [node 5] ICP22 created new session: [4:7574436002571392536:2094]
2025-11-19T14:00:09.492233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436002571392536:2094] [node 5] ICS09 handshake done sender: [4:7574436002571392505:2108] self: [4:7574436002571392504:0] peer: [5:7574436002550440288:0] socket: 209 qp: -1
2025-11-19T14:00:09.492238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436002571392536:2094] [node 5] ICS10 traffic start
2025-11-19T14:00:09.492245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:09.492342Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436002571392525:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:09.493261Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574435999511695238:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8009) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574435999511695328:2121] held: no
2025-11-19T14:00:09.493347Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435999511695333:2122] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:09.492764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436002571392436:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.492779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:09.492785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436002571392436:2049] [node 2] ICP22 created new session: [4:7574436002571392538:2049]
2025-11-19T14:00:09.492792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436002571392538:2049] [node 2] ICS09 handshake done sender: [4:7574436002571392525:2120] self: [4:7574436002571392527:0] peer: [2:7574436003355473376:0] socket: 234 qp: -1
2025-11-19T14:00:09.492797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436002571392538:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:09.492803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:09.494756Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574435999511695328:2121] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:09.499979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435999511695270:2073] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.499998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:09.500007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435999511695270:2073] [node 4] ICP22 created new session: [6:7574435999511695338:2073]
2025-11-19T14:00:09.500014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435999511695338:2073] [node 4] ICS09 handshake done sender: [6:7574435999511695333:2122] self: [6:7574435999511695334:0] peer: [4:7574436002571392500:0] socket: 218 qp: -1
2025-11-19T14:00:09.500025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435999511695338:2073] [node 4] ICS10 traffic start
2025-11-19T14:00:09.500036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:09.503967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574435999511695238:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:09.503999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:09.504009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574435999511695238:2048] [node 1] ICP22 created new session: [6:7574435999511695340:2048]
2025-11-19T14:00:09.504017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574435999511695340:2048] [node 1] ICS09 handshake done sender: [6:7574435999511695328:2121] self: [6:7574435999511695335:0] peer: [1:7574435999360775453:0] socket: 250 qp: -1
2025-11-19T14:00:09.504026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574435999511695340:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:09.504045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:09.527108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:09.527216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436003355473403:2048] [node 1] ICS01 socket: 251 reason# EndOfStream
2025-11-19T14:00:09.527228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436003355473302:2048] [node 1] ICP30 unregister session Session# [2:7574436003355473403:2048] VirtualId# [2:7574436003355473398:0]
2025-11-19T14:00:09.527235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436003355473403:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:09.527274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:09.527296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:09.531816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:09.531929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436000325208029:2048] [node 1] ICS01 socket: 223 reason# EndOfStream
2025-11-19T14:00:09.531940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436000325207937:2048] [node 1] ICP30 unregister session Session# [3:7574436000325208029:2048] VirtualId# [3:7574436000325208020:0]
2025-11-19T14:00:09.531947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436000325208029:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:09.532003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:09.532037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:09.536127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:09.536211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435999511695340:2048] [node 1] ICS01 socket: 250 reason# EndOfStream
2025-11-19T14:00:09.536220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435999511695238:2048] [node 1] ICP30 unregister session Session# [6:7574435999511695340:2048] VirtualId# [6:7574435999511695335:0]
2025-11-19T14:00:09.536227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435999511695340:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:09.536275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:09.536296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:09.537462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:09.537532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436002571392521:2048] [node 1] ICS01 socket: 224 reason# EndOfStream
2025-11-19T14:00:09.537538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436002571392435:2048] [node 1] ICP30 unregister session Session# [4:7574436002571392521:2048] VirtualId# [4:7574436002571392517:0]
2025-11-19T14:00:09.537543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436002571392521:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:09.537585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:09.537605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:09.537634Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:09.537703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436002550440275:2048] [node 1] ICS01 socket: 226 reason# EndOfStream
2025-11-19T14:00:09.537707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436002550440188:2048] [node 1] ICP30 unregister session Session# [5:7574436002550440275:2048] VirtualId# [5:7574436002550440270:0]
2025-11-19T14:00:09.537711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436002550440275:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:09.537734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:09.537758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:09.543923Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:09.544008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436000325208033:2049] [node 2] ICS01 socket: 211 reason# EndOfStream
2025-11-19T14:00:09.544016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436000325207938:2049] [node 2] ICP30 unregister session Session# [3:7574436000325208033:2049] VirtualId# [3:7574436000325208019:0]
2025-11-19T14:00:09.544022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436000325208033:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:09.544070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:09.544094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:09.558366Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:09.558538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:09.558614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436002571392538:2049] [node 2] ICS01 socket: 234 reason# EndOfStream
2025-11-19T14:00:09.558620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436002571392436:2049] [node 2] ICP30 unregister session Session# [4:7574436002571392538:2049] VirtualId# [4:7574436002571392527:0]
2025-11-19T14:00:09.558624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436002571392538:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:09.558655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:09.558677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:09.558474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436002550440282:2049] [node 2] ICS01 socket: 213 reason# EndOfStream
2025-11-19T14:00:09.558487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436002550440189:2049] [node 2] ICP30 unregister session Session# [5:7574436002550440282:2049] VirtualId# [5:7574436002550440272:0]
2025-11-19T14:00:09.558494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436002550440282:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:09.558539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:09.558565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:09.563517Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:09.563607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435999511695329:2049] [node 2] ICS01 socket: 210 reason# EndOfStream
2025-11-19T14:00:09.563614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435999511695239:2049] [node 2] ICP30 unregister session Session# [6:7574435999511695329:2049] VirtualId# [6:7574435999511695320:0]
2025-11-19T14:00:09.563621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435999511695329:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:09.563668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:09.563691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:09.590085Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:09.590202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436002550440284:2050] [node 3] ICS01 socket: 201 reason# EndOfStream
2025-11-19T14:00:09.590213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436002550440190:2050] [node 3] ICP30 unregister session Session# [5:7574436002550440284:2050] VirtualId# [5:7574436002550440269:0]
2025-11-19T14:00:09.590222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436002550440284:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:09.590285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:09.590315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:09.590354Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:09.590421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435999511695323:2050] [node 3] ICS01 socket: 227 reason# EndOfStream
2025-11-19T14:00:09.590426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435999511695240:2050] [node 3] ICP30 unregister session Session# [6:7574435999511695323:2050] VirtualId# [6:7574435999511695321:0]
2025-11-19T14:00:09.590432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435999511695323:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:09.590460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:09.590480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:09.612189Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:09.612298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574435999511695338:2073] [node 4] ICS01 socket: 218 reason# EndOfStream
2025-11-19T14:00:09.612308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574435999511695270:2073] [node 4] ICP30 unregister session Session# [6:7574435999511695338:2073] VirtualId# [6:7574435999511695334:0]
2025-11-19T14:00:09.612317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574435999511695338:2073] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:09.612366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:09.612389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:09.700367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436002823280722:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:09.700379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436002823280723:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:09.755878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436003198712147:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:09.755902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436003198712148:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:09.795909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436000881050519:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:09.795921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436000881050520:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:09.888420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436003383607612:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:09.888449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436003383607613:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:09.888454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436003383607614:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:09.933338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436003130458487:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:09.933354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436003130458488:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:09.933360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436003130458489:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:10.013432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436006350843307:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:10.013448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436006350843308:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:10.013452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436006350843309:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:10.088243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436006350843350:2089] [node 5] ICP01 ready to work
2025-11-19T14:00:10.088419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436007118248079:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.089741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436007678574969:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25467
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.091490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436007425425844:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:836
2025-11-19T14:00:10.089628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005176017875:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14459
2025-11-19T14:00:10.093057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436006350843358:2093] [node 4] ICP01 ready to work
2025-11-19T14:00:10.092095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436007493679507:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:10.092162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436007118248090:2111] [node 5] ICP01 ready to work
2025-11-19T14:00:10.092260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436007118248091:2112] [node 6] ICP01 ready to work
2025-11-19T14:00:10.095965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436007493679512:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:10.095492Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436007425425844:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.095688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436007425425846:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:10.096106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436007425425846:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.096258Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005176017875:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.097757Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436007678574969:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.098174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436003383607612:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.101590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436007678574971:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25467
2025-11-19T14:00:10.103502Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436006350843354:2091] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.103524Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574436006350843354:2091] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.104219Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436007678574971:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.104444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436007678574973:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:10.104765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436007678574973:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.103431Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436007118248083:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.103453Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436007118248083:2109] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.108372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436007425425848:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:836
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.109738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436007493679517:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52546
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.110454Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436007118248081:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.110471Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436007118248081:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.108517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005176017877:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14459
2025-11-19T14:00:10.108600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005176017878:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14459
2025-11-19T14:00:10.110434Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005176017877:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.111463Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436006350843372:2105] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.111483Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574436006350843372:2105] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.112090Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436007425425848:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.112444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436003130458488:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.113032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436007493679519:2113] [node 6] ICP01 ready to work
2025-11-19T14:00:10.114631Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436007118248081:2108] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:10.112582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436007678574975:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25467
2025-11-19T14:00:10.112623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436000881050519:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.112675Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436000881050519:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16353365ns) {Configure:104(16352945ns) {ProcessConfigured:135(15997275ns) {IncomingHandshake:261(15986648ns) {DropIncomingHandshake:441(65ns) PrepareNewSessionHandshake:476(311ns) {SwitchToState:178(99ns)} GenerateSessionVirtualId:988(320ns) IssueIncomingHandshakeReply:193(15982650ns) {SwitchToState:178(787ns)}}}}}
2025-11-19T14:00:10.112860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436005176017882:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:10.114421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436005176017885:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:10.116329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436002823280723:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.116353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:10.116359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436002823280723:2049] [node 3] ICP22 created new session: [1:7574436007118248097:2049]
2025-11-19T14:00:10.116369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436007118248097:2049] [node 3] ICS09 handshake done sender: [1:7574436007118248081:2108] self: [1:7574436007118248080:0] peer: [3:7574436005176017876:0] socket: 198 qp: -1
2025-11-19T14:00:10.116378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436007118248097:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:10.116402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:10.117546Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436007118248083:2109] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:10.117014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436000881050520:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.117078Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005176017878:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.117244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436005176017886:2112] [node 5] ICP01 ready to work
2025-11-19T14:00:10.117646Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436007493679509:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.117655Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436007493679509:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.124523Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436006350843372:2105] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:10.124666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574436006350843358:2093] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.124683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:10.124690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436006350843358:2093] [node 4] ICP22 created new session: [6:7574436006350843386:2093]
2025-11-19T14:00:10.124696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436006350843386:2093] [node 4] ICS09 handshake done sender: [6:7574436006350843372:2105] self: [6:7574436006350843362:0] peer: [4:7574436007678574974:0] socket: 211 qp: -1
2025-11-19T14:00:10.124704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436006350843386:2093] [node 4] ICS10 traffic start
2025-11-19T14:00:10.124713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:10.127582Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436007493679505:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.127597Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436007493679505:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.128619Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436003130458487:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16458276ns) {Configure:104(16457259ns) {ProcessConfigured:135(16111397ns) {StartInitialHandshake:157(16110582ns) {DropHandshakes:469(1000ns) {DropIncomingHandshake:441(150ns) DropOutgoingHandshake:456(355ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:476(1071ns) {SwitchToState:178(38ns)}}}}}
2025-11-19T14:00:10.131837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436007118248091:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(36951644ns) {Configure:104(36951070ns) {ProcessConfigured:135(36754788ns) {StartInitialHandshake:157(36754304ns) {DropHandshakes:469(203ns) {DropIncomingHandshake:441(38ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(131ns) PrepareNewSessionHandshake:476(2045ns) {SwitchToState:178(1435ns)}}}}}
2025-11-19T14:00:10.131885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436007118248079:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.131900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:10.131910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436007118248079:2107] [node 4] ICP22 created new session: [1:7574436007118248102:2107]
2025-11-19T14:00:10.131917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436007118248102:2107] [node 4] ICS09 handshake done sender: [1:7574436007118248083:2109] self: [1:7574436007118248082:0] peer: [4:7574436007678574970:0] socket: 199 qp: -1
2025-11-19T14:00:10.131926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436007118248102:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:10.131950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.134116Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436005176017882:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21206006ns) {Configure:104(21205350ns) {ProcessConfigured:135(20939341ns) {StartInitialHandshake:157(20938847ns) {DropHandshakes:469(567ns) {DropIncomingHandshake:441(91ns) DropOutgoingHandshake:456(107ns)} GenerateSessionVirtualId:988(57ns) PrepareNewSessionHandshake:476(1360ns) {SwitchToState:178(1108ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.136060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436006350843389:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47400
2025-11-19T14:00:10.136082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436006350843390:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47400
2025-11-19T14:00:10.133688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436007118248104:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26659
2025-11-19T14:00:10.133744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436007118248105:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26659
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.140595Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436007118248104:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.139885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436003383607614:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(26066944ns) {Configure:104(26066442ns) {ProcessConfigured:135(25773012ns) {StartInitialHandshake:157(25771958ns) {DropHandshakes:469(675ns) {DropIncomingHandshake:441(82ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:476(2212ns) {SwitchToState:178(1289ns)}}}}}
2025-11-19T14:00:10.140158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436007678574982:2113] [node 5] ICP01 ready to work
2025-11-19T14:00:10.143849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005176017894:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14459
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.146345Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436006350843354:2091] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:10.145656Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436007493679517:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.146178Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436003198712147:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(48699402ns) {Configure:104(48698836ns) {ProcessConfigured:135(48407179ns) {StartInitialHandshake:157(48406767ns) {DropHandshakes:469(643ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:476(958ns) {SwitchToState:178(777ns)}}}}}
2025-11-19T14:00:10.146287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436003198712147:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.145006Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436005176017885:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(27706013ns) {Configure:104(27705419ns) {ProcessConfigured:135(27438075ns) {StartInitialHandshake:157(27437122ns) {DropHandshakes:469(567ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(75ns) PrepareNewSessionHandshake:476(619ns) {SwitchToState:178(276ns)}}}}}
2025-11-19T14:00:10.145111Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436005176017875:2107] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:10.146510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574436006350843350:2089] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.146529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:10.146536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436006350843350:2089] [node 5] ICP22 created new session: [6:7574436006350843393:2089]
2025-11-19T14:00:10.146542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436006350843393:2089] [node 5] ICS09 handshake done sender: [6:7574436006350843354:2091] self: [6:7574436006350843353:0] peer: [5:7574436007425425847:0] socket: 202 qp: -1
2025-11-19T14:00:10.146549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436006350843393:2089] [node 5] ICS10 traffic start
2025-11-19T14:00:10.146559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:10.148368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436007493679519:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20263502ns) {Configure:104(20262328ns) {ProcessConfigured:135(19935379ns) {StartInitialHandshake:157(19934675ns) {DropHandshakes:469(948ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(367ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:476(978ns) {SwitchToState:178(672ns)}}}}}
2025-11-19T14:00:10.149142Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436007493679505:2108] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:10.148620Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436007678574975:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.149825Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436007493679509:2109] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:10.150320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436007493679522:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52546
2025-11-19T14:00:10.150376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436003198712148:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.150393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:10.150404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436003198712148:2049] [node 3] ICP22 created new session: [2:7574436007493679527:2049]
2025-11-19T14:00:10.150410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436007493679527:2049] [node 3] ICS09 handshake done sender: [2:7574436007493679505:2108] self: [2:7574436007493679503:0] peer: [3:7574436005176017879:0] socket: 204 qp: -1
2025-11-19T14:00:10.150421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436007493679527:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:10.150430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.150573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436007425425858:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:836
2025-11-19T14:00:10.150751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436003383607613:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.151924Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436007118248087:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.151943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436007118248087:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.150029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005176017895:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14459
2025-11-19T14:00:10.153961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436000881050519:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.153989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:10.153998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436000881050519:2048] [node 1] ICP22 created new session: [3:7574436005176017897:2048]
2025-11-19T14:00:10.154009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436005176017897:2048] [node 1] ICS09 handshake done sender: [3:7574436005176017875:2107] self: [3:7574436005176017876:0] peer: [1:7574436007118248080:0] socket: 200 qp: -1
2025-11-19T14:00:10.154018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436005176017897:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:10.154059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.159175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436007678574986:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25467
2025-11-19T14:00:10.160662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436007425425859:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:836
2025-11-19T14:00:10.161439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436007425425857:2111] [node 4] ICP01 ready to work
2025-11-19T14:00:10.164207Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436006350843389:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.164273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436006350843307:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.163822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436007425425860:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:836
2025-11-19T14:00:10.166359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436007493679507:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.166383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:10.166391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436007493679507:2107] [node 5] ICP22 created new session: [2:7574436007493679530:2107]
2025-11-19T14:00:10.166399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436007493679530:2107] [node 5] ICS09 handshake done sender: [2:7574436007493679509:2109] self: [2:7574436007493679508:0] peer: [5:7574436007425425849:0] socket: 205 qp: -1
2025-11-19T14:00:10.166407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436007493679530:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:10.166419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:10.166460Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436007118248101:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.166473Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436007118248101:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.167244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436006350843396:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47400
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.165237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005176017899:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14459
2025-11-19T14:00:10.167978Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436006350843389:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:10.167876Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436007118248101:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:10.167681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436007678574987:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25467
2025-11-19T14:00:10.167098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436007493679522:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.167287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436007493679529:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52546
2025-11-19T14:00:10.168633Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436003198712148:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(18152216ns) {HandleHandshakeStatus:329(18151346ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(400ns) SwitchToState:178(67ns) ProcessPendingSessionEvents:495(7881781ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7862982ns max# 7791213ns}}}
2025-11-19T14:00:10.168759Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436007493679529:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.168053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436006350843307:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.168088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:10.168092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436006350843307:2048] [node 1] ICP22 created new session: [6:7574436006350843398:2048]
2025-11-19T14:00:10.168099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436006350843398:2048] [node 1] ICS09 handshake done sender: [6:7574436006350843389:2115] self: [6:7574436006350843395:0] peer: [1:7574436007118248094:0] socket: 232 qp: -1
2025-11-19T14:00:10.168104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436006350843398:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:10.168111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:10.169181Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436007118248087:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.170306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005176017900:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14459
2025-11-19T14:00:10.165760Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436007425425858:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.166334Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436007425425844:2108] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:10.169844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436003130458487:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.169963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436007425425863:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:836
2025-11-19T14:00:10.170917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436007425425846:2109] [node 6] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.170933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:10.170939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436007425425846:2109] [node 6] ICP22 created new session: [5:7574436007425425866:2109]
2025-11-19T14:00:10.170945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436007425425866:2109] [node 6] ICS09 handshake done sender: [5:7574436007425425844:2108] self: [5:7574436007425425847:0] peer: [6:7574436006350843353:0] socket: 203 qp: -1
2025-11-19T14:00:10.170950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436007425425866:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:10.170970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:10.171584Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436007678574982:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(31178860ns) {Configure:104(31177509ns) {ProcessConfigured:135(30865985ns) {StartInitialHandshake:157(30864961ns) {DropHandshakes:469(1287ns) {DropIncomingHandshake:441(205ns) DropOutgoingHandshake:456(321ns)} GenerateSessionVirtualId:988(266ns) PrepareNewSessionHandshake:476(794ns) {SwitchToState:178(298ns)}}}}}
2025-11-19T14:00:10.171681Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436007678574969:2108] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:10.172408Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436007678574971:2109] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:10.172464Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436007493679515:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.172481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436007493679515:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.172684Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436007425425859:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.172527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436006350843397:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47400
2025-11-19T14:00:10.172719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436003130458489:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.172853Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005176017895:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.176144Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436000881050519:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22085534ns) {HandleHandshakeStatus:329(22084521ns) {DropIncomingHandshake:441(455ns) DropOutgoingHandshake:456(198ns) SwitchToState:178(815ns) ProcessPendingSessionEvents:495(15454324ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15436745ns max# 15348391ns}}}
2025-11-19T14:00:10.176453Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436005176017877:2108] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:10.176485Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436007425425848:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:10.178964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436000881050520:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.178989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:10.178997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436000881050520:2049] [node 2] ICP22 created new session: [3:7574436005176017902:2049]
2025-11-19T14:00:10.179004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436005176017902:2049] [node 2] ICS09 handshake done sender: [3:7574436005176017877:2108] self: [3:7574436005176017879:0] peer: [2:7574436007493679503:0] socket: 208 qp: -1
2025-11-19T14:00:10.179013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436005176017902:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:10.179021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:10.179645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436003130458488:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.179661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:10.179667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436003130458488:2049] [node 2] ICP22 created new session: [5:7574436007425425869:2049]
2025-11-19T14:00:10.179672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436007425425869:2049] [node 2] ICS09 handshake done sender: [5:7574436007425425848:2110] self: [5:7574436007425425849:0] peer: [2:7574436007493679508:0] socket: 206 qp: -1
2025-11-19T14:00:10.179678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436007425425869:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:10.179697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.181094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436007493679535:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52546
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.181191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436007118248091:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.181223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:10.181230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436007118248091:2112] [node 6] ICP22 created new session: [1:7574436007118248110:2112]
2025-11-19T14:00:10.181237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436007118248110:2112] [node 6] ICS09 handshake done sender: [1:7574436007118248101:2114] self: [1:7574436007118248094:0] peer: [6:7574436006350843395:0] socket: 229 qp: -1
2025-11-19T14:00:10.181244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436007118248110:2112] [node 6] ICS10 traffic start
2025-11-19T14:00:10.181252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:10.181614Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436007118248093:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.181620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436007118248093:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.182079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436002823280722:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.182089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:10.182096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436002823280722:2048] [node 2] ICP22 created new session: [1:7574436007118248112:2048]
2025-11-19T14:00:10.182102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436007118248112:2048] [node 2] ICS09 handshake done sender: [1:7574436007118248087:2110] self: [1:7574436007118248086:0] peer: [2:7574436007493679525:0] socket: 209 qp: -1
2025-11-19T14:00:10.182107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436007118248112:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:10.182114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:10.182953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436007118248114:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26659
2025-11-19T14:00:10.183270Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436006350843390:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.183389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436006350843309:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.184699Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436007118248093:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:10.184720Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436007493679517:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:10.184712Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005176017890:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.184723Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436005176017890:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.184887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436003198712147:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.184911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:10.184924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436003198712147:2048] [node 1] ICP22 created new session: [2:7574436007493679537:2048]
2025-11-19T14:00:10.184931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436007493679537:2048] [node 1] ICS09 handshake done sender: [2:7574436007493679517:2112] self: [2:7574436007493679525:0] peer: [1:7574436007118248086:0] socket: 210 qp: -1
2025-11-19T14:00:10.184937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436007493679537:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:10.184948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.185066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436007678574973:2110] [node 6] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.185085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:10.185101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436007678574973:2110] [node 6] ICP22 created new session: [4:7574436007678574991:2110]
2025-11-19T14:00:10.185106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436007678574991:2110] [node 6] ICS09 handshake done sender: [4:7574436007678574971:2109] self: [4:7574436007678574974:0] peer: [6:7574436006350843362:0] socket: 212 qp: -1
2025-11-19T14:00:10.185112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436007678574991:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:10.185118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:10.185422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436003383607612:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.185427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:10.185431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436003383607612:2048] [node 1] ICP22 created new session: [4:7574436007678574993:2048]
2025-11-19T14:00:10.185435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436007678574993:2048] [node 1] ICS09 handshake done sender: [4:7574436007678574969:2108] self: [4:7574436007678574970:0] peer: [1:7574436007118248082:0] socket: 201 qp: -1
2025-11-19T14:00:10.185439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436007678574993:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:10.185443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:10.185761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436007678574989:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25467
2025-11-19T14:00:10.185834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436007678574990:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25467
2025-11-19T14:00:10.187100Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436007118248105:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.187136Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436007118248114:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.187428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436007118248090:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.187442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:10.187449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436007118248090:2111] [node 5] ICP22 created new session: [1:7574436007118248116:2111]
2025-11-19T14:00:10.187455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436007118248116:2111] [node 5] ICS09 handshake done sender: [1:7574436007118248093:2113] self: [1:7574436007118248092:0] peer: [5:7574436007425425864:0] socket: 219 qp: -1
2025-11-19T14:00:10.187461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436007118248116:2111] [node 5] ICS10 traffic start
2025-11-19T14:00:10.187468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:10.187846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436007118248090:2111] [node 5] ICP18 handshake (actor [1:7574436007118248105:2116]) is too old
2025-11-19T14:00:10.187860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436002823280722:2048] [node 2] ICP18 handshake (actor [1:7574436007118248114:2117]) is too old
2025-11-19T14:00:10.188064Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436006350843397:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.188355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436006350843308:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.186904Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436007678574990:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.192034Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005176017888:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.192057Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436005176017888:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.196000Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436007493679526:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.196020Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436007493679526:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.197112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436007425425871:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:836
2025-11-19T14:00:10.197269Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005176017900:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.199847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436007425425872:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:836
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.200540Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436007493679515:2111] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:10.200614Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436005176017890:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:10.201639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436005176017886:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.201661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:10.201668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436005176017886:2112] [node 5] ICP22 created new session: [3:7574436005176017907:2112]
2025-11-19T14:00:10.201676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436005176017907:2112] [node 5] ICS09 handshake done sender: [3:7574436005176017890:2114] self: [3:7574436005176017889:0] peer: [5:7574436007425425868:0] socket: 231 qp: -1
2025-11-19T14:00:10.201682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436005176017907:2112] [node 5] ICS10 traffic start
2025-11-19T14:00:10.201693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:10.201183Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436007493679526:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:10.202147Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005176017896:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.202155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436005176017896:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.200253Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436007678574989:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.200367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436007678574997:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25467
2025-11-19T14:00:10.200392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436003383607614:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.202295Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436007678574975:2111] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:10.203555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436003383607613:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.203571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:10.203580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436003383607613:2049] [node 2] ICP22 created new session: [4:7574436007678574999:2049]
2025-11-19T14:00:10.203586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436007678574999:2049] [node 2] ICS09 handshake done sender: [4:7574436007678574975:2111] self: [4:7574436007678574985:0] peer: [2:7574436007493679514:0] socket: 215 qp: -1
2025-11-19T14:00:10.203594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436007678574999:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:10.203601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:10.204152Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436003130458487:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9064) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436007425425858:2114] held: no
2025-11-19T14:00:10.204298Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436007425425872:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.205650Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436007425425858:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:10.205899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436007493679512:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.205924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:10.205931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436007493679512:2110] [node 4] ICP22 created new session: [2:7574436007493679541:2110]
2025-11-19T14:00:10.205938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436007493679541:2110] [node 4] ICS09 handshake done sender: [2:7574436007493679515:2111] self: [2:7574436007493679514:0] peer: [4:7574436007678574985:0] socket: 214 qp: -1
2025-11-19T14:00:10.205957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436007493679541:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:10.205965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:10.206208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436007493679519:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.206219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:10.206223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436007493679519:2113] [node 6] ICP22 created new session: [2:7574436007493679543:2113]
2025-11-19T14:00:10.206227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436007493679543:2113] [node 6] ICS09 handshake done sender: [2:7574436007493679526:2116] self: [2:7574436007493679521:0] peer: [6:7574436006350843401:0] socket: 252 qp: -1
2025-11-19T14:00:10.206231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436007493679543:2113] [node 6] ICS10 traffic start
2025-11-19T14:00:10.206238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.206782Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436005176017888:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:10.206104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436003130458489:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31544) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436007425425859:2115] held: no
2025-11-19T14:00:10.206191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436003130458487:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.206201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:10.206206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436003130458487:2048] [node 1] ICP22 created new session: [5:7574436007425425874:2048]
2025-11-19T14:00:10.206211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436007425425874:2048] [node 1] ICS09 handshake done sender: [5:7574436007425425858:2114] self: [5:7574436007425425864:0] peer: [1:7574436007118248092:0] socket: 221 qp: -1
2025-11-19T14:00:10.206216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436007425425874:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:10.206222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:10.206514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436007425425857:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.206542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436007425425873:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:836
2025-11-19T14:00:10.207115Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436007425425859:2115] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:10.206899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436005176017882:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.206916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:10.206921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436005176017882:2111] [node 6] ICP22 created new session: [3:7574436005176017910:2111]
2025-11-19T14:00:10.206926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436005176017910:2111] [node 6] ICS09 handshake done sender: [3:7574436005176017888:2113] self: [3:7574436005176017883:0] peer: [6:7574436006350843400:0] socket: 230 qp: -1
2025-11-19T14:00:10.206931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436005176017910:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:10.206938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:10.209921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436006350843308:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17101) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436006350843397:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.211130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436006350843402:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47400
2025-11-19T14:00:10.211140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436006350843403:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47400
2025-11-19T14:00:10.211608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436003130458489:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.211632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:10.211639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436003130458489:2050] [node 3] ICP22 created new session: [5:7574436007425425877:2050]
2025-11-19T14:00:10.211646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436007425425877:2050] [node 3] ICS09 handshake done sender: [5:7574436007425425859:2115] self: [5:7574436007425425868:0] peer: [3:7574436005176017889:0] socket: 236 qp: -1
2025-11-19T14:00:10.211652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436007425425877:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:10.211662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:10.212552Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436007678574988:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.212564Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436007678574988:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.213394Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436007678574988:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:10.214916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436007678574982:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.214938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:10.214945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436007678574982:2113] [node 5] ICP22 created new session: [4:7574436007678575002:2113]
2025-11-19T14:00:10.214951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436007678575002:2113] [node 5] ICS09 handshake done sender: [4:7574436007678574988:2117] self: [4:7574436007678574983:0] peer: [5:7574436007425425876:0] socket: 261 qp: -1
2025-11-19T14:00:10.214956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436007678575002:2113] [node 5] ICS10 traffic start
2025-11-19T14:00:10.214964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:10.213259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436007425425879:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:836
2025-11-19T14:00:10.213583Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436007425425872:2120] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:10.213796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436007425425857:2111] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.213818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:10.213827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436007425425857:2111] [node 4] ICP22 created new session: [5:7574436007425425880:2111]
2025-11-19T14:00:10.213834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436007425425880:2111] [node 4] ICS09 handshake done sender: [5:7574436007425425872:2120] self: [5:7574436007425425876:0] peer: [4:7574436007678574983:0] socket: 256 qp: -1
2025-11-19T14:00:10.213841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436007425425880:2111] [node 4] ICS10 traffic start
2025-11-19T14:00:10.213852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:10.219657Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436005176017896:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:10.219806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436005176017885:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.219830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:10.219838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436005176017885:2110] [node 4] ICP22 created new session: [3:7574436005176017913:2110]
2025-11-19T14:00:10.219846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436005176017913:2110] [node 4] ICS09 handshake done sender: [3:7574436005176017896:2117] self: [3:7574436005176017887:0] peer: [4:7574436007678574998:0] socket: 248 qp: -1
2025-11-19T14:00:10.219852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436005176017913:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:10.219863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:10.220095Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436006350843390:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.220449Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436003383607613:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16801923ns) {HandleHandshakeStatus:329(16801303ns) {DropIncomingHandshake:441(147ns) DropOutgoingHandshake:456(1493ns) SwitchToState:178(234ns) ProcessPendingSessionEvents:495(10192052ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10167669ns max# 10086363ns}}}
2025-11-19T14:00:10.220823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436007678575004:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25467
2025-11-19T14:00:10.221235Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436006350843397:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:10.221311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436006350843309:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.221327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:10.221334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436006350843309:2050] [node 3] ICP22 created new session: [6:7574436006350843404:2050]
2025-11-19T14:00:10.221339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436006350843404:2050] [node 3] ICS09 handshake done sender: [6:7574436006350843390:2116] self: [6:7574436006350843400:0] peer: [3:7574436005176017883:0] socket: 233 qp: -1
2025-11-19T14:00:10.221346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436006350843404:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:10.221353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:10.221628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436006350843308:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.221634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:10.221637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436006350843308:2049] [node 2] ICP22 created new session: [6:7574436006350843406:2049]
2025-11-19T14:00:10.221643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436006350843406:2049] [node 2] ICS09 handshake done sender: [6:7574436006350843397:2118] self: [6:7574436006350843401:0] peer: [2:7574436007493679521:0] socket: 253 qp: -1
2025-11-19T14:00:10.221647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436006350843406:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:10.221653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:10.224041Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436007678574989:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:10.225562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436003383607614:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.225586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:10.225593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436003383607614:2050] [node 3] ICP22 created new session: [4:7574436007678575005:2050]
2025-11-19T14:00:10.225600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436007678575005:2050] [node 3] ICS09 handshake done sender: [4:7574436007678574989:2118] self: [4:7574436007678574998:0] peer: [3:7574436005176017887:0] socket: 257 qp: -1
2025-11-19T14:00:10.225607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436007678575005:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:10.225614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:10.252286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:10.252395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436007493679537:2048] [node 1] ICS01 socket: 210 reason# EndOfStream
2025-11-19T14:00:10.252406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436003198712147:2048] [node 1] ICP30 unregister session Session# [2:7574436007493679537:2048] VirtualId# [2:7574436007493679525:0]
2025-11-19T14:00:10.252415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436007493679537:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.252452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:10.252474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:10.256086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:10.256156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574436005176017897:2048] [node 1] ICS07 socket disconnect 200 reason# EndOfStream
2025-11-19T14:00:10.256162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436005176017897:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.256188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:10.256206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436005176017897:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:10.264037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:10.264155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436007678574993:2048] [node 1] ICS07 socket disconnect 201 reason# EndOfStream
2025-11-19T14:00:10.264163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436007678574993:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.264217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:10.264240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436007678574993:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:10.264731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436000881050519:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9064) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:10.264750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436005176017897:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:10.264755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436000881050519:2048] [node 1] ICP30 unregister session Session# [3:7574436005176017897:2048] VirtualId# [3:7574436005176017876:0]
2025-11-19T14:00:10.264815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:10.264823Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436000881050519:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:10.264827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:10.265076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436003383607612:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9064) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:10.265087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436007678574993:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:10.265091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436003383607612:2048] [node 1] ICP30 unregister session Session# [4:7574436007678574993:2048] VirtualId# [4:7574436007678574970:0]
2025-11-19T14:00:10.265135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:10.265140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436003383607612:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:10.265142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9064) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:10.265134Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:10.265198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436006350843398:2048] [node 1] ICS01 socket: 232 reason# EndOfStream
2025-11-19T14:00:10.265203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436006350843307:2048] [node 1] ICP30 unregister session Session# [6:7574436006350843398:2048] VirtualId# [6:7574436006350843395:0]
2025-11-19T14:00:10.265207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436006350843398:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.265233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:10.265253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:10.264859Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:10.264942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436007425425874:2048] [node 1] ICS01 socket: 221 reason# EndOfStream
2025-11-19T14:00:10.264950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436003130458487:2048] [node 1] ICP30 unregister session Session# [5:7574436007425425874:2048] VirtualId# [5:7574436007425425864:0]
2025-11-19T14:00:10.264956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436007425425874:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.265003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:10.265030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:10.276139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574436000881050519:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:10.275938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574436003383607612:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:10.283853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:10.283982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436005176017902:2049] [node 2] ICS01 socket: 208 reason# EndOfStream
2025-11-19T14:00:10.283994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436000881050520:2049] [node 2] ICP30 unregister session Session# [3:7574436005176017902:2049] VirtualId# [3:7574436005176017879:0]
2025-11-19T14:00:10.284003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436005176017902:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.284059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:10.284085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:10.311423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:10.311542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436007678574999:2049] [node 2] ICS07 socket disconnect 215 reason# EndOfStream
2025-11-19T14:00:10.311550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436007678574999:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.311612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:10.311636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436007678574999:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:10.314095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436003383607613:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17101) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:10.314114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436007678574999:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:10.314121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436003383607613:2049] [node 2] ICP30 unregister session Session# [4:7574436007678574999:2049] VirtualId# [4:7574436007678574985:0]
2025-11-19T14:00:10.314183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:10.314189Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436003383607613:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17101) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:10.314193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17101) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:10.314549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:10.314630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436006350843406:2049] [node 2] ICS01 socket: 253 reason# EndOfStream
2025-11-19T14:00:10.314636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436006350843308:2049] [node 2] ICP30 unregister session Session# [6:7574436006350843406:2049] VirtualId# [6:7574436006350843401:0]
2025-11-19T14:00:10.314641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436006350843406:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.314691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:10.314714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:10.311856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:10.311954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436007425425869:2049] [node 2] ICS01 socket: 206 reason# EndOfStream
2025-11-19T14:00:10.311964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436003130458488:2049] [node 2] ICP30 unregister session Session# [5:7574436007425425869:2049] VirtualId# [5:7574436007425425849:0]
2025-11-19T14:00:10.311971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436007425425869:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.312024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:10.312052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:10.322816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:10.322930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436007425425877:2050] [node 3] ICS01 socket: 236 reason# EndOfStream
2025-11-19T14:00:10.322941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436003130458489:2050] [node 3] ICP30 unregister session Session# [5:7574436007425425877:2050] VirtualId# [5:7574436007425425868:0]
2025-11-19T14:00:10.322950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436007425425877:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.323008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:10.323032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:10.326204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:10.326304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436006350843404:2050] [node 3] ICS01 socket: 233 reason# EndOfStream
2025-11-19T14:00:10.326316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436006350843309:2050] [node 3] ICP30 unregister session Session# [6:7574436006350843404:2050] VirtualId# [6:7574436006350843400:0]
2025-11-19T14:00:10.326324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436006350843404:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.326377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:10.326404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:10.342809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T14:00:10.342929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436007425425880:2111] [node 4] ICS01 socket: 256 reason# EndOfStream
2025-11-19T14:00:10.342941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436007425425857:2111] [node 4] ICP30 unregister session Session# [5:7574436007425425880:2111] VirtualId# [5:7574436007425425876:0]
2025-11-19T14:00:10.342950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436007425425880:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.343000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:10.343027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:10.343852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:10.343934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436006350843386:2093] [node 4] ICS01 socket: 211 reason# EndOfStream
2025-11-19T14:00:10.343942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436006350843358:2093] [node 4] ICP30 unregister session Session# [6:7574436006350843386:2093] VirtualId# [6:7574436006350843362:0]
2025-11-19T14:00:10.343949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436006350843386:2093] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.343993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:10.344015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:10.372662Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T14:00:10.372785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436006350843393:2089] [node 5] ICS01 socket: 202 reason# EndOfStream
2025-11-19T14:00:10.372796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436006350843350:2089] [node 5] ICP30 unregister session Session# [6:7574436006350843393:2089] VirtualId# [6:7574436006350843353:0]
2025-11-19T14:00:10.372805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436006350843393:2089] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.372860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T14:00:10.372890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:10.552125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436006844371296:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:10.552139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436006844371297:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:10.601347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436003490011093:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:10.601364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436003490011094:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:10.668008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436005896857482:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:10.668022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436005896857483:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:10.700110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436003978843472:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:10.700123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436003978843473:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:10.700127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436003978843474:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:10.735967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436003905375572:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:10.735987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436003905375573:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:10.735991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436003905375574:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:10.771563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436005654353636:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:10.771576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436005654353637:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:10.771581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436005654353638:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:10.808150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436003905375610:2075] [node 4] ICP01 ready to work
2025-11-19T14:00:10.808244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436005654353656:2059] [node 4] ICP01 ready to work
2025-11-19T14:00:10.808374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436003905375618:2073] [node 6] ICP01 ready to work
2025-11-19T14:00:10.809481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436006844371359:2107] [node 6] ICP01 ready to work
2025-11-19T14:00:10.809591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436006844371361:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:10.809981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436005896857542:2105] [node 4] ICP01 ready to work
2025-11-19T14:00:10.811934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436003490011154:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.818795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436003978843535:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14863
2025-11-19T14:00:10.818826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436003978843536:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14863
2025-11-19T14:00:10.820076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436006844371360:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:10.824892Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436003905375573:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16921511ns) {Configure:104(16920485ns) {ProcessConfigured:135(14324632ns) {StartInitialHandshake:157(14323651ns) {DropHandshakes:469(531ns) {DropIncomingHandshake:441(24ns) DropOutgoingHandshake:456(14ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:476(1527ns) {SwitchToState:178(910ns)}}}}}
2025-11-19T14:00:10.825020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436003905375618:2073] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16580331ns) {Configure:104(16580030ns) {ProcessConfigured:135(14246922ns) {StartInitialHandshake:157(14246376ns) {DropHandshakes:469(218ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(21ns)} GenerateSessionVirtualId:988(181ns) PrepareNewSessionHandshake:476(448ns) {SwitchToState:178(93ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.825921Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436006844371359:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16286555ns) {Configure:104(16286087ns) {ProcessConfigured:135(14655581ns) {StartInitialHandshake:157(14655285ns) {DropHandshakes:469(270ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(23ns)} GenerateSessionVirtualId:988(63ns) PrepareNewSessionHandshake:476(1106ns) {SwitchToState:178(579ns)}}}}}
2025-11-19T14:00:10.826683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436005654353681:2072] [node 5] ICP01 ready to work
2025-11-19T14:00:10.825078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436003490011160:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:10.825558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436003490011161:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33324
2025-11-19T14:00:10.826604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436005896857546:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:10.826984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436003978843534:2106] [node 6] ICP01 ready to work
2025-11-19T14:00:10.827502Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436003978843535:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.835810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436005896857482:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25338922ns) {Configure:104(15576807ns) {ProcessConfigured:135(15266519ns) {StartInitialHandshake:157(15265549ns) {DropHandshakes:469(650ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(879ns) {SwitchToState:178(285ns)}}}}}
2025-11-19T14:00:10.835992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436005896857551:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:10.835329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436006844371367:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26408
2025-11-19T14:00:10.835354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436006844371368:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26408
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.838759Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436005654353638:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(30485351ns) {Configure:104(30484725ns) {ProcessConfigured:135(29481990ns) {StartInitialHandshake:157(29481264ns) {DropHandshakes:469(243ns) {DropIncomingHandshake:441(30ns) DropOutgoingHandshake:456(25ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:476(936ns) {SwitchToState:178(710ns)}}}}}
2025-11-19T14:00:10.839019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436005654353711:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37988
2025-11-19T14:00:10.839032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436005654353712:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37988
2025-11-19T14:00:10.839094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436005654353713:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37988
2025-11-19T14:00:10.840004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436003490011157:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.844364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005896857559:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3655
2025-11-19T14:00:10.844449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005896857560:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3655
2025-11-19T14:00:10.844460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005896857561:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3655
2025-11-19T14:00:10.844659Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005896857559:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.847878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436005896857546:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21184996ns) {Configure:104(21184043ns) {ProcessConfigured:135(20832589ns) {StartInitialHandshake:157(20831761ns) {DropHandshakes:469(541ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(171ns) PrepareNewSessionHandshake:476(1448ns) {SwitchToState:178(820ns)}}}}}
2025-11-19T14:00:10.850526Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436005654353681:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23697306ns) {Configure:104(23696638ns) {ProcessConfigured:135(23321716ns) {StartInitialHandshake:157(23321267ns) {DropHandshakes:469(435ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:476(1409ns) {SwitchToState:178(86ns)}}}}}
2025-11-19T14:00:10.850651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436005654353637:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(42480126ns) {Configure:104(42479309ns) {ProcessConfigured:135(41193443ns) {StartInitialHandshake:157(41192764ns) {DropHandshakes:469(174ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(149ns) PrepareNewSessionHandshake:476(642ns) {SwitchToState:178(367ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.851257Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436006844371360:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(25210268ns) {Configure:104(25209413ns) {ProcessConfigured:135(24859187ns) {StartInitialHandshake:157(24858919ns) {DropHandshakes:469(343ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:476(1468ns) {SwitchToState:178(1191ns)}}}}}
2025-11-19T14:00:10.851551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436006844371374:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26408
2025-11-19T14:00:10.852546Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436006844371296:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(44300053ns) {Configure:104(44299426ns) {ProcessConfigured:135(41438071ns) {StartInitialHandshake:157(41436833ns) {DropHandshakes:469(695ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(358ns) PrepareNewSessionHandshake:476(1378ns) {SwitchToState:178(657ns)}}}}}
2025-11-19T14:00:10.851877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436003490011094:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(23106516ns) {Configure:104(23105732ns) {ProcessConfigured:135(22791015ns) {StartInitialHandshake:157(22790658ns) {DropHandshakes:469(527ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:476(1033ns) {SwitchToState:178(855ns)}}}}}
2025-11-19T14:00:10.851969Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003490011161:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.853035Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436005654353712:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.853080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436005654353636:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.853266Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436006844371366:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.853274Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436006844371366:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.854319Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436006844371366:2110] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:10.854407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436006844371359:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.854425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:10.854432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436006844371359:2107] [node 6] ICP22 created new session: [1:7574436006844371378:2107]
2025-11-19T14:00:10.854438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436006844371378:2107] [node 6] ICS09 handshake done sender: [1:7574436006844371366:2110] self: [1:7574436006844371364:0] peer: [6:7574436005654353717:0] socket: 218 qp: -1
2025-11-19T14:00:10.854446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436006844371378:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:10.854468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.855907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436003905375649:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4121
2025-11-19T14:00:10.855945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436006844371297:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(47505617ns) {Configure:104(47502754ns) {ProcessConfigured:135(44700362ns) {StartInitialHandshake:157(44699535ns) {DropHandshakes:469(327ns) {DropIncomingHandshake:441(16ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:476(869ns) {SwitchToState:178(645ns)}}}}}
2025-11-19T14:00:10.857911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436005896857542:2105] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(47857048ns) {Configure:104(47855802ns) {ProcessConfigured:135(47498487ns) {StartInitialHandshake:157(47497940ns) {DropHandshakes:469(493ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(185ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:476(1046ns) {SwitchToState:178(821ns)}}}}}
2025-11-19T14:00:10.857843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436003490011093:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(26017672ns) {Configure:104(18206459ns) {ProcessConfigured:135(17871967ns) {StartInitialHandshake:157(17871573ns) {DropHandshakes:469(829ns) {DropIncomingHandshake:441(115ns) DropOutgoingHandshake:456(125ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(1124ns) {SwitchToState:178(894ns)}}}}}
2025-11-19T14:00:10.858049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436003978843541:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.862438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436003490011154:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(50401739ns) {Configure:104(50400692ns) {ProcessConfigured:135(50026294ns) {StartInitialHandshake:157(50025720ns) {DropHandshakes:469(827ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(1412ns) {SwitchToState:178(917ns)}}}}}
2025-11-19T14:00:10.862650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436003490011168:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33324
2025-11-19T14:00:10.862666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436003490011169:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33324
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.863432Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436006844371368:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.863546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436006844371382:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26408
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.864641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436003978843549:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14863
2025-11-19T14:00:10.864791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436003978843534:2106] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(37700914ns) {Configure:104(23496387ns) {ProcessConfigured:135(23128919ns) {StartInitialHandshake:157(23128349ns) {DropHandshakes:469(801ns) {DropIncomingHandshake:441(140ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(331ns) PrepareNewSessionHandshake:476(713ns) {SwitchToState:178(225ns)}}}}}
2025-11-19T14:00:10.868119Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436003905375649:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.868331Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436003978843549:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.868361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436003978843472:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.868472Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005896857560:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.871209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436003490011174:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33324
2025-11-19T14:00:10.871351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436003490011157:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19206360ns) {Configure:104(19205415ns) {ProcessConfigured:135(18870185ns) {StartInitialHandshake:157(18869344ns) {DropHandshakes:469(991ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(451ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:476(852ns) {SwitchToState:178(461ns)}}}}}
2025-11-19T14:00:10.872313Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003490011174:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.872342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436003490011093:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.872617Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003490011168:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.872313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436003905375572:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(42541590ns) {Configure:104(42539978ns) {ProcessConfigured:135(42150302ns) {StartInitialHandshake:157(42149788ns) {DropHandshakes:469(629ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(321ns) PrepareNewSessionHandshake:476(1875ns) {SwitchToState:178(1075ns)}}}}}
2025-11-19T14:00:10.872414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436003905375572:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.872704Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436006844371370:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.872721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436006844371370:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.884356Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436006844371382:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.884563Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436006844371374:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.888756Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436006844371376:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.888772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436006844371376:2116] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.888822Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436006844371373:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.888830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436006844371373:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.889547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436005654353718:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37988
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.890778Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003490011169:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.890347Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436006844371373:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:10.891206Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436005654353711:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.891236Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436005654353713:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.891253Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436005654353712:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:10.891461Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436006844371370:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:10.891759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436006844371361:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.890929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436003978843555:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14863
2025-11-19T14:00:10.891614Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436003978843549:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:10.891786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:10.891794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436006844371361:2109] [node 4] ICP22 created new session: [1:7574436006844371388:2109]
2025-11-19T14:00:10.891800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436006844371388:2109] [node 4] ICS09 handshake done sender: [1:7574436006844371370:2113] self: [1:7574436006844371365:0] peer: [4:7574436003978843554:0] socket: 219 qp: -1
2025-11-19T14:00:10.891807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436006844371388:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:10.891817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:10.892769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436003978843472:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.892790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:10.892797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436003978843472:2048] [node 1] ICP22 created new session: [4:7574436003978843556:2048]
2025-11-19T14:00:10.892802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436003978843556:2048] [node 1] ICS09 handshake done sender: [4:7574436003978843549:2115] self: [4:7574436003978843554:0] peer: [1:7574436006844371365:0] socket: 228 qp: -1
2025-11-19T14:00:10.892807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436003978843556:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:10.892817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:10.890494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436003905375652:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4121
2025-11-19T14:00:10.891377Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436003905375649:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:10.892253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436003905375572:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.892266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:10.892271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436003905375572:2048] [node 1] ICP22 created new session: [5:7574436003905375653:2048]
2025-11-19T14:00:10.892276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436003905375653:2048] [node 1] ICS09 handshake done sender: [5:7574436003905375649:2114] self: [5:7574436003905375651:0] peer: [1:7574436006844371369:0] socket: 237 qp: -1
2025-11-19T14:00:10.892281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436003905375653:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:10.892300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:10.900324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005896857566:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3655
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.900578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436006844371360:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.900608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:10.900614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436006844371360:2108] [node 5] ICP22 created new session: [1:7574436006844371391:2108]
2025-11-19T14:00:10.900621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436006844371391:2108] [node 5] ICS09 handshake done sender: [1:7574436006844371373:2114] self: [1:7574436006844371369:0] peer: [5:7574436003905375651:0] socket: 232 qp: -1
2025-11-19T14:00:10.900629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436006844371391:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:10.900636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:10.901583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436005654353636:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.901602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:10.901608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436005654353636:2048] [node 1] ICP22 created new session: [6:7574436005654353720:2048]
2025-11-19T14:00:10.901614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436005654353720:2048] [node 1] ICS09 handshake done sender: [6:7574436005654353712:2114] self: [6:7574436005654353717:0] peer: [1:7574436006844371364:0] socket: 225 qp: -1
2025-11-19T14:00:10.901618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436005654353720:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:10.901636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:10.901923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436005654353637:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.901939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436005654353638:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.902096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436005654353719:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37988
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.905604Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005896857555:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.905616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436005896857555:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.905893Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005896857566:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.905161Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436005654353719:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.905197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436005654353681:2072] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.906551Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436003978843536:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.906493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436003905375656:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4121
2025-11-19T14:00:10.906481Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436006844371376:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:10.906349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005896857561:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.906792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436006844371296:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.906814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:10.906819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436006844371296:2048] [node 2] ICP22 created new session: [1:7574436006844371394:2048]
2025-11-19T14:00:10.906825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436006844371394:2048] [node 2] ICS09 handshake done sender: [1:7574436006844371376:2116] self: [1:7574436006844371362:0] peer: [2:7574436003490011177:0] socket: 236 qp: -1
2025-11-19T14:00:10.906830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436006844371394:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:10.906838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:10.907412Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436003905375656:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.907443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436003905375610:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.907565Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436003978843551:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.907573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436003978843551:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.908324Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436003978843551:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:10.908471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436003978843541:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.908490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:10.908496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436003978843541:2112] [node 5] ICP22 created new session: [4:7574436003978843561:2112]
2025-11-19T14:00:10.908502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436003978843561:2112] [node 5] ICS09 handshake done sender: [4:7574436003978843551:2116] self: [4:7574436003978843550:0] peer: [5:7574436003905375657:0] socket: 249 qp: -1
2025-11-19T14:00:10.908507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436003978843561:2112] [node 5] ICS10 traffic start
2025-11-19T14:00:10.908515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.909071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436003905375658:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4121
2025-11-19T14:00:10.909414Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436003905375647:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.909422Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436003905375647:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.909537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005896857567:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3655
2025-11-19T14:00:10.909759Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005896857567:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.909788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436005896857482:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.910060Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436006844371381:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.910074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436006844371381:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.910908Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436003905375656:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:10.911219Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436006844371381:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:10.911756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436006844371297:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.911782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:10.911792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436006844371297:2049] [node 3] ICP22 created new session: [1:7574436006844371397:2049]
2025-11-19T14:00:10.911799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436006844371397:2049] [node 3] ICS09 handshake done sender: [1:7574436006844371381:2117] self: [1:7574436006844371363:0] peer: [3:7574436005896857570:0] socket: 209 qp: -1
2025-11-19T14:00:10.911807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436006844371397:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:10.911815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:10.909498Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003490011163:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.909513Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436003490011163:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.914592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436005896857483:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.914679Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436005896857555:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:10.914302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436003490011093:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10343) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436003490011174:2117] held: no
2025-11-19T14:00:10.914524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436003490011180:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33324
2025-11-19T14:00:10.914984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436005896857551:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.915001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:10.915008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436005896857551:2109] [node 6] ICP22 created new session: [3:7574436005896857572:2109]
2025-11-19T14:00:10.915014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436005896857572:2109] [node 6] ICS09 handshake done sender: [3:7574436005896857555:2112] self: [3:7574436005896857554:0] peer: [6:7574436005654353723:0] socket: 223 qp: -1
2025-11-19T14:00:10.915020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436005896857572:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:10.915027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:10.915133Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003490011167:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.915142Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436003490011167:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.915700Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436003490011174:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:10.919094Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436003490011167:2113] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:10.919249Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436003490011163:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:10.919356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436003490011094:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.919373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:10.919379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436003490011094:2049] [node 3] ICP22 created new session: [2:7574436003490011184:2049]
2025-11-19T14:00:10.919398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436003490011184:2049] [node 3] ICS09 handshake done sender: [2:7574436003490011167:2113] self: [2:7574436003490011165:0] peer: [3:7574436005896857568:0] socket: 241 qp: -1
2025-11-19T14:00:10.919403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436003490011184:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:10.919409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:10.919600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436003490011160:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.919606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:10.919611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436003490011160:2109] [node 6] ICP22 created new session: [2:7574436003490011186:2109]
2025-11-19T14:00:10.919614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436003490011186:2109] [node 6] ICS09 handshake done sender: [2:7574436003490011163:2111] self: [2:7574436003490011162:0] peer: [6:7574436005654353722:0] socket: 214 qp: -1
2025-11-19T14:00:10.919618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436003490011186:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:10.919622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:10.918582Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436003905375647:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:10.918696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436003905375618:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.918709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:10.918718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436003905375618:2073] [node 6] ICP22 created new session: [5:7574436003905375660:2073]
2025-11-19T14:00:10.918724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436003905375660:2073] [node 6] ICS09 handshake done sender: [5:7574436003905375647:2113] self: [5:7574436003905375625:0] peer: [6:7574436005654353724:0] socket: 248 qp: -1
2025-11-19T14:00:10.918730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436003905375660:2073] [node 6] ICS10 traffic start
2025-11-19T14:00:10.918740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.920679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436003978843564:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14863
2025-11-19T14:00:10.921330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436003905375610:2075] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.921351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:10.921360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436003905375610:2075] [node 4] ICP22 created new session: [5:7574436003905375662:2075]
2025-11-19T14:00:10.921367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436003905375662:2075] [node 4] ICS09 handshake done sender: [5:7574436003905375656:2117] self: [5:7574436003905375657:0] peer: [4:7574436003978843550:0] socket: 216 qp: -1
2025-11-19T14:00:10.921373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436003905375662:2075] [node 4] ICS10 traffic start
2025-11-19T14:00:10.921386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.928536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436005896857482:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10343) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436005896857567:2119] held: no
2025-11-19T14:00:10.928842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005896857574:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3655
2025-11-19T14:00:10.929190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436005896857575:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3655
2025-11-19T14:00:10.928912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436003490011093:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.928933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:10.928940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436003490011093:2048] [node 1] ICP22 created new session: [2:7574436003490011188:2048]
2025-11-19T14:00:10.928948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436003490011188:2048] [node 1] ICS09 handshake done sender: [2:7574436003490011174:2117] self: [2:7574436003490011177:0] peer: [1:7574436006844371362:0] socket: 239 qp: -1
2025-11-19T14:00:10.928954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436003490011188:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:10.928975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.929665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436003905375664:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4121
2025-11-19T14:00:10.929682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436003905375665:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4121
2025-11-19T14:00:10.929692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436003905375666:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4121
2025-11-19T14:00:10.930431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436005654353638:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18190) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436005654353713:2115] held: no
2025-11-19T14:00:10.930931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436003978843564:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.930976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436003978843473:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.932255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436005654353726:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37988
2025-11-19T14:00:10.932307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436005654353727:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37988
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.932879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436003978843566:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14863
2025-11-19T14:00:10.933017Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003490011171:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.933028Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436003490011171:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.933945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436003978843567:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14863
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.934680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574436006844371359:2107] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:10.935483Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436003490011171:2116] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:10.935630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436003490011154:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.935644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:10.935649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436003490011154:2107] [node 4] ICP22 created new session: [2:7574436003490011192:2107]
2025-11-19T14:00:10.935654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436003490011192:2107] [node 4] ICS09 handshake done sender: [2:7574436003490011171:2116] self: [2:7574436003490011156:0] peer: [4:7574436003978843565:0] socket: 242 qp: -1
2025-11-19T14:00:10.935658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436003490011192:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:10.935662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:10.932437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436005896857483:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11394) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436005896857566:2118] held: no
2025-11-19T14:00:10.932613Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436005896857567:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:10.934112Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436005896857566:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:10.935809Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436003978843553:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.935818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436003978843553:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.936225Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436003978843564:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:10.936304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436003978843473:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.936316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:10.936321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436003978843473:2049] [node 2] ICP22 created new session: [4:7574436003978843568:2049]
2025-11-19T14:00:10.936327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436003978843568:2049] [node 2] ICS09 handshake done sender: [4:7574436003978843564:2119] self: [4:7574436003978843565:0] peer: [2:7574436003490011156:0] socket: 262 qp: -1
2025-11-19T14:00:10.936331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436003978843568:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:10.936336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:10.934239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436005654353728:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37988
2025-11-19T14:00:10.934395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436005654353729:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37988
2025-11-19T14:00:10.934508Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436005654353726:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.934526Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436005654353713:2115] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:10.934993Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436005654353711:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:10.935048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436005654353638:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.935061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:10.935067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436005654353638:2050] [node 3] ICP22 created new session: [6:7574436005654353730:2050]
2025-11-19T14:00:10.935072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436005654353730:2050] [node 3] ICS09 handshake done sender: [6:7574436005654353713:2115] self: [6:7574436005654353723:0] peer: [3:7574436005896857554:0] socket: 226 qp: -1
2025-11-19T14:00:10.935078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436005654353730:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:10.935086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:10.935173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436005654353656:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.935184Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436005654353719:2119] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:10.935284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436005654353637:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.935292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:10.935296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436005654353637:2049] [node 2] ICP22 created new session: [6:7574436005654353733:2049]
2025-11-19T14:00:10.935300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436005654353733:2049] [node 2] ICS09 handshake done sender: [6:7574436005654353711:2113] self: [6:7574436005654353722:0] peer: [2:7574436003490011162:0] socket: 224 qp: -1
2025-11-19T14:00:10.935304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436005654353733:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:10.935324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:10.940017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436005896857551:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(24932391ns) {HandleHandshakeStatus:329(24931401ns) {DropIncomingHandshake:441(3851ns) DropOutgoingHandshake:456(145ns) SwitchToState:178(78ns) ProcessPendingSessionEvents:495(18366345ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11189862ns max# 11108032ns}}}
2025-11-19T14:00:10.940352Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436003905375610:2075] [node 4] ICP35 event processing took too much time PendingConnection:278(29257085ns) {HandleHandshakeStatus:329(29256541ns) {DropIncomingHandshake:441(586ns) DropOutgoingHandshake:456(1735ns) SwitchToState:178(181ns) ProcessPendingSessionEvents:495(9617417ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9594899ns max# 9524102ns}}}
2025-11-19T14:00:10.940380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574436003905375610:2075] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:10.940617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436005654353681:2072] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.940631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:10.940636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436005654353681:2072] [node 5] ICP22 created new session: [6:7574436005654353735:2072]
2025-11-19T14:00:10.940641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436005654353735:2072] [node 5] ICS09 handshake done sender: [6:7574436005654353719:2119] self: [6:7574436005654353724:0] peer: [5:7574436003905375625:0] socket: 250 qp: -1
2025-11-19T14:00:10.940646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436005654353735:2072] [node 5] ICS10 traffic start
2025-11-19T14:00:10.940653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:10.942604Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436003905375666:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.942654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436003905375573:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.942844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436005654353737:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37988
2025-11-19T14:00:10.943068Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436003490011175:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.943078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436003490011175:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.943893Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436003978843553:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:10.944036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436003978843534:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.944057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:10.944062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436003978843534:2106] [node 6] ICP22 created new session: [4:7574436003978843571:2106]
2025-11-19T14:00:10.944067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436003978843571:2106] [node 6] ICS09 handshake done sender: [4:7574436003978843553:2117] self: [4:7574436003978843548:0] peer: [6:7574436005654353732:0] socket: 255 qp: -1
2025-11-19T14:00:10.944071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436003978843571:2106] [node 6] ICS10 traffic start
2025-11-19T14:00:10.944078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.944597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436003905375668:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4121
2025-11-19T14:00:10.944705Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436003490011175:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:10.944786Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436003905375666:2121] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:10.944846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436003905375573:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.944861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:10.944866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436003905375573:2049] [node 2] ICP22 created new session: [5:7574436003905375669:2049]
2025-11-19T14:00:10.944872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436003905375669:2049] [node 2] ICS09 handshake done sender: [5:7574436003905375666:2121] self: [5:7574436003905375667:0] peer: [2:7574436003490011170:0] socket: 264 qp: -1
2025-11-19T14:00:10.944877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436003905375669:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:10.944883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:10.945162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436003490011157:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.945176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:10.945184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436003490011157:2108] [node 5] ICP22 created new session: [2:7574436003490011195:2108]
2025-11-19T14:00:10.945188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436003490011195:2108] [node 5] ICS09 handshake done sender: [2:7574436003490011175:2118] self: [2:7574436003490011170:0] peer: [5:7574436003905375667:0] socket: 230 qp: -1
2025-11-19T14:00:10.945192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436003490011195:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:10.945198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:10.952320Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436005654353726:2120] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:10.955365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436005654353656:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.955394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:10.955402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436005654353656:2059] [node 4] ICP22 created new session: [6:7574436005654353738:2059]
2025-11-19T14:00:10.955407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436005654353738:2059] [node 4] ICS09 handshake done sender: [6:7574436005654353726:2120] self: [6:7574436005654353732:0] peer: [4:7574436003978843548:0] socket: 257 qp: -1
2025-11-19T14:00:10.955414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436005654353738:2059] [node 4] ICS10 traffic start
2025-11-19T14:00:10.955423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:10.961100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436005896857482:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.961124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:10.961133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436005896857482:2048] [node 1] ICP22 created new session: [3:7574436005896857578:2048]
2025-11-19T14:00:10.961140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436005896857578:2048] [node 1] ICS09 handshake done sender: [3:7574436005896857567:2119] self: [3:7574436005896857570:0] peer: [1:7574436006844371363:0] socket: 252 qp: -1
2025-11-19T14:00:10.961148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436005896857578:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:10.961154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:10.961329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436005896857483:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.961334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:10.961338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436005896857483:2049] [node 2] ICP22 created new session: [3:7574436005896857580:2049]
2025-11-19T14:00:10.961341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436005896857580:2049] [node 2] ICS09 handshake done sender: [3:7574436005896857566:2118] self: [3:7574436005896857568:0] peer: [2:7574436003490011165:0] socket: 243 qp: -1
2025-11-19T14:00:10.961346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436005896857580:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:10.961352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:10.965865Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436003905375664:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.966031Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574436003905375618:2073] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:10.966053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436003905375574:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:10.966324Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005896857562:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.966338Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436005896857562:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.967091Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436005896857562:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:10.967146Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436003978843566:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.967186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436003978843474:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.967395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436003905375672:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4121
2025-11-19T14:00:10.967530Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436005896857564:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:10.967537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436005896857564:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:10.968141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436005896857546:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.968159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:10.968168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436005896857546:2108] [node 5] ICP22 created new session: [3:7574436005896857583:2108]
2025-11-19T14:00:10.968174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436005896857583:2108] [node 5] ICS09 handshake done sender: [3:7574436005896857562:2116] self: [3:7574436005896857548:0] peer: [5:7574436003905375671:0] socket: 204 qp: -1
2025-11-19T14:00:10.968180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436005896857583:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:10.968197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:10.968819Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436003905375664:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:10.968933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436003905375574:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.968950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:10.968955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436003905375574:2050] [node 3] ICP22 created new session: [5:7574436003905375673:2050]
2025-11-19T14:00:10.968961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436003905375673:2050] [node 3] ICS09 handshake done sender: [5:7574436003905375664:2119] self: [5:7574436003905375671:0] peer: [3:7574436005896857548:0] socket: 234 qp: -1
2025-11-19T14:00:10.968966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436003905375673:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:10.968974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:10.970971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436003978843574:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14863
2025-11-19T14:00:10.971099Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436005896857564:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:10.971218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436005896857542:2105] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:10.971235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:10.971240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436005896857542:2105] [node 4] ICP22 created new session: [3:7574436005896857586:2105]
2025-11-19T14:00:10.971246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436005896857586:2105] [node 4] ICS09 handshake done sender: [3:7574436005896857564:2117] self: [3:7574436005896857543:0] peer: [4:7574436003978843573:0] socket: 268 qp: -1
2025-11-19T14:00:10.971250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436005896857586:2105] [node 4] ICS10 traffic start
2025-11-19T14:00:10.971266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:10.971431Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436003978843566:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:10.971506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436003978843474:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:10.971516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:10.971521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436003978843474:2050] [node 3] ICP22 created new session: [4:7574436003978843575:2050]
2025-11-19T14:00:10.971526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436003978843575:2050] [node 3] ICS09 handshake done sender: [4:7574436003978843566:2120] self: [4:7574436003978843573:0] peer: [3:7574436005896857543:0] socket: 269 qp: -1
2025-11-19T14:00:10.971530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436003978843575:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:10.971534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:10.989862Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:10.989951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436003490011188:2048] [node 1] ICS01 socket: 239 reason# EndOfStream
2025-11-19T14:00:10.989962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436003490011093:2048] [node 1] ICP30 unregister session Session# [2:7574436003490011188:2048] VirtualId# [2:7574436003490011177:0]
2025-11-19T14:00:10.989969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436003490011188:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.990007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:10.990026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:10.990425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:10.990491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436005896857578:2048] [node 1] ICS01 socket: 252 reason# EndOfStream
2025-11-19T14:00:10.990496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436005896857482:2048] [node 1] ICP30 unregister session Session# [3:7574436005896857578:2048] VirtualId# [3:7574436005896857570:0]
2025-11-19T14:00:10.990501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436005896857578:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.990525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:10.990541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:10.999627Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:10.999716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436005654353720:2048] [node 1] ICS01 socket: 225 reason# EndOfStream
2025-11-19T14:00:10.999724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436005654353636:2048] [node 1] ICP30 unregister session Session# [6:7574436005654353720:2048] VirtualId# [6:7574436005654353717:0]
2025-11-19T14:00:10.999731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436005654353720:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.999797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:10.999823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:10.999869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:10.999932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436003905375653:2048] [node 1] ICS01 socket: 237 reason# EndOfStream
2025-11-19T14:00:10.999936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436003905375572:2048] [node 1] ICP30 unregister session Session# [5:7574436003905375653:2048] VirtualId# [5:7574436003905375651:0]
2025-11-19T14:00:10.999940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436003905375653:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:10.999964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:10.999986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:11.000019Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:11.000082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436003978843556:2048] [node 1] ICS01 socket: 228 reason# EndOfStream
2025-11-19T14:00:11.000086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436003978843472:2048] [node 1] ICP30 unregister session Session# [4:7574436003978843556:2048] VirtualId# [4:7574436003978843554:0]
2025-11-19T14:00:11.000090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436003978843556:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.000114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:11.000134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:11.012462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:11.012550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436005896857580:2049] [node 2] ICS01 socket: 243 reason# EndOfStream
2025-11-19T14:00:11.012559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436005896857483:2049] [node 2] ICP30 unregister session Session# [3:7574436005896857580:2049] VirtualId# [3:7574436005896857568:0]
2025-11-19T14:00:11.012565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436005896857580:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.012598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:11.012618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:11.022802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:11.022890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436003978843568:2049] [node 2] ICS07 socket disconnect 262 reason# EndOfStream
2025-11-19T14:00:11.022896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436003978843568:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.022955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:11.022977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436003978843568:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:11.023360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:11.023430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436005654353733:2049] [node 2] ICS07 socket disconnect 224 reason# EndOfStream
2025-11-19T14:00:11.023434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436005654353733:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.023464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:11.023485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436005654353733:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:11.023204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:11.023263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436003905375669:2049] [node 2] ICS01 socket: 264 reason# EndOfStream
2025-11-19T14:00:11.023270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436003905375573:2049] [node 2] ICP30 unregister session Session# [5:7574436003905375669:2049] VirtualId# [5:7574436003905375667:0]
2025-11-19T14:00:11.023275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436003905375669:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.023302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:11.023324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:11.024751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436005654353637:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11394) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:11.024763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436005654353733:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:11.024768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436005654353637:2049] [node 2] ICP30 unregister session Session# [6:7574436005654353733:2049] VirtualId# [6:7574436005654353722:0]
2025-11-19T14:00:11.024808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:11.024814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436005654353637:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11394) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:11.024816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11394) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:11.024966Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436003978843473:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11394) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:11.024977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436003978843568:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:11.024981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436003978843473:2049] [node 2] ICP30 unregister session Session# [4:7574436003978843568:2049] VirtualId# [4:7574436003978843565:0]
2025-11-19T14:00:11.025013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:11.025016Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436003978843473:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11394) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:11.025018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11394) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:11.032036Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:11.032128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436005654353730:2050] [node 3] ICS01 socket: 226 reason# EndOfStream
2025-11-19T14:00:11.032137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436005654353638:2050] [node 3] ICP30 unregister session Session# [6:7574436005654353730:2050] VirtualId# [6:7574436005654353723:0]
2025-11-19T14:00:11.032145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436005654353730:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.032221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:11.032248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:11.032417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436005654353637:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:11.037710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:11.037809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436003905375673:2050] [node 3] ICS01 socket: 234 reason# EndOfStream
2025-11-19T14:00:11.037818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436003905375574:2050] [node 3] ICP30 unregister session Session# [5:7574436003905375673:2050] VirtualId# [5:7574436003905375671:0]
2025-11-19T14:00:11.037826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436003905375673:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.037881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:11.037906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:11.061114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:11.061219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436005654353738:2059] [node 4] ICS01 socket: 257 reason# EndOfStream
2025-11-19T14:00:11.061231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436005654353656:2059] [node 4] ICP30 unregister session Session# [6:7574436005654353738:2059] VirtualId# [6:7574436005654353732:0]
2025-11-19T14:00:11.061239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436005654353738:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.061307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:11.061332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:11.467591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436009391804984:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:11.467612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436009391804985:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:11.512024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436008421909388:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:11.512047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436008421909389:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:11.558125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436010629654898:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:11.558137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436010629654899:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:11.592178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436011708287881:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:11.592190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436011708287882:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:11.592194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436011708287883:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:11.635932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436010319987838:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:11.635948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436010319987839:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:11.635953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436010319987840:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:11.671858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436010575612908:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:11.671870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436010575612909:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:11.671874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436010575612910:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.695416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436010629654945:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49221
2025-11-19T14:00:11.694122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436009391805039:2101] [node 4] ICP01 ready to work
2025-11-19T14:00:11.696088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436010319987879:2087] [node 6] ICP01 ready to work
2025-11-19T14:00:11.695509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436010629654946:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49221
2025-11-19T14:00:11.695946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436010629654951:2099] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.698042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436008421909453:2104] [node 5] ICP01 ready to work
2025-11-19T14:00:11.698138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436008421909454:2106] [node 6] ICP01 ready to work
2025-11-19T14:00:11.698212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436008421909455:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.708025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436011708287937:2097] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.709134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436010319987894:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44414
2025-11-19T14:00:11.709125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436010629654960:2105] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.709868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436010319987895:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44414
2025-11-19T14:00:11.711343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436009391805054:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.713542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436011708287941:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30216
2025-11-19T14:00:11.713686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436009391804985:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17020128ns) {Configure:104(17019462ns) {ProcessConfigured:135(16653586ns) {StartInitialHandshake:157(16653356ns) {DropHandshakes:469(386ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:476(751ns) {SwitchToState:178(551ns)}}}}}
2025-11-19T14:00:11.713224Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436010319987879:2087] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17063571ns) {Configure:104(17063180ns) {ProcessConfigured:135(16700317ns) {StartInitialHandshake:157(16700118ns) {DropHandshakes:469(252ns) {DropIncomingHandshake:441(40ns) DropOutgoingHandshake:456(29ns)} GenerateSessionVirtualId:988(67ns) PrepareNewSessionHandshake:476(1511ns) {SwitchToState:178(449ns)}}}}}
2025-11-19T14:00:11.714720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436008421909444:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3372
2025-11-19T14:00:11.715452Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436010629654946:2095] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.714715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436009391805050:2106] [node 5] ICP01 ready to work
2025-11-19T14:00:11.715849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436011708287927:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30216
2025-11-19T14:00:11.716043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436010575612924:2062] [node 4] ICP01 ready to work
2025-11-19T14:00:11.719168Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436010575612909:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(21101770ns) {Configure:104(21101184ns) {ProcessConfigured:135(20745358ns) {StartInitialHandshake:157(20744653ns) {DropHandshakes:469(498ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:476(1213ns) {SwitchToState:178(571ns)}}}}}
2025-11-19T14:00:11.716167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436010629654969:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:11.719876Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436008421909444:2101] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.719897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436008421909453:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21631190ns) {Configure:104(21630772ns) {ProcessConfigured:135(21296606ns) {StartInitialHandshake:157(21296344ns) {DropHandshakes:469(494ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(42ns) PrepareNewSessionHandshake:476(1784ns) {SwitchToState:178(33ns)}}}}}
2025-11-19T14:00:11.723277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436011708287927:2091] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.719948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436008421909388:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.724102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436010575612945:2065] [node 5] ICP01 ready to work
2025-11-19T14:00:11.724224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436010319987883:2089] [node 4] ICP01 ready to work
2025-11-19T14:00:11.726838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436010629654899:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(31158355ns) {Configure:104(31157787ns) {ProcessConfigured:135(30745314ns) {StartInitialHandshake:157(30744688ns) {DropHandshakes:469(466ns) {DropIncomingHandshake:441(119ns) DropOutgoingHandshake:456(79ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(1490ns) {SwitchToState:178(981ns)}}}}}
2025-11-19T14:00:11.726928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436010629654899:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.727242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436008421909454:2106] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(28641473ns) {Configure:104(12607708ns) {ProcessConfigured:135(12232569ns) {StartInitialHandshake:157(12232061ns) {DropHandshakes:469(470ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(44ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:476(1310ns) {SwitchToState:178(389ns)}}}}}
2025-11-19T14:00:11.728080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436010575612908:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(28077963ns) {Configure:104(13267835ns) {ProcessConfigured:135(12916879ns) {StartInitialHandshake:157(12916659ns) {DropHandshakes:469(495ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(37ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(710ns) {SwitchToState:178(275ns)}}}}}
2025-11-19T14:00:11.727739Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436008421909442:2100] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.727762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436008421909442:2100] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:11.730426Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436008421909455:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(31804034ns) {Configure:104(15390548ns) {ProcessConfigured:135(15168675ns) {StartInitialHandshake:157(15167848ns) {DropHandshakes:469(113ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(24ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(846ns) {SwitchToState:178(18ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.731952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436010575612983:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52850
2025-11-19T14:00:11.732023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436010575612984:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52850
2025-11-19T14:00:11.732142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436010575612910:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(33938413ns) {Configure:104(16276584ns) {ProcessConfigured:135(15902541ns) {StartInitialHandshake:157(15902039ns) {DropHandshakes:469(480ns) {DropIncomingHandshake:441(30ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(901ns) {SwitchToState:178(128ns)}}}}}
2025-11-19T14:00:11.735871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436010629654898:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(19737470ns) {Configure:104(19736637ns) {ProcessConfigured:135(19353247ns) {StartInitialHandshake:157(19352716ns) {DropHandshakes:469(454ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:476(1385ns) {SwitchToState:178(317ns)}}}}}
2025-11-19T14:00:11.739690Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436009391805043:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.739704Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436009391805043:2103] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.740176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436008421909469:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3372
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.740197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436008421909470:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3372
2025-11-19T14:00:11.740239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436008421909471:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3372
2025-11-19T14:00:11.740972Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436010575612983:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.741048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436010575612908:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.742064Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436008421909470:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.743143Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436009391805041:2102] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.743163Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436009391805041:2102] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:11.741328Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436010319987895:2101] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.750833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436010575612945:2065] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(26594539ns) {Configure:104(26593676ns) {ProcessConfigured:135(26251134ns) {StartInitialHandshake:157(26249944ns) {DropHandshakes:469(544ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(99ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(1426ns) {SwitchToState:178(321ns)}}}}}
2025-11-19T14:00:11.750994Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436010629654945:2094] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.750729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436011708287959:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30216
2025-11-19T14:00:11.751622Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436011708287941:2100] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.751663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436011708287883:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.751855Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436010629654964:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.751865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436010629654964:2107] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:11.754727Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436010575612984:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.754767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436010575612945:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.753910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436010319987883:2089] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(29593937ns) {Configure:104(29593188ns) {ProcessConfigured:135(29267221ns) {StartInitialHandshake:157(29266555ns) {DropHandshakes:469(527ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(284ns) PrepareNewSessionHandshake:476(1082ns) {SwitchToState:178(540ns)}}}}}
2025-11-19T14:00:11.754114Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436010319987894:2100] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.754194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436010319987883:2089] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.754997Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436010319987896:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.755001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436010319987896:2102] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:11.755919Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436010629654964:2107] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:11.756150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436010629654951:2099] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.756174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:11.756182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436010629654951:2099] [node 4] ICP22 created new session: [3:7574436010629654981:2099]
2025-11-19T14:00:11.756188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436010629654981:2099] [node 4] ICS09 handshake done sender: [3:7574436010629654964:2107] self: [3:7574436010629654953:0] peer: [4:7574436011708287960:0] socket: 224 qp: -1
2025-11-19T14:00:11.756195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436010629654981:2099] [node 4] ICS10 traffic start
2025-11-19T14:00:11.756206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:11.757463Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436008421909442:2100] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:11.757588Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436009391805056:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.757606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436009391805056:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:11.757802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436008421909389:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.757821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:11.757827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436008421909389:2049] [node 3] ICP22 created new session: [2:7574436008421909474:2049]
2025-11-19T14:00:11.757833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436008421909474:2049] [node 3] ICS09 handshake done sender: [2:7574436008421909442:2100] self: [2:7574436008421909441:0] peer: [3:7574436010629654976:0] socket: 211 qp: -1
2025-11-19T14:00:11.757838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436008421909474:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:11.757857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:11.758034Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436009391805043:2103] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:11.758109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436009391805039:2101] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.758119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:11.758124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436009391805039:2101] [node 4] ICP22 created new session: [1:7574436009391805064:2101]
2025-11-19T14:00:11.758128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436009391805064:2101] [node 4] ICS09 handshake done sender: [1:7574436009391805043:2103] self: [1:7574436009391805042:0] peer: [4:7574436011708287958:0] socket: 213 qp: -1
2025-11-19T14:00:11.758132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436009391805064:2101] [node 4] ICS10 traffic start
2025-11-19T14:00:11.758137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:11.757392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436011708287957:2114] [node 6] ICP01 ready to work
2025-11-19T14:00:11.757854Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436010319987896:2102] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:11.759774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436011708287881:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.759820Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436011708287881:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(35999371ns) {IncomingHandshake:261(35998690ns) {DropIncomingHandshake:441(182ns) PrepareNewSessionHandshake:476(478ns) {SwitchToState:178(207ns)} GenerateSessionVirtualId:988(270ns) IssueIncomingHandshakeReply:193(35986357ns) {SwitchToState:178(1127ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.766619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436010319987879:2087] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.766648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:11.766657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436010319987879:2087] [node 6] ICP22 created new session: [5:7574436010319987918:2087]
2025-11-19T14:00:11.766664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436010319987918:2087] [node 6] ICS09 handshake done sender: [5:7574436010319987896:2102] self: [5:7574436010319987880:0] peer: [6:7574436010575612989:0] socket: 227 qp: -1
2025-11-19T14:00:11.766675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436010319987918:2087] [node 6] ICS10 traffic start
2025-11-19T14:00:11.766696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:11.765137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436009391805066:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28426
2025-11-19T14:00:11.765161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436009391805067:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28426
2025-11-19T14:00:11.765251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436009391805068:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28426
2025-11-19T14:00:11.765263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436009391805069:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28426
2025-11-19T14:00:11.767432Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436010629654969:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(51216135ns) {Configure:104(51215661ns) {ProcessConfigured:135(50658955ns) {StartInitialHandshake:157(50658320ns) {DropHandshakes:469(179ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(18ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:476(1114ns) {SwitchToState:178(506ns)}}}}}
2025-11-19T14:00:11.765439Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436011708287882:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(49413187ns) {Configure:104(49412135ns) {ProcessConfigured:135(49033591ns) {StartInitialHandshake:157(49033079ns) {DropHandshakes:469(353ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:476(1110ns) {SwitchToState:178(802ns)}}}}}
2025-11-19T14:00:11.767031Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436010319987838:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(50735417ns) {Configure:104(50734628ns) {ProcessConfigured:135(50403976ns) {StartInitialHandshake:157(50403531ns) {DropHandshakes:469(462ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:476(331ns) {SwitchToState:178(69ns)}}}}}
2025-11-19T14:00:11.767262Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436010319987840:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(50937137ns) {Configure:104(50936833ns) {ProcessConfigured:135(50605177ns) {StartInitialHandshake:157(50604962ns) {DropHandshakes:469(214ns) {DropIncomingHandshake:441(24ns) DropOutgoingHandshake:456(37ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(419ns) {SwitchToState:178(243ns)}}}}}
2025-11-19T14:00:11.767295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436010319987840:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.769224Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436009391805067:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.771883Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436011708287939:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.771916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436011708287939:2099] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:11.771863Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436008421909471:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.772281Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436010629654962:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.772296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436010629654962:2106] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:11.771980Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436009391805041:2102] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.776307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436010319987924:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44414
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.780962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436011708287883:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17856) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436011708287941:2100] held: no
2025-11-19T14:00:11.781181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436011708287963:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30216
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.781970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436009391804984:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.781995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:11.782002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436009391804984:2048] [node 2] ICP22 created new session: [1:7574436009391805073:2048]
2025-11-19T14:00:11.782009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436009391805073:2048] [node 2] ICS09 handshake done sender: [1:7574436009391805041:2102] self: [1:7574436009391805040:0] peer: [2:7574436008421909465:0] socket: 212 qp: -1
2025-11-19T14:00:11.782015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436009391805073:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:11.782024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:11.782762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436010629654899:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11277) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436010629654946:2095] held: no
2025-11-19T14:00:11.782982Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436010319987924:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.783033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436010319987839:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.781333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436010575612990:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52850
2025-11-19T14:00:11.782422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436010575612991:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52850
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.783631Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436010629654951:2099] [node 4] ICP35 event processing took too much time PendingConnection:278(27378590ns) {HandleHandshakeStatus:329(27377222ns) {DropIncomingHandshake:441(5082ns) DropOutgoingHandshake:456(205ns) SwitchToState:178(196ns) ProcessPendingSessionEvents:495(25548791ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25530412ns max# 25460934ns}}}
2025-11-19T14:00:11.784481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436010629654985:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49221
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.784657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436010319987926:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44414
2025-11-19T14:00:11.784923Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436008421909464:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.784936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436008421909464:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.785833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436010319987927:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44414
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.785944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436010629654986:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49221
2025-11-19T14:00:11.785993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436010629654987:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49221
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.783270Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436010575612984:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:11.783355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436010575612945:2065] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.783369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:11.783374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436010575612945:2065] [node 5] ICP22 created new session: [6:7574436010575612992:2065]
2025-11-19T14:00:11.783379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436010575612992:2065] [node 5] ICS09 handshake done sender: [6:7574436010575612984:2113] self: [6:7574436010575612989:0] peer: [5:7574436010319987880:0] socket: 238 qp: -1
2025-11-19T14:00:11.783384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436010575612992:2065] [node 5] ICS10 traffic start
2025-11-19T14:00:11.783429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.786186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436011708287964:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30216
2025-11-19T14:00:11.786250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436011708287965:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30216
2025-11-19T14:00:11.786591Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436011708287964:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.786613Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436011708287941:2100] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:11.786686Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436011708287927:2091] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:11.787328Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436009391805069:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.787418Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436009391805056:2110] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:11.787198Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436010629654946:2095] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:11.786750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436011708287883:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.786765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:11.786771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436011708287883:2050] [node 3] ICP22 created new session: [4:7574436011708287967:2050]
2025-11-19T14:00:11.786776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436011708287967:2050] [node 3] ICS09 handshake done sender: [4:7574436011708287941:2100] self: [4:7574436011708287960:0] peer: [3:7574436010629654953:0] socket: 228 qp: -1
2025-11-19T14:00:11.786780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436011708287967:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:11.786795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:11.786966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436011708287881:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.786974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:11.786977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436011708287881:2048] [node 1] ICP22 created new session: [4:7574436011708287969:2048]
2025-11-19T14:00:11.786982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436011708287969:2048] [node 1] ICS09 handshake done sender: [4:7574436011708287927:2091] self: [4:7574436011708287958:0] peer: [1:7574436009391805042:0] socket: 217 qp: -1
2025-11-19T14:00:11.786985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436011708287969:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:11.786989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:11.787933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436011708287957:2114] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27892619ns) {Configure:104(27891676ns) {ProcessConfigured:135(27376068ns) {StartInitialHandshake:157(27375202ns) {DropHandshakes:469(665ns) {DropIncomingHandshake:441(136ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(419ns) PrepareNewSessionHandshake:476(771ns) {SwitchToState:178(529ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.789703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436009391805054:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.789742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:11.789794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436009391805054:2109] [node 6] ICP22 created new session: [1:7574436009391805078:2109]
2025-11-19T14:00:11.789824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436009391805078:2109] [node 6] ICS09 handshake done sender: [1:7574436009391805056:2110] self: [1:7574436009391805055:0] peer: [6:7574436010575612986:0] socket: 225 qp: -1
2025-11-19T14:00:11.789853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436009391805078:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:11.789884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:11.788631Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436008421909464:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.793904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436008421909389:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(36003204ns) {HandleHandshakeStatus:329(36002493ns) {DropIncomingHandshake:441(4074ns) DropOutgoingHandshake:456(212ns) SwitchToState:178(77ns) ProcessPendingSessionEvents:495(21714136ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21691170ns max# 21623313ns}}}
2025-11-19T14:00:11.793932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436008421909453:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.793946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:11.793953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436008421909453:2104] [node 5] ICP22 created new session: [2:7574436008421909479:2104]
2025-11-19T14:00:11.793957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436008421909479:2104] [node 5] ICS09 handshake done sender: [2:7574436008421909464:2113] self: [2:7574436008421909456:0] peer: [5:7574436010319987925:0] socket: 232 qp: -1
2025-11-19T14:00:11.793963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436008421909479:2104] [node 5] ICS10 traffic start
2025-11-19T14:00:11.793970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:11.794177Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436008421909388:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2671) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436008421909444:2101] held: no
2025-11-19T14:00:11.794845Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436010629654986:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.794966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436010629654898:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.795094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436010629654899:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.795102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:11.795109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436010629654899:2049] [node 2] ICP22 created new session: [3:7574436010629654990:2049]
2025-11-19T14:00:11.795115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436010629654990:2049] [node 2] ICS09 handshake done sender: [3:7574436010629654946:2095] self: [3:7574436010629654976:0] peer: [2:7574436008421909441:0] socket: 216 qp: -1
2025-11-19T14:00:11.795121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436010629654990:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:11.795127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:11.794638Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436010575612991:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.794699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436010575612996:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52850
2025-11-19T14:00:11.794733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436010575612909:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.794737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436009391805039:2101] [node 4] ICP35 event processing took too much time PendingConnection:278(36535029ns) {HandleHandshakeStatus:329(36534441ns) {DropIncomingHandshake:441(135ns) DropOutgoingHandshake:456(121ns) SwitchToState:178(185ns) ProcessPendingSessionEvents:495(22400761ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22374861ns max# 22313687ns}}}
2025-11-19T14:00:11.794894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574436009391805039:2101] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:11.796033Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436010319987924:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:11.796103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436010319987839:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.796117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:11.796123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436010319987839:2049] [node 2] ICP22 created new session: [5:7574436010319987928:2049]
2025-11-19T14:00:11.796129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436010319987928:2049] [node 2] ICS09 handshake done sender: [5:7574436010319987924:2117] self: [5:7574436010319987925:0] peer: [2:7574436008421909456:0] socket: 234 qp: -1
2025-11-19T14:00:11.796134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436010319987928:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:11.796142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:11.795443Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436011708287939:2099] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:11.795163Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436008421909467:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.795175Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436008421909467:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:11.796419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436011708287937:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.796435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:11.796441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436011708287937:2097] [node 5] ICP22 created new session: [4:7574436011708287975:2097]
2025-11-19T14:00:11.796447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436011708287975:2097] [node 5] ICS09 handshake done sender: [4:7574436011708287939:2099] self: [4:7574436011708287938:0] peer: [5:7574436010319987915:0] socket: 220 qp: -1
2025-11-19T14:00:11.796452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436011708287975:2097] [node 5] ICS10 traffic start
2025-11-19T14:00:11.796458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:11.796429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436009391805058:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.796438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436009391805058:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:11.797121Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436010629654962:2106] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:11.796625Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436010575612996:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.796653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436010575612924:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.796870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436010575612995:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52850
2025-11-19T14:00:11.797157Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436010575612983:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:11.797663Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436011708287972:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.797670Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436011708287972:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:11.798230Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436011708287972:2120] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:11.798264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436010575612908:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.798279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:11.798284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436010575612908:2048] [node 1] ICP22 created new session: [6:7574436010575612999:2048]
2025-11-19T14:00:11.798289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436010575612999:2048] [node 1] ICS09 handshake done sender: [6:7574436010575612983:2112] self: [6:7574436010575612986:0] peer: [1:7574436009391805055:0] socket: 229 qp: -1
2025-11-19T14:00:11.798293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436010575612999:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:11.798299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:11.798912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436010629654960:2105] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.798927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:11.798934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436010629654960:2105] [node 5] ICP22 created new session: [3:7574436010629654993:2105]
2025-11-19T14:00:11.798939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436010629654993:2105] [node 5] ICS09 handshake done sender: [3:7574436010629654962:2106] self: [3:7574436010629654961:0] peer: [5:7574436010319987923:0] socket: 222 qp: -1
2025-11-19T14:00:11.798945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436010629654993:2105] [node 5] ICS10 traffic start
2025-11-19T14:00:11.798953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:11.799172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436011708287957:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.799184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:11.799189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436011708287957:2114] [node 6] ICP22 created new session: [4:7574436011708287978:2114]
2025-11-19T14:00:11.799193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436011708287978:2114] [node 6] ICS09 handshake done sender: [4:7574436011708287972:2120] self: [4:7574436011708287961:0] peer: [6:7574436010575612998:0] socket: 236 qp: -1
2025-11-19T14:00:11.799198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436011708287978:2114] [node 6] ICS10 traffic start
2025-11-19T14:00:11.799203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:11.796981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436009391805077:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28426
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.805282Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436008421909467:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:11.805395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436008421909454:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.805407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:11.805413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436008421909454:2106] [node 6] ICP22 created new session: [2:7574436008421909483:2106]
2025-11-19T14:00:11.805419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436008421909483:2106] [node 6] ICS09 handshake done sender: [2:7574436008421909467:2114] self: [2:7574436008421909460:0] peer: [6:7574436010575612997:0] socket: 250 qp: -1
2025-11-19T14:00:11.805424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436008421909483:2106] [node 6] ICS10 traffic start
2025-11-19T14:00:11.805431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:11.805058Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436010319987926:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.805117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436010319987930:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44414
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.805773Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436009391805077:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.807963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436010319987931:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44414
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.811921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436011708287980:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30216
2025-11-19T14:00:11.812343Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436011708287980:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.812378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436011708287882:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.812494Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436008421909468:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.812506Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436008421909468:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.813420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436011708287982:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30216
2025-11-19T14:00:11.813555Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436008421909468:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:11.813637Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436011708287980:2121] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:11.813728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436011708287882:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.813746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:11.813752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436011708287882:2049] [node 2] ICP22 created new session: [4:7574436011708287983:2049]
2025-11-19T14:00:11.813758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436011708287983:2049] [node 2] ICS09 handshake done sender: [4:7574436011708287980:2121] self: [4:7574436011708287981:0] peer: [2:7574436008421909461:0] socket: 219 qp: -1
2025-11-19T14:00:11.813763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436011708287983:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:11.813771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:11.814112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436008421909455:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.814123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:11.814129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436008421909455:2108] [node 4] ICP22 created new session: [2:7574436008421909486:2108]
2025-11-19T14:00:11.814134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436008421909486:2108] [node 4] ICS09 handshake done sender: [2:7574436008421909468:2115] self: [2:7574436008421909461:0] peer: [4:7574436011708287981:0] socket: 271 qp: -1
2025-11-19T14:00:11.814138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436008421909486:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:11.814143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:11.816658Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436009391805062:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.816671Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436009391805062:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.816016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436010319987839:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(19799358ns) {HandleHandshakeStatus:329(19798888ns) {DropIncomingHandshake:441(196ns) DropOutgoingHandshake:456(1572ns) SwitchToState:178(149ns) ProcessPendingSessionEvents:495(19717421ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19691946ns max# 19626830ns}}}
2025-11-19T14:00:11.816044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436010319987838:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.816318Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436010319987894:2100] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:11.817795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436010575613001:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52850
2025-11-19T14:00:11.817818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436010575613002:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52850
2025-11-19T14:00:11.818125Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436011708287937:2097] [node 5] ICP35 event processing took too much time PendingConnection:278(21600566ns) {HandleHandshakeStatus:329(21599893ns) {DropIncomingHandshake:441(3476ns) DropOutgoingHandshake:456(92ns) SwitchToState:178(77ns) ProcessPendingSessionEvents:495(21522584ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21507084ns max# 21446043ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.819096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436010629654995:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49221
2025-11-19T14:00:11.818649Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436009391805062:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:11.818221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436010319987883:2089] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.818238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:11.818243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436010319987883:2089] [node 4] ICP22 created new session: [5:7574436010319987934:2089]
2025-11-19T14:00:11.818248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436010319987934:2089] [node 4] ICS09 handshake done sender: [5:7574436010319987894:2100] self: [5:7574436010319987915:0] peer: [4:7574436011708287938:0] socket: 221 qp: -1
2025-11-19T14:00:11.818253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436010319987934:2089] [node 4] ICS10 traffic start
2025-11-19T14:00:11.818260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:11.818546Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436010319987895:2101] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:11.819692Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436009391805058:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.819857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436010575613003:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52850
2025-11-19T14:00:11.819792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436009391804985:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.819808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:11.819818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436009391804985:2049] [node 3] ICP22 created new session: [1:7574436009391805084:2049]
2025-11-19T14:00:11.819824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436009391805084:2049] [node 3] ICS09 handshake done sender: [1:7574436009391805058:2111] self: [1:7574436009391805047:0] peer: [3:7574436010629654989:0] socket: 254 qp: -1
2025-11-19T14:00:11.819830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436009391805084:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:11.819836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:11.820138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436010319987933:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44414
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.820313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436008421909488:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3372
2025-11-19T14:00:11.820438Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436008421909469:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.820463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436008421909454:2106] [node 6] ICP18 handshake (actor [2:7574436008421909469:2116]) is too old
2025-11-19T14:00:11.820835Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436008421909444:2101] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:11.821268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436010319987840:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.821283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:11.821289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436010319987840:2050] [node 3] ICP22 created new session: [5:7574436010319987936:2050]
2025-11-19T14:00:11.821294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436010319987936:2050] [node 3] ICS09 handshake done sender: [5:7574436010319987895:2101] self: [5:7574436010319987923:0] peer: [3:7574436010629654961:0] socket: 223 qp: -1
2025-11-19T14:00:11.821299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436010319987936:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:11.821306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:11.822030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436008421909388:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.822044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:11.822050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436008421909388:2048] [node 1] ICP22 created new session: [2:7574436008421909490:2048]
2025-11-19T14:00:11.822055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436008421909490:2048] [node 1] ICS09 handshake done sender: [2:7574436008421909444:2101] self: [2:7574436008421909465:0] peer: [1:7574436009391805040:0] socket: 215 qp: -1
2025-11-19T14:00:11.822061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436008421909490:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:11.822068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:11.822104Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436010575612996:2120] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:11.822171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436009391805050:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.822188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:11.822193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436009391805050:2106] [node 5] ICP22 created new session: [1:7574436009391805086:2106]
2025-11-19T14:00:11.822197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436009391805086:2106] [node 5] ICS09 handshake done sender: [1:7574436009391805062:2114] self: [1:7574436009391805060:0] peer: [5:7574436010319987932:0] socket: 262 qp: -1
2025-11-19T14:00:11.822201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436009391805086:2106] [node 5] ICS10 traffic start
2025-11-19T14:00:11.822220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.820606Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436010629654987:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.820987Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436010629654898:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2671) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436010629654986:2118] held: no
2025-11-19T14:00:11.823987Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436010629654986:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:11.824756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436010629654898:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.824770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:11.824777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436010629654898:2048] [node 1] ICP22 created new session: [3:7574436010629654997:2048]
2025-11-19T14:00:11.824783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436010629654997:2048] [node 1] ICS09 handshake done sender: [3:7574436010629654986:2118] self: [3:7574436010629654989:0] peer: [1:7574436009391805047:0] socket: 264 qp: -1
2025-11-19T14:00:11.824789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436010629654997:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:11.824797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:11.825437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436008421909489:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3372
2025-11-19T14:00:11.825690Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436010575612909:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11277) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436010575612991:2118] held: no
2025-11-19T14:00:11.825775Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436010575612991:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:11.825920Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436011708287959:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.825957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574436011708287957:2114] [node 6] ICP18 handshake (actor [4:7574436011708287959:2115]) is too old
2025-11-19T14:00:11.832155Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436010575613001:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.832176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436010575612909:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.832192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:11.832199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436010575612909:2049] [node 2] ICP22 created new session: [6:7574436010575613004:2049]
2025-11-19T14:00:11.832205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436010575613004:2049] [node 2] ICS09 handshake done sender: [6:7574436010575612991:2118] self: [6:7574436010575612997:0] peer: [2:7574436008421909460:0] socket: 259 qp: -1
2025-11-19T14:00:11.832213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436010575613004:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:11.832235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:11.832419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436010575612924:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.832426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:11.832431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436010575612924:2062] [node 4] ICP22 created new session: [6:7574436010575613006:2062]
2025-11-19T14:00:11.832435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436010575613006:2062] [node 4] ICS09 handshake done sender: [6:7574436010575612996:2120] self: [6:7574436010575612998:0] peer: [4:7574436011708287961:0] socket: 251 qp: -1
2025-11-19T14:00:11.832438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436010575613006:2062] [node 4] ICS10 traffic start
2025-11-19T14:00:11.832443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:11.834663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436010575612910:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:11.834795Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436010629654984:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:11.834815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436010629654984:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:11.836021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436010319987838:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2671) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436010319987926:2118] held: no
2025-11-19T14:00:11.836181Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436010319987926:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:11.836562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436010319987838:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.836573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:11.836583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436010319987838:2048] [node 1] ICP22 created new session: [5:7574436010319987939:2048]
2025-11-19T14:00:11.836589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436010319987939:2048] [node 1] ICS09 handshake done sender: [5:7574436010319987926:2118] self: [5:7574436010319987932:0] peer: [1:7574436009391805060:0] socket: 266 qp: -1
2025-11-19T14:00:11.836598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436010319987939:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:11.836610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:11.835639Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436010629654984:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:11.835719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436010629654969:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:11.835735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:11.835741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436010629654969:2110] [node 6] ICP22 created new session: [3:7574436010629655000:2110]
2025-11-19T14:00:11.835797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436010629655000:2110] [node 6] ICS09 handshake done sender: [3:7574436010629654984:2116] self: [3:7574436010629654971:0] peer: [6:7574436010575613008:0] socket: 270 qp: -1
2025-11-19T14:00:11.835803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436010629655000:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:11.835810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:11.844879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436010575612910:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17856) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436010575613001:2121] held: no
2025-11-19T14:00:11.845068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436010575613009:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52850
2025-11-19T14:00:11.846354Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436010575613001:2121] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:11.846613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436010575612910:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:11.846631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:11.846637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436010575612910:2050] [node 3] ICP22 created new session: [6:7574436010575613010:2050]
2025-11-19T14:00:11.846645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436010575613010:2050] [node 3] ICS09 handshake done sender: [6:7574436010575613001:2121] self: [6:7574436010575613008:0] peer: [3:7574436010629654971:0] socket: 273 qp: -1
2025-11-19T14:00:11.846652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436010575613010:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:11.846661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:11.848247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436008421909455:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:11.867482Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:11.867576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436008421909490:2048] [node 1] ICS01 socket: 215 reason# EndOfStream
2025-11-19T14:00:11.867585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436008421909388:2048] [node 1] ICP30 unregister session Session# [2:7574436008421909490:2048] VirtualId# [2:7574436008421909465:0]
2025-11-19T14:00:11.867592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436008421909490:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.867632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:11.867653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:11.868182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:11.868250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436010629654997:2048] [node 1] ICS01 socket: 264 reason# EndOfStream
2025-11-19T14:00:11.868256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436010629654898:2048] [node 1] ICP30 unregister session Session# [3:7574436010629654997:2048] VirtualId# [3:7574436010629654989:0]
2025-11-19T14:00:11.868261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436010629654997:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.868288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:11.868305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:11.893939Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:11.894054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436011708287969:2048] [node 1] ICS01 socket: 217 reason# EndOfStream
2025-11-19T14:00:11.894063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436011708287881:2048] [node 1] ICP30 unregister session Session# [4:7574436011708287969:2048] VirtualId# [4:7574436011708287958:0]
2025-11-19T14:00:11.894070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436011708287969:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.894128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:11.894155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:11.894353Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:11.894424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436010575612999:2048] [node 1] ICS01 socket: 229 reason# EndOfStream
2025-11-19T14:00:11.894428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436010575612908:2048] [node 1] ICP30 unregister session Session# [6:7574436010575612999:2048] VirtualId# [6:7574436010575612986:0]
2025-11-19T14:00:11.894433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436010575612999:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.894459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:11.894480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:11.900125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:11.900210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436010629654990:2049] [node 2] ICS01 socket: 216 reason# EndOfStream
2025-11-19T14:00:11.900220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436010629654899:2049] [node 2] ICP30 unregister session Session# [3:7574436010629654990:2049] VirtualId# [3:7574436010629654976:0]
2025-11-19T14:00:11.900227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436010629654990:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.900265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:11.900283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:11.894197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:11.894259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436010319987939:2048] [node 1] ICS01 socket: 266 reason# EndOfStream
2025-11-19T14:00:11.894263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436010319987838:2048] [node 1] ICP30 unregister session Session# [5:7574436010319987939:2048] VirtualId# [5:7574436010319987932:0]
2025-11-19T14:00:11.894267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436010319987939:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.894297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:11.894321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:11.903905Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:11.903990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436011708287983:2049] [node 2] ICS01 socket: 219 reason# EndOfStream
2025-11-19T14:00:11.903997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436011708287882:2049] [node 2] ICP30 unregister session Session# [4:7574436011708287983:2049] VirtualId# [4:7574436011708287981:0]
2025-11-19T14:00:11.904004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436011708287983:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.904051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:11.904073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:11.907960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:11.908046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574436010319987928:2049] [node 2] ICS07 socket disconnect 234 reason# EndOfStream
2025-11-19T14:00:11.908052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436010319987928:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.908104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:11.908127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574436010319987928:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:11.908357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:11.908423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436010575613004:2049] [node 2] ICS01 socket: 259 reason# EndOfStream
2025-11-19T14:00:11.908428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436010575612909:2049] [node 2] ICP30 unregister session Session# [6:7574436010575613004:2049] VirtualId# [6:7574436010575612997:0]
2025-11-19T14:00:11.908434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436010575613004:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.908466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:11.908488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:11.916038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436010319987839:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11277) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:11.916068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436010319987928:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:11.916075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436010319987839:2049] [node 2] ICP30 unregister session Session# [5:7574436010319987928:2049] VirtualId# [5:7574436010319987925:0]
2025-11-19T14:00:11.916142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:11.916151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574436010319987839:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11277) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:11.916156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11277) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:11.926257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574436010319987839:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:11.931105Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:11.931209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436010319987936:2050] [node 3] ICS01 socket: 223 reason# EndOfStream
2025-11-19T14:00:11.931222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436010319987840:2050] [node 3] ICP30 unregister session Session# [5:7574436010319987936:2050] VirtualId# [5:7574436010319987923:0]
2025-11-19T14:00:11.931231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436010319987936:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.931289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:11.931318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:11.932282Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:11.932356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436010575613010:2050] [node 3] ICS01 socket: 273 reason# EndOfStream
2025-11-19T14:00:11.932362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436010575612910:2050] [node 3] ICP30 unregister session Session# [6:7574436010575613010:2050] VirtualId# [6:7574436010575613008:0]
2025-11-19T14:00:11.932367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436010575613010:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.932414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:11.932435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:11.973202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:11.973313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436010575613006:2062] [node 4] ICS01 socket: 251 reason# EndOfStream
2025-11-19T14:00:11.973323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436010575612924:2062] [node 4] ICP30 unregister session Session# [6:7574436010575613006:2062] VirtualId# [6:7574436010575612998:0]
2025-11-19T14:00:11.973330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436010575613006:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:11.973399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:11.973424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:12.152189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436014367007101:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:12.152205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436014367007102:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:12.187923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436012882376309:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:12.187936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436012882376310:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:12.234423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436014535222420:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:12.234436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436014535222421:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:12.283843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436012684378964:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:12.283859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436012684378965:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:12.283865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436012684378966:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:12.305990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436015459500901:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:12.306011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436015459500902:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:12.306016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436015459500903:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:12.334562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436014367007144:2089] [node 6] ICP01 ready to work
2025-11-19T14:00:12.335473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436014367007149:2090] [node 4] ICP01 ready to work
2025-11-19T14:00:12.335578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436014367007150:2091] [node 5] ICP01 ready to work
2025-11-19T14:00:12.336434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436015459500935:2080] [node 6] ICP01 ready to work
2025-11-19T14:00:12.336020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436012882376357:2092] [node 4] ICP01 ready to work
2025-11-19T14:00:12.336116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436012882376358:2093] [node 5] ICP01 ready to work
2025-11-19T14:00:12.336187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436012882376359:2094] [node 6] ICP01 ready to work
2025-11-19T14:00:12.347687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436014535222476:2077] [node 4] ICP01 ready to work
2025-11-19T14:00:12.347817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436014535222479:2078] [node 5] ICP01 ready to work
2025-11-19T14:00:12.347347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436015459500956:2097] [node 4] ICP01 ready to work
2025-11-19T14:00:12.347989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436014535222481:2079] [node 6] ICP01 ready to work
2025-11-19T14:00:12.358867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436012684378964:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(19926819ns) {Configure:104(19925731ns) {ProcessConfigured:135(19567849ns) {StartInitialHandshake:157(19567362ns) {DropHandshakes:469(564ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(85ns)} GenerateSessionVirtualId:988(174ns) PrepareNewSessionHandshake:476(689ns) {SwitchToState:178(149ns)}}}}}
2025-11-19T14:00:12.359099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436012684379029:2076] [node 6] ICP01 ready to work
2025-11-19T14:00:12.359298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436012684379030:2080] [node 5] ICP01 ready to work
2025-11-19T14:00:12.357622Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436014367007102:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21969209ns) {Configure:104(21968871ns) {ProcessConfigured:135(21632527ns) {StartInitialHandshake:157(21632168ns) {DropHandshakes:469(519ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(1081ns) {SwitchToState:178(301ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.361101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436012882376309:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24853390ns) {Configure:104(24852963ns) {ProcessConfigured:135(24502970ns) {StartInitialHandshake:157(24502626ns) {DropHandshakes:469(458ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:476(1012ns) {SwitchToState:178(474ns)}}}}}
2025-11-19T14:00:12.360657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436012684379031:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28773
2025-11-19T14:00:12.360683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436012684379032:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28773
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.360819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436015699943901:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:12.360891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436015699943902:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:12.360945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436015699943903:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:12.361240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436015699943959:2059] [node 4] ICP01 ready to work
2025-11-19T14:00:12.361354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436015699943960:2062] [node 5] ICP01 ready to work
2025-11-19T14:00:12.375894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436015459500902:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(28392227ns) {Configure:104(28388676ns) {ProcessConfigured:135(27965206ns) {StartInitialHandshake:157(27964356ns) {DropHandshakes:469(797ns) {DropIncomingHandshake:441(122ns) DropOutgoingHandshake:456(148ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:476(1762ns) {SwitchToState:178(1015ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.377048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436015459500976:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2648
2025-11-19T14:00:12.378378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436014535222497:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20230
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.382905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436012882376385:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13686
2025-11-19T14:00:12.382928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436012882376386:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13686
2025-11-19T14:00:12.384684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436014367007144:2089] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29652) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:12.384792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [1:7574436014367007144:2089] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29652) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:12.384799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29652) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.384949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436014367007178:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31778
2025-11-19T14:00:12.384518Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436012684379029:2076] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29652) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:12.384628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436012684379029:2076] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29652) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:12.384631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29652) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.393574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436014535222500:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20230
2025-11-19T14:00:12.395955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436015699943901:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(34496734ns) {Configure:104(34495776ns) {ProcessConfigured:135(34092302ns) {StartInitialHandshake:157(34091739ns) {DropHandshakes:469(440ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:476(2236ns) {SwitchToState:178(901ns)}}}}}
2025-11-19T14:00:12.399425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [1:7574436014367007144:2089] [node 6] ICP33 wake up from error state
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.399359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574436012684379029:2076] [node 6] ICP33 wake up from error state
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.402021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436014367007181:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31778
2025-11-19T14:00:12.404006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436012684379044:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28773
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.406196Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436012684379031:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.407616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436012684378966:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:12.405085Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436014535222500:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.405449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436014535222421:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:12.405490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436014535222501:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20230
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.410459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436014367007182:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31778
2025-11-19T14:00:12.410515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436014367007183:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31778
2025-11-19T14:00:12.410784Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436014367007181:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.410823Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436014367007178:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.412752Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436012882376378:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.412772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436012882376378:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.416706Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436015459500976:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.416854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436015459500903:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:12.416882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436015459500980:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2648
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.418730Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436012684379032:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.418823Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436012684379044:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.418920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436012684378964:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:12.418930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436012684378965:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:12.418952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436012684379047:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28773
2025-11-19T14:00:12.417163Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436014535222486:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.417187Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436014535222486:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:12.417604Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436014535222497:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.417646Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436014535222501:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.417699Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436014535222488:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.417702Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436014535222488:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:12.418389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436014535222420:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:12.419894Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436014535222486:2108] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:12.420086Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436012684379031:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:12.420191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436012684378966:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.420210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:12.420216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436012684378966:2050] [node 3] ICP22 created new session: [4:7574436012684379050:2050]
2025-11-19T14:00:12.420222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436012684379050:2050] [node 3] ICS09 handshake done sender: [4:7574436012684379031:2111] self: [4:7574436012684379046:0] peer: [3:7574436014535222484:0] socket: 218 qp: -1
2025-11-19T14:00:12.420228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436012684379050:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:12.420236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:12.422177Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436014367007168:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.422196Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436014367007168:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:12.422455Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436012882376385:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.423724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436015459500982:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2648
2025-11-19T14:00:12.423758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436015459500983:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2648
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.426154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436015699943970:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54387
2025-11-19T14:00:12.424469Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436012882376380:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.424486Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436012882376380:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:12.425037Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436012882376386:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.427193Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436012882376378:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:12.427352Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436012882376380:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:12.427456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436012882376310:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.427473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:12.427481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436012882376310:2049] [node 3] ICP22 created new session: [2:7574436012882376395:2049]
2025-11-19T14:00:12.427488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436012882376395:2049] [node 3] ICS09 handshake done sender: [2:7574436012882376378:2111] self: [2:7574436012882376377:0] peer: [3:7574436014535222502:0] socket: 234 qp: -1
2025-11-19T14:00:12.427497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436012882376395:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:12.427508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:12.427809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436012882376357:2092] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.427820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:12.427826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436012882376357:2092] [node 4] ICP22 created new session: [2:7574436012882376397:2092]
2025-11-19T14:00:12.427831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436012882376397:2092] [node 4] ICS09 handshake done sender: [2:7574436012882376380:2112] self: [2:7574436012882376379:0] peer: [4:7574436012684379049:0] socket: 235 qp: -1
2025-11-19T14:00:12.427834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436012882376397:2092] [node 4] ICS10 traffic start
2025-11-19T14:00:12.427840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.428088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436012882376393:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13686
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.431228Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436014367007170:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.431245Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436014367007170:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:12.432033Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436014367007183:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.432056Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436014367007182:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.433464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436014367007187:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31778
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.435527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436015699943980:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54387
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.440130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436015459500984:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2648
2025-11-19T14:00:12.441734Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436014535222488:2109] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:12.442004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436014535222476:2077] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.442028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:12.442035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436014535222476:2077] [node 4] ICP22 created new session: [3:7574436014535222510:2077]
2025-11-19T14:00:12.442052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436014535222510:2077] [node 4] ICS09 handshake done sender: [3:7574436014535222486:2108] self: [3:7574436014535222484:0] peer: [4:7574436012684379046:0] socket: 213 qp: -1
2025-11-19T14:00:12.442059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436014535222510:2077] [node 4] ICS10 traffic start
2025-11-19T14:00:12.442071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:12.442505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436014535222479:2078] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.442516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:12.442521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436014535222479:2078] [node 5] ICP22 created new session: [3:7574436014535222512:2078]
2025-11-19T14:00:12.442526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436014535222512:2078] [node 5] ICS09 handshake done sender: [3:7574436014535222488:2109] self: [3:7574436014535222487:0] peer: [5:7574436015459500981:0] socket: 214 qp: -1
2025-11-19T14:00:12.442552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436014535222512:2078] [node 5] ICS10 traffic start
2025-11-19T14:00:12.442559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:12.443417Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436015459500984:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.443640Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436015459500976:2115] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:12.443728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436015459500902:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.443930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436012684378965:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30261) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436012684379044:2117] held: no
2025-11-19T14:00:12.445652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436012684379053:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28773
2025-11-19T14:00:12.443742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436015459500903:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.446169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:12.446179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436015459500903:2050] [node 3] ICP22 created new session: [5:7574436015459500986:2050]
2025-11-19T14:00:12.446195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436015459500986:2050] [node 3] ICS09 handshake done sender: [5:7574436015459500976:2115] self: [5:7574436015459500981:0] peer: [3:7574436014535222487:0] socket: 215 qp: -1
2025-11-19T14:00:12.446203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436015459500986:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:12.446211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:12.449828Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436012882376382:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.449846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436012882376382:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:12.452064Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436012882376382:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.453263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436014535222507:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20230
2025-11-19T14:00:12.453587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436012882376358:2093] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.453610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:12.453617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436012882376358:2093] [node 5] ICP22 created new session: [2:7574436012882376402:2093]
2025-11-19T14:00:12.453624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436012882376402:2093] [node 5] ICS09 handshake done sender: [2:7574436012882376382:2113] self: [2:7574436012882376381:0] peer: [5:7574436015459500985:0] socket: 231 qp: -1
2025-11-19T14:00:12.453631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436012882376402:2093] [node 5] ICS10 traffic start
2025-11-19T14:00:12.453641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.455617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436015459500989:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2648
2025-11-19T14:00:12.456088Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436014367007168:2109] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:12.456307Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436014367007170:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.456816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436014535222508:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20230
2025-11-19T14:00:12.456889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436014535222509:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20230
2025-11-19T14:00:12.457181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436014367007102:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.457208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:12.457214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436014367007102:2049] [node 3] ICP22 created new session: [1:7574436014367007192:2049]
2025-11-19T14:00:12.457221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436014367007192:2049] [node 3] ICS09 handshake done sender: [1:7574436014367007168:2109] self: [1:7574436014367007151:0] peer: [3:7574436014535222505:0] socket: 216 qp: -1
2025-11-19T14:00:12.457227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436014367007192:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:12.457236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:12.457505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436014367007149:2090] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.457520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:12.457524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436014367007149:2090] [node 4] ICP22 created new session: [1:7574436014367007194:2090]
2025-11-19T14:00:12.457528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436014367007194:2090] [node 4] ICS09 handshake done sender: [1:7574436014367007170:2110] self: [1:7574436014367007169:0] peer: [4:7574436012684379048:0] socket: 217 qp: -1
2025-11-19T14:00:12.457532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436014367007194:2090] [node 4] ICS10 traffic start
2025-11-19T14:00:12.457537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:12.458465Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436014535222500:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.458707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436012882376404:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13686
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.458911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436012684379054:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28773
2025-11-19T14:00:12.459090Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436014535222497:2113] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:12.459239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436014535222421:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.459253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:12.459259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436014535222421:2049] [node 2] ICP22 created new session: [3:7574436014535222516:2049]
2025-11-19T14:00:12.459264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436014535222516:2049] [node 2] ICS09 handshake done sender: [3:7574436014535222500:2116] self: [3:7574436014535222502:0] peer: [2:7574436012882376377:0] socket: 240 qp: -1
2025-11-19T14:00:12.459270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436014535222516:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:12.459277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:12.459695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436014535222420:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.459704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:12.459709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436014535222420:2048] [node 1] ICP22 created new session: [3:7574436014535222518:2048]
2025-11-19T14:00:12.459713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436014535222518:2048] [node 1] ICS09 handshake done sender: [3:7574436014535222497:2113] self: [3:7574436014535222505:0] peer: [1:7574436014367007151:0] socket: 221 qp: -1
2025-11-19T14:00:12.459718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436014535222518:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:12.459724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.462184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436012882376405:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13686
2025-11-19T14:00:12.466213Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436015459500980:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.466247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436015459500902:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30261) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436015459500984:2119] held: no
2025-11-19T14:00:12.466351Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436015459500984:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.467465Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436015699943970:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.467694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436015699943960:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:12.467718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436015699943981:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54387
2025-11-19T14:00:12.468642Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436014367007187:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.468702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436014367007144:2089] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:12.468888Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436012882376405:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.468915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436012882376309:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:12.471060Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436015459500983:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.471085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436015459500902:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.471108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:12.471119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436015459500902:2049] [node 2] ICP22 created new session: [5:7574436015459500990:2049]
2025-11-19T14:00:12.471126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436015459500990:2049] [node 2] ICS09 handshake done sender: [5:7574436015459500984:2119] self: [5:7574436015459500985:0] peer: [2:7574436012882376381:0] socket: 262 qp: -1
2025-11-19T14:00:12.471136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436015459500990:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:12.471146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:12.471385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436015459500901:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:12.471546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436015459500956:2097] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:12.472208Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436014367007174:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.472223Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436014367007174:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:12.472690Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436014367007172:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.472694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436014367007172:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:12.472219Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436014535222508:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.474482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436012882376407:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13686
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.474614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436015459500994:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2648
2025-11-19T14:00:12.474856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436015699943980:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.474914Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436015699943967:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.474921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574436015699943967:2110] [node 1] ICRDMA Non success qp response from remote side
2025-11-19T14:00:12.475912Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436012882376393:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.475953Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436012882376384:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.475956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436012882376384:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:12.477190Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436012882376384:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:12.477285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436012882376359:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.477305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:12.477313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436012882376359:2094] [node 6] ICP22 created new session: [2:7574436012882376410:2094]
2025-11-19T14:00:12.477320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436012882376410:2094] [node 6] ICS09 handshake done sender: [2:7574436012882376384:2114] self: [2:7574436012882376383:0] peer: [6:7574436015699943984:0] socket: 257 qp: -1
2025-11-19T14:00:12.477327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436012882376410:2094] [node 6] ICS10 traffic start
2025-11-19T14:00:12.477338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:12.477843Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436015459500954:2098] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.477855Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436015459500954:2098] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:12.479070Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436015459500954:2098] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:12.479178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436015459500935:2080] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.479193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:12.479201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436015459500935:2080] [node 6] ICP22 created new session: [5:7574436015459500996:2080]
2025-11-19T14:00:12.479207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436015459500996:2080] [node 6] ICS09 handshake done sender: [5:7574436015459500954:2098] self: [5:7574436015459500936:0] peer: [6:7574436015699943982:0] socket: 224 qp: -1
2025-11-19T14:00:12.479214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436015459500996:2080] [node 6] ICS10 traffic start
2025-11-19T14:00:12.479224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:12.479685Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436014367007174:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:12.475564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436015699943902:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:12.479589Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436015699943967:2110] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:12.480026Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436012882376405:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:12.480309Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436012684379044:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:12.480506Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436012684379032:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:12.480591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436012684378964:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8828) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436012684379032:2112] held: no
2025-11-19T14:00:12.480695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436012684378964:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.480714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:12.480720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436012684378964:2048] [node 1] ICP22 created new session: [4:7574436012684379056:2048]
2025-11-19T14:00:12.480728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436012684379056:2048] [node 1] ICS09 handshake done sender: [4:7574436012684379032:2112] self: [4:7574436012684379048:0] peer: [1:7574436014367007169:0] socket: 219 qp: -1
2025-11-19T14:00:12.480736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436012684379056:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:12.480745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:12.481074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436012684379040:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.481080Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436012684379040:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:12.480206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436012882376309:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.480223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:12.480228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436012882376309:2048] [node 1] ICP22 created new session: [2:7574436012882376412:2048]
2025-11-19T14:00:12.480238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436012882376412:2048] [node 1] ICS09 handshake done sender: [2:7574436012882376405:2119] self: [2:7574436012882376406:0] peer: [1:7574436014367007173:0] socket: 236 qp: -1
2025-11-19T14:00:12.480245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436012882376412:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:12.480254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.485492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436012684378965:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.485518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:12.485525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436012684378965:2049] [node 2] ICP22 created new session: [4:7574436012684379058:2049]
2025-11-19T14:00:12.485536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436012684379058:2049] [node 2] ICS09 handshake done sender: [4:7574436012684379044:2117] self: [4:7574436012684379049:0] peer: [2:7574436012882376379:0] socket: 239 qp: -1
2025-11-19T14:00:12.485547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436012684379058:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:12.485558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:12.486055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436012684379055:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28773
2025-11-19T14:00:12.487317Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436014367007172:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.487783Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436015459500980:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.487110Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436015699943981:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.487135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574436015699943901:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.487159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:12.487169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436015699943901:2048] [node 1] ICP22 created new session: [6:7574436015699943987:2048]
2025-11-19T14:00:12.487178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436015699943987:2048] [node 1] ICS09 handshake done sender: [6:7574436015699943967:2110] self: [6:7574436015699943962:0] peer: [1:7574436014367007197:0] socket: 252 qp: -1
2025-11-19T14:00:12.487186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436015699943987:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:12.487194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:12.487490Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436015699943902:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30261) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436015699943980:2116] held: no
2025-11-19T14:00:12.487803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436015699943985:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54387
2025-11-19T14:00:12.487933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436015459500998:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2648
2025-11-19T14:00:12.487946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436015459500901:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.487970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:12.487976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436015459500901:2048] [node 1] ICP22 created new session: [5:7574436015459500999:2048]
2025-11-19T14:00:12.487982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436015459500999:2048] [node 1] ICS09 handshake done sender: [5:7574436015459500980:2116] self: [5:7574436015459500992:0] peer: [1:7574436014367007171:0] socket: 245 qp: -1
2025-11-19T14:00:12.487988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436015459500999:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:12.487995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:12.489819Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436012684379040:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:12.490048Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436012684379055:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.490063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436012684379030:2080] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.490073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:12.490080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436012684379030:2080] [node 5] ICP22 created new session: [4:7574436012684379061:2080]
2025-11-19T14:00:12.490086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436012684379061:2080] [node 5] ICS09 handshake done sender: [4:7574436012684379040:2116] self: [4:7574436012684379039:0] peer: [5:7574436015459500993:0] socket: 241 qp: -1
2025-11-19T14:00:12.490092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436012684379061:2080] [node 5] ICS10 traffic start
2025-11-19T14:00:12.490099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:12.490386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574436012684379030:2080] [node 5] ICP18 handshake (actor [4:7574436012684379055:2121]) is too old
2025-11-19T14:00:12.491049Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436015459500983:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.492380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436015699943986:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54387
2025-11-19T14:00:12.493354Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436014535222492:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.493368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436014535222492:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.492986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436015699943903:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:12.493248Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436015699943980:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:12.495151Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436015699943970:2112] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:12.494762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436012684379063:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28773
2025-11-19T14:00:12.495344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436015699943902:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.495364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:12.495373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436015699943902:2049] [node 2] ICP22 created new session: [6:7574436015699943993:2049]
2025-11-19T14:00:12.495379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436015699943993:2049] [node 2] ICS09 handshake done sender: [6:7574436015699943980:2116] self: [6:7574436015699943984:0] peer: [2:7574436012882376383:0] socket: 261 qp: -1
2025-11-19T14:00:12.495385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436015699943993:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:12.495394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:12.495662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436015699943960:2062] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.495672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:12.495678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436015699943960:2062] [node 5] ICP22 created new session: [6:7574436015699943995:2062]
2025-11-19T14:00:12.495682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436015699943995:2062] [node 5] ICS09 handshake done sender: [6:7574436015699943970:2112] self: [6:7574436015699943982:0] peer: [5:7574436015459500936:0] socket: 243 qp: -1
2025-11-19T14:00:12.495687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436015699943995:2062] [node 5] ICS10 traffic start
2025-11-19T14:00:12.495693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:12.497769Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436014535222492:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.500048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436015699943992:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54387
2025-11-19T14:00:12.498111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436014367007101:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.498138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:12.498146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436014367007101:2048] [node 2] ICP22 created new session: [1:7574436014367007201:2048]
2025-11-19T14:00:12.498152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436014367007201:2048] [node 2] ICS09 handshake done sender: [1:7574436014367007174:2112] self: [1:7574436014367007173:0] peer: [2:7574436012882376406:0] socket: 220 qp: -1
2025-11-19T14:00:12.498157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436014367007201:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:12.498166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:12.498557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436014367007150:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.498570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:12.498575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436014367007150:2091] [node 5] ICP22 created new session: [1:7574436014367007203:2091]
2025-11-19T14:00:12.498580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436014367007203:2091] [node 5] ICS09 handshake done sender: [1:7574436014367007172:2111] self: [1:7574436014367007171:0] peer: [5:7574436015459500992:0] socket: 244 qp: -1
2025-11-19T14:00:12.498586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436014367007203:2091] [node 5] ICS10 traffic start
2025-11-19T14:00:12.498591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:12.501585Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436015699943981:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:12.501953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436015699943903:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.501971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:12.501978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436015699943903:2050] [node 3] ICP22 created new session: [6:7574436015699943997:2050]
2025-11-19T14:00:12.501984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436015699943997:2050] [node 3] ICS09 handshake done sender: [6:7574436015699943981:2117] self: [6:7574436015699943991:0] peer: [3:7574436014535222491:0] socket: 229 qp: -1
2025-11-19T14:00:12.501990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436015699943997:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:12.501997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:12.503053Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436012684379063:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.503113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436012684379029:2076] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.506457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436015459500956:2097] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.506494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:12.506506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436015459500956:2097] [node 4] ICP22 created new session: [5:7574436015459501001:2097]
2025-11-19T14:00:12.506514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436015459501001:2097] [node 4] ICS09 handshake done sender: [5:7574436015459500983:2118] self: [5:7574436015459500993:0] peer: [4:7574436012684379039:0] socket: 250 qp: -1
2025-11-19T14:00:12.506526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436015459501001:2097] [node 4] ICS10 traffic start
2025-11-19T14:00:12.506542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:12.506773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574436015459500956:2097] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:12.507796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436015459501003:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2648
2025-11-19T14:00:12.507966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436014535222481:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.508003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:12.508010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436014535222481:2079] [node 6] ICP22 created new session: [3:7574436014535222522:2079]
2025-11-19T14:00:12.508016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436014535222522:2079] [node 6] ICS09 handshake done sender: [3:7574436014535222492:2111] self: [3:7574436014535222491:0] peer: [6:7574436015699943991:0] socket: 269 qp: -1
2025-11-19T14:00:12.508024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436014535222522:2079] [node 6] ICS10 traffic start
2025-11-19T14:00:12.508033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.510952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436014367007200:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31778
2025-11-19T14:00:12.510160Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436015699943974:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:12.510176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574436015699943974:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:12.511260Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436014367007187:2119] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:12.511767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436012684379065:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28773
2025-11-19T14:00:12.512029Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436015699943974:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:12.512440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436014367007144:2089] [node 6] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.512462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:12.512470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436014367007144:2089] [node 6] ICP22 created new session: [1:7574436014367007205:2089]
2025-11-19T14:00:12.512476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436014367007205:2089] [node 6] ICS09 handshake done sender: [1:7574436014367007187:2119] self: [1:7574436014367007197:0] peer: [6:7574436015699943962:0] socket: 258 qp: -1
2025-11-19T14:00:12.512484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436014367007205:2089] [node 6] ICS10 traffic start
2025-11-19T14:00:12.512494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:12.513178Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436012684379063:2122] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:12.513267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436012684379029:2076] [node 6] ICP19 incoming handshake succeeded
2025-11-19T14:00:12.513282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:12.513289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436012684379029:2076] [node 6] ICP22 created new session: [4:7574436012684379066:2076]
2025-11-19T14:00:12.513298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436012684379066:2076] [node 6] ICS09 handshake done sender: [4:7574436012684379063:2122] self: [4:7574436012684379064:0] peer: [6:7574436015699943973:0] socket: 278 qp: -1
2025-11-19T14:00:12.513305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436012684379066:2076] [node 6] ICS10 traffic start
2025-11-19T14:00:12.513316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:12.513903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574436015699943959:2059] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:12.513922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:12.513928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436015699943959:2059] [node 4] ICP22 created new session: [6:7574436015699944000:2059]
2025-11-19T14:00:12.513933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436015699944000:2059] [node 4] ICS09 handshake done sender: [6:7574436015699943974:2114] self: [6:7574436015699943973:0] peer: [4:7574436012684379064:0] socket: 232 qp: -1
2025-11-19T14:00:12.513939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436015699944000:2059] [node 4] ICS10 traffic start
2025-11-19T14:00:12.513947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:12.521296Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:12.521394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436012882376412:2048] [node 1] ICS01 socket: 236 reason# EndOfStream
2025-11-19T14:00:12.521404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436012882376309:2048] [node 1] ICP30 unregister session Session# [2:7574436012882376412:2048] VirtualId# [2:7574436012882376406:0]
2025-11-19T14:00:12.521414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436012882376412:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:12.521461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:12.521483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:12.522340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:12.522422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436014535222518:2048] [node 1] ICS01 socket: 221 reason# EndOfStream
2025-11-19T14:00:12.522430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436014535222420:2048] [node 1] ICP30 unregister session Session# [3:7574436014535222518:2048] VirtualId# [3:7574436014535222505:0]
2025-11-19T14:00:12.522437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436014535222518:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:12.522469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:12.522489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:12.535486Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:12.535618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436015699943987:2048] [node 1] ICS01 socket: 252 reason# EndOfStream
2025-11-19T14:00:12.535630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436015699943901:2048] [node 1] ICP30 unregister session Session# [6:7574436015699943987:2048] VirtualId# [6:7574436015699943962:0]
2025-11-19T14:00:12.535641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436015699943987:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:12.535713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:12.535774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:12.535922Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:12.536038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436015459500999:2048] [node 1] ICS01 socket: 245 reason# EndOfStream
2025-11-19T14:00:12.536049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436015459500901:2048] [node 1] ICP30 unregister session Session# [5:7574436015459500999:2048] VirtualId# [5:7574436015459500992:0]
2025-11-19T14:00:12.536057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436015459500999:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:12.536119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:12.536145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:12.535961Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:12.536055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436012684379056:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-11-19T14:00:12.536061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436012684378964:2048] [node 1] ICP30 unregister session Session# [4:7574436012684379056:2048] VirtualId# [4:7574436012684379048:0]
2025-11-19T14:00:12.536067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436012684379056:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:12.536134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:12.536166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:12.544437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:12.544536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436014535222516:2049] [node 2] ICS01 socket: 240 reason# EndOfStream
2025-11-19T14:00:12.544545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436014535222421:2049] [node 2] ICP30 unregister session Session# [3:7574436014535222516:2049] VirtualId# [3:7574436014535222502:0]
2025-11-19T14:00:12.544552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436014535222516:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:12.544595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:12.544618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:12.574962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:12.575105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436012684379058:2049] [node 2] ICS01 socket: 239 reason# EndOfStream
2025-11-19T14:00:12.575118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436012684378965:2049] [node 2] ICP30 unregister session Session# [4:7574436012684379058:2049] VirtualId# [4:7574436012684379049:0]
2025-11-19T14:00:12.575128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436012684379058:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:12.575197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:12.575225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:12.575533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:12.575609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436015699943993:2049] [node 2] ICS01 socket: 261 reason# EndOfStream
2025-11-19T14:00:12.575615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436015699943902:2049] [node 2] ICP30 unregister session Session# [6:7574436015699943993:2049] VirtualId# [6:7574436015699943984:0]
2025-11-19T14:00:12.575620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436015699943993:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:12.575654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:12.575679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:12.575972Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:12.576073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436015459500990:2049] [node 2] ICS01 socket: 262 reason# EndOfStream
2025-11-19T14:00:12.576081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436015459500902:2049] [node 2] ICP30 unregister session Session# [5:7574436015459500990:2049] VirtualId# [5:7574436015459500985:0]
2025-11-19T14:00:12.576089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436015459500990:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:12.576149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:12.576174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:12.611119Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:12.611214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436015459500986:2050] [node 3] ICS01 socket: 215 reason# EndOfStream
2025-11-19T14:00:12.611225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436015459500903:2050] [node 3] ICP30 unregister session Session# [5:7574436015459500986:2050] VirtualId# [5:7574436015459500981:0]
2025-11-19T14:00:12.611232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436015459500986:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:12.611287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:12.611311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:12.616645Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:12.616774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436015699943997:2050] [node 3] ICS01 socket: 229 reason# EndOfStream
2025-11-19T14:00:12.616787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436015699943903:2050] [node 3] ICP30 unregister session Session# [6:7574436015699943997:2050] VirtualId# [6:7574436015699943991:0]
2025-11-19T14:00:12.616798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436015699943997:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:12.616866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:12.616896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:12.627843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:12.627966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436015699944000:2059] [node 4] ICS01 socket: 232 reason# EndOfStream
2025-11-19T14:00:12.627990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436015699943959:2059] [node 4] ICP30 unregister session Session# [6:7574436015699944000:2059] VirtualId# [6:7574436015699943973:0]
2025-11-19T14:00:12.628001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436015699944000:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:12.628067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:12.628105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:12.648018Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T14:00:12.648159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436015699943995:2062] [node 5] ICS01 socket: 243 reason# EndOfStream
2025-11-19T14:00:12.648170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436015699943960:2062] [node 5] ICP30 unregister session Session# [6:7574436015699943995:2062] VirtualId# [6:7574436015699943982:0]
2025-11-19T14:00:12.648178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436015699943995:2062] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:12.648235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T14:00:12.648261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:12.824654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436012635130746:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:12.824668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436012635130747:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:12.914012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436012922223452:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:12.914035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436012922223453:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:13.028333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436018490813500:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:13.028350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436018490813501:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:13.096293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436017491790455:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:13.096318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436017491790456:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:13.096323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436017491790457:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:13.175907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436016715181137:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:13.175927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436016715181138:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:13.175931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436016715181139:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:13.223894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436020194369988:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:13.223911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436020194369989:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:13.223916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436020194369990:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.278950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436017217190807:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24924
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.278160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436020194370035:2091] [node 5] ICP01 ready to work
2025-11-19T14:00:13.280348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436016715181195:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41800
2025-11-19T14:00:13.280480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436016930098103:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:13.283944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436017491790515:2105] [node 6] ICP01 ready to work
2025-11-19T14:00:13.284790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436020194370042:2094] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.289519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436018490813560:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57361
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.289124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436016930098108:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:13.289758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436016930098114:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:13.290213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436018490813561:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57361
2025-11-19T14:00:13.292597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436018490813564:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:13.292705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436018490813565:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.312183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436017217190815:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:13.312501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436017217190817:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:13.312625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436017217190818:2110] [node 5] ICP01 ready to work
2025-11-19T14:00:13.314057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436017217190816:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24924
2025-11-19T14:00:13.315737Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436018490813564:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(22993185ns) {Configure:104(22992680ns) {ProcessConfigured:135(22583778ns) {StartInitialHandshake:157(22583451ns) {DropHandshakes:469(608ns) {DropIncomingHandshake:441(87ns) DropOutgoingHandshake:456(92ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:476(574ns) {SwitchToState:178(448ns)}}}}}
2025-11-19T14:00:13.315534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436020194369990:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(22999891ns) {Configure:104(22997762ns) {ProcessConfigured:135(22590144ns) {StartInitialHandshake:157(22589860ns) {DropHandshakes:469(1332ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(96ns)} GenerateSessionVirtualId:988(313ns) PrepareNewSessionHandshake:476(1424ns) {SwitchToState:178(723ns)}}}}}
2025-11-19T14:00:13.316340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436018490813572:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:13.318786Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436017217190807:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.323245Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436017491790515:2105] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(39212345ns) {Configure:104(39211456ns) {ProcessConfigured:135(38825169ns) {StartInitialHandshake:157(38824216ns) {DropHandshakes:469(601ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(86ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:476(2203ns) {SwitchToState:178(1395ns)}}}}}
2025-11-19T14:00:13.325367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436018490813501:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17358861ns) {Configure:104(17357365ns) {ProcessConfigured:135(16949609ns) {StartInitialHandshake:157(16949138ns) {DropHandshakes:469(1136ns) {DropIncomingHandshake:441(167ns) DropOutgoingHandshake:456(277ns)} GenerateSessionVirtualId:988(310ns) PrepareNewSessionHandshake:476(1149ns) {SwitchToState:178(214ns)}}}}}
2025-11-19T14:00:13.325421Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436018490813560:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.325480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436018490813501:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.328867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436017491790521:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.331964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436016715181139:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(45933934ns) {Configure:104(45932300ns) {ProcessConfigured:135(45554907ns) {StartInitialHandshake:157(45554091ns) {DropHandshakes:469(575ns) {DropIncomingHandshake:441(81ns) DropOutgoingHandshake:456(102ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:476(1727ns) {SwitchToState:178(972ns)}}}}}
2025-11-19T14:00:13.332138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436016715181203:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:13.332457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436016715181204:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.336989Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436017217190809:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.337016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436017217190809:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:13.337076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436017491790525:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43087
2025-11-19T14:00:13.337215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436017491790526:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43087
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.341571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436016930098121:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40291
2025-11-19T14:00:13.341595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436016930098122:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40291
2025-11-19T14:00:13.341640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436016930098123:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40291
2025-11-19T14:00:13.341700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436017217190830:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24924
2025-11-19T14:00:13.343681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436017491790527:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43087
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.348470Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436016930098122:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.348628Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436017217190816:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.348658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436012922223452:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:13.348806Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436016930098111:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.348812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436016930098111:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.349489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436017217190832:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24924
2025-11-19T14:00:13.349595Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436016930098111:2111] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:13.349812Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436017217190816:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.351546Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436017491790525:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.351586Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436017491790526:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.351681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436017491790536:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43087
2025-11-19T14:00:13.351707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436017491790456:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:13.349886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436012922223452:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.349903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:13.349910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436012922223452:2048] [node 1] ICP22 created new session: [2:7574436017217190833:2048]
2025-11-19T14:00:13.349916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436017217190833:2048] [node 1] ICS09 handshake done sender: [2:7574436017217190816:2112] self: [2:7574436017217190831:0] peer: [1:7574436016930098110:0] socket: 236 qp: -1
2025-11-19T14:00:13.349923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436017217190833:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:13.349932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:13.352205Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436017217190809:2107] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.348045Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436016715181195:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.354282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436016715181216:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41800
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.364015Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436017217190820:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.364033Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436017217190820:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:13.363997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020194370066:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-19T14:00:13.364385Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436017491790527:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.364435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436017491790457:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:13.364538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436012922223453:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.364562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:13.364571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436012922223453:2049] [node 3] ICP22 created new session: [2:7574436017217190836:2049]
2025-11-19T14:00:13.364578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436017217190836:2049] [node 3] ICS09 handshake done sender: [2:7574436017217190809:2107] self: [2:7574436017217190808:0] peer: [3:7574436018490813581:0] socket: 226 qp: -1
2025-11-19T14:00:13.364586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436017217190836:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:13.364596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:13.364599Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436018490813571:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.364612Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436018490813571:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:13.368052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436016715181217:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41800
2025-11-19T14:00:13.368215Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436016715181216:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.368250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436016715181137:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.371899Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436016930098121:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.372487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436017491790540:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43087
2025-11-19T14:00:13.372613Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436020194370066:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.372653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436020194370042:2094] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:13.373379Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436017217190820:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:13.374393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436017217190815:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.374414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:13.374420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436017217190815:2108] [node 4] ICP22 created new session: [2:7574436017217190841:2108]
2025-11-19T14:00:13.374426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436017217190841:2108] [node 4] ICS09 handshake done sender: [2:7574436017217190820:2113] self: [2:7574436017217190819:0] peer: [4:7574436017491790538:0] socket: 237 qp: -1
2025-11-19T14:00:13.374432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436017217190841:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:13.374439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:13.369861Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436018490813561:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.369938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436018490813582:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57361
2025-11-19T14:00:13.369950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436018490813583:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57361
2025-11-19T14:00:13.372982Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436017491790524:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.372993Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436017491790524:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:13.374411Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436017491790526:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:13.374510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436017491790456:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.374524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:13.374532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436017491790456:2049] [node 2] ICP22 created new session: [4:7574436017491790542:2049]
2025-11-19T14:00:13.374538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436017491790542:2049] [node 2] ICS09 handshake done sender: [4:7574436017491790526:2113] self: [4:7574436017491790538:0] peer: [2:7574436017217190819:0] socket: 243 qp: -1
2025-11-19T14:00:13.374546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436017491790542:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:13.374552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:13.375433Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436017491790524:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:13.375543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436017491790515:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.375562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:13.375569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436017491790515:2105] [node 6] ICP22 created new session: [4:7574436017491790545:2105]
2025-11-19T14:00:13.375574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436017491790545:2105] [node 6] ICS09 handshake done sender: [4:7574436017491790524:2111] self: [4:7574436017491790516:0] peer: [6:7574436020194370067:0] socket: 257 qp: -1
2025-11-19T14:00:13.375580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436017491790545:2105] [node 6] ICS10 traffic start
2025-11-19T14:00:13.375587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.376411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436016715181220:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41800
2025-11-19T14:00:13.376443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436016715181221:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41800
2025-11-19T14:00:13.379960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436012635130746:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.380004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:13.380014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436012635130746:2048] [node 2] ICP22 created new session: [1:7574436016930098127:2048]
2025-11-19T14:00:13.380020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436016930098127:2048] [node 2] ICS09 handshake done sender: [1:7574436016930098111:2111] self: [1:7574436016930098110:0] peer: [2:7574436017217190831:0] socket: 230 qp: -1
2025-11-19T14:00:13.380037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436016930098127:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:13.380045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:13.380070Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436016930098113:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.380078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436016930098113:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.383097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436018490813500:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:13.382841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020194370068:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-19T14:00:13.382877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020194370069:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-19T14:00:13.382940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020194370070:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.384429Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436020194370066:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:13.384500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436020194370042:2094] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.384510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:13.384515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436020194370042:2094] [node 4] ICP22 created new session: [6:7574436020194370072:2094]
2025-11-19T14:00:13.384520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436020194370072:2094] [node 4] ICS09 handshake done sender: [6:7574436020194370066:2115] self: [6:7574436020194370067:0] peer: [4:7574436017491790516:0] socket: 259 qp: -1
2025-11-19T14:00:13.384525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436020194370072:2094] [node 4] ICS10 traffic start
2025-11-19T14:00:13.384532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:13.384844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020194370071:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-19T14:00:13.387585Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436020194370071:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.387618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436020194369990:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:13.383272Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436018490813560:2107] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:13.383502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436018490813501:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.383515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:13.383520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436018490813501:2049] [node 2] ICP22 created new session: [3:7574436018490813587:2049]
2025-11-19T14:00:13.383526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436018490813587:2049] [node 2] ICS09 handshake done sender: [3:7574436018490813560:2107] self: [3:7574436018490813581:0] peer: [2:7574436017217190808:0] socket: 227 qp: -1
2025-11-19T14:00:13.383531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436018490813587:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:13.383537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:13.385716Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436018490813571:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:13.385800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436018490813565:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.385810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:13.385816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436018490813565:2110] [node 4] ICP22 created new session: [3:7574436018490813590:2110]
2025-11-19T14:00:13.385821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436018490813590:2110] [node 4] ICS09 handshake done sender: [3:7574436018490813571:2113] self: [3:7574436018490813570:0] peer: [4:7574436017491790539:0] socket: 240 qp: -1
2025-11-19T14:00:13.385826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436018490813590:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:13.385832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:13.387727Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436018490813576:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.387732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436018490813576:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:13.388205Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436016715181217:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.388276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436016715181139:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.388490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020194370076:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.388909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436017491790547:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43087
2025-11-19T14:00:13.388589Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436018490813576:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:13.388906Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436020194370071:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:13.389216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436020194369990:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.389231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:13.389236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436020194369990:2050] [node 3] ICP22 created new session: [6:7574436020194370077:2050]
2025-11-19T14:00:13.389251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436020194370077:2050] [node 3] ICS09 handshake done sender: [6:7574436020194370071:2119] self: [6:7574436020194370075:0] peer: [3:7574436018490813575:0] socket: 275 qp: -1
2025-11-19T14:00:13.389257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436020194370077:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:13.389263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:13.389850Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436016715181221:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.389887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436016715181203:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:13.389693Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436018490813582:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.389726Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436018490813569:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.389729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436018490813569:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:13.390374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436018490813572:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.390382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:13.390388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436018490813572:2111] [node 6] ICP22 created new session: [3:7574436018490813593:2111]
2025-11-19T14:00:13.390393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436018490813593:2111] [node 6] ICS09 handshake done sender: [3:7574436018490813576:2115] self: [3:7574436018490813575:0] peer: [6:7574436020194370075:0] socket: 274 qp: -1
2025-11-19T14:00:13.390398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436018490813593:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:13.390405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:13.394989Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436017217190830:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.395877Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436016715181220:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.395919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436016715181138:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:13.395856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436018490813584:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57361
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.396565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436016715181225:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41800
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.396567Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436012635130746:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(44149301ns) {HandleHandshakeStatus:329(44148681ns) {DropIncomingHandshake:441(9325ns) DropOutgoingHandshake:456(421ns) SwitchToState:178(189ns) ProcessPendingSessionEvents:495(143603ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 103091ns max# 34341ns}}}
2025-11-19T14:00:13.396618Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436016930098123:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.398319Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436016930098107:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.398324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436016930098107:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:13.400358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436016930098129:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40291
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.400520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436016715181226:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41800
2025-11-19T14:00:13.400672Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436016930098113:2112] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:13.400748Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436017491790527:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:13.400337Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436020194370069:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.400374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436020194369989:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:13.400994Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436016715181216:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:13.401078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436016715181137:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.401097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:13.401104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436016715181137:2048] [node 1] ICP22 created new session: [5:7574436016715181227:2048]
2025-11-19T14:00:13.401110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436016715181227:2048] [node 1] ICS09 handshake done sender: [5:7574436016715181216:2116] self: [5:7574436016715181219:0] peer: [1:7574436016930098112:0] socket: 232 qp: -1
2025-11-19T14:00:13.401116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436016715181227:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:13.401124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:13.401694Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436018490813569:2112] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:13.401296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436017217190824:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.401310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436017217190824:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:13.401864Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436018490813584:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.401876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436018490813564:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.401889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:13.401894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436018490813564:2109] [node 5] ICP22 created new session: [3:7574436018490813598:2109]
2025-11-19T14:00:13.401899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436018490813598:2109] [node 5] ICS09 handshake done sender: [3:7574436018490813569:2112] self: [3:7574436018490813566:0] peer: [5:7574436016715181222:0] socket: 239 qp: -1
2025-11-19T14:00:13.401904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436018490813598:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:13.401909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:13.402401Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436016715181217:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:13.402533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436016715181139:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.402545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:13.402550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436016715181139:2050] [node 3] ICP22 created new session: [5:7574436016715181230:2050]
2025-11-19T14:00:13.402565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436016715181230:2050] [node 3] ICS09 handshake done sender: [5:7574436016715181217:2117] self: [5:7574436016715181222:0] peer: [3:7574436018490813566:0] socket: 252 qp: -1
2025-11-19T14:00:13.402569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436016715181230:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:13.402574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:13.402210Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436017491790536:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.402224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436017491790457:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.402236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:13.402241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436017491790457:2050] [node 3] ICP22 created new session: [4:7574436017491790548:2050]
2025-11-19T14:00:13.402247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436017491790548:2050] [node 3] ICS09 handshake done sender: [4:7574436017491790527:2114] self: [4:7574436017491790539:0] peer: [3:7574436018490813570:0] socket: 244 qp: -1
2025-11-19T14:00:13.402252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436017491790548:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:13.402260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:13.403366Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436017217190826:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.403378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436017217190826:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:13.403446Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436017491790535:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.403455Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436017491790535:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:13.405150Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436017217190824:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:13.404032Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436016930098107:2109] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:13.405318Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436016930098129:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.405383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436012635130747:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.405396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:13.405401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436012635130747:2049] [node 3] ICP22 created new session: [1:7574436016930098134:2049]
2025-11-19T14:00:13.405407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436016930098134:2049] [node 3] ICS09 handshake done sender: [1:7574436016930098107:2109] self: [1:7574436016930098104:0] peer: [3:7574436018490813585:0] socket: 228 qp: -1
2025-11-19T14:00:13.405411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436016930098134:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:13.405417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:13.405586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436016930098108:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.405591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:13.405594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436016930098108:2108] [node 5] ICP22 created new session: [1:7574436016930098136:2108]
2025-11-19T14:00:13.405598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436016930098136:2108] [node 5] ICS09 handshake done sender: [1:7574436016930098113:2112] self: [1:7574436016930098112:0] peer: [5:7574436016715181219:0] socket: 231 qp: -1
2025-11-19T14:00:13.406033Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436020194370068:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.405601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436016930098136:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:13.405606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:13.406070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436020194369988:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:13.406322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436017217190817:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.406341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:13.406347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436017217190817:2109] [node 6] ICP22 created new session: [2:7574436017217190845:2109]
2025-11-19T14:00:13.406351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436017217190845:2109] [node 6] ICS09 handshake done sender: [2:7574436017217190824:2115] self: [2:7574436017217190823:0] peer: [6:7574436020194370079:0] socket: 264 qp: -1
2025-11-19T14:00:13.406357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436017217190845:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:13.406366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:13.408130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436020194369989:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436020194370069:2117] held: no
2025-11-19T14:00:13.408572Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436016930098117:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.408582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436016930098117:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.409502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436017491790551:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43087
2025-11-19T14:00:13.409603Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436016930098117:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:13.409688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436016930098114:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.409703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:13.409708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436016930098114:2110] [node 6] ICP22 created new session: [1:7574436016930098141:2110]
2025-11-19T14:00:13.409713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436016930098141:2110] [node 6] ICS09 handshake done sender: [1:7574436016930098117:2113] self: [1:7574436016930098116:0] peer: [6:7574436020194370080:0] socket: 260 qp: -1
2025-11-19T14:00:13.409718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436016930098141:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:13.409724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:13.416167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436018490813572:2111] [node 6] ICP35 event processing took too much time PendingConnection:278(25644184ns) {HandleHandshakeStatus:329(25643556ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(141ns) SwitchToState:178(66ns) ProcessPendingSessionEvents:495(25578046ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25551220ns max# 25462006ns}}}
2025-11-19T14:00:13.416659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574436018490813565:2110] [node 4] ICP18 handshake (actor [3:7574436018490813584:2119]) is too old
2025-11-19T14:00:13.419458Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436017491790535:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:13.419635Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436017217190826:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:13.419704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436017491790521:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.419727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:13.419735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436017491790521:2108] [node 5] ICP22 created new session: [4:7574436017491790553:2108]
2025-11-19T14:00:13.419741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436017491790553:2108] [node 5] ICS09 handshake done sender: [4:7574436017491790535:2117] self: [4:7574436017491790534:0] peer: [5:7574436016715181223:0] socket: 268 qp: -1
2025-11-19T14:00:13.419848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436017491790553:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:13.419859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:13.420001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436017217190818:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.420020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:13.420025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436017217190818:2110] [node 5] ICP22 created new session: [2:7574436017217190848:2110]
2025-11-19T14:00:13.420030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436017217190848:2110] [node 5] ICS09 handshake done sender: [2:7574436017217190826:2116] self: [2:7574436017217190825:0] peer: [5:7574436016715181224:0] socket: 265 qp: -1
2025-11-19T14:00:13.420036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436017217190848:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:13.420043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.423799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436016715181232:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41800
2025-11-19T14:00:13.424509Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436017491790551:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.424547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436017491790455:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:13.424685Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436016930098120:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.424696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436016930098120:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.425697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436017491790556:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43087
2025-11-19T14:00:13.425842Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436020194369988:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25501) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436020194370068:2116] held: no
2025-11-19T14:00:13.426316Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436016930098120:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:13.426488Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436017491790551:2121] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:13.426548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436017491790455:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.426565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:13.426571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436017491790455:2048] [node 1] ICP22 created new session: [4:7574436017491790557:2048]
2025-11-19T14:00:13.426578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436017491790557:2048] [node 1] ICS09 handshake done sender: [4:7574436017491790551:2121] self: [4:7574436017491790555:0] peer: [1:7574436016930098106:0] socket: 245 qp: -1
2025-11-19T14:00:13.426583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436017491790557:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:13.426591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:13.422279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436018490813597:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57361
2025-11-19T14:00:13.427510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436016930098103:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.427528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:13.427533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436016930098103:2107] [node 4] ICP22 created new session: [1:7574436016930098144:2107]
2025-11-19T14:00:13.427538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436016930098144:2107] [node 4] ICS09 handshake done sender: [1:7574436016930098120:2114] self: [1:7574436016930098106:0] peer: [4:7574436017491790555:0] socket: 222 qp: -1
2025-11-19T14:00:13.427543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436016930098144:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:13.427549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.428930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436016715181233:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41800
2025-11-19T14:00:13.429163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020194370081:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.430822Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436018490813500:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25501) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436018490813561:2108] held: no
2025-11-19T14:00:13.431359Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436018490813561:2108] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:13.431445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436018490813500:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.431460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:13.431466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436018490813500:2048] [node 1] ICP22 created new session: [3:7574436018490813600:2048]
2025-11-19T14:00:13.431472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436018490813600:2048] [node 1] ICS09 handshake done sender: [3:7574436018490813561:2108] self: [3:7574436018490813585:0] peer: [1:7574436016930098104:0] socket: 229 qp: -1
2025-11-19T14:00:13.431479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436018490813600:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:13.431486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:13.429925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020194370083:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-19T14:00:13.430079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020194370082:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-19T14:00:13.430143Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436020194370069:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:13.436009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436016715181138:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436016715181220:2118] held: no
2025-11-19T14:00:13.436136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436016715181203:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20392) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436016715181221:2119] held: no
2025-11-19T14:00:13.436299Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436016715181220:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:13.436532Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436016715181221:2119] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:13.436703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436016715181203:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.436719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:13.436726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436016715181203:2107] [node 4] ICP22 created new session: [5:7574436016715181234:2107]
2025-11-19T14:00:13.436731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436016715181234:2107] [node 4] ICS09 handshake done sender: [5:7574436016715181221:2119] self: [5:7574436016715181223:0] peer: [4:7574436017491790534:0] socket: 270 qp: -1
2025-11-19T14:00:13.436738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436016715181234:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:13.436747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:13.437125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436016715181138:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.437138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:13.437144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436016715181138:2049] [node 2] ICP22 created new session: [5:7574436016715181236:2049]
2025-11-19T14:00:13.437150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436016715181236:2049] [node 2] ICS09 handshake done sender: [5:7574436016715181220:2118] self: [5:7574436016715181224:0] peer: [2:7574436017217190825:0] socket: 269 qp: -1
2025-11-19T14:00:13.437155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436016715181236:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:13.437162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:13.437392Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436020194370082:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.437421Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436020194370068:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:13.437497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436020194369989:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.437508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:13.437515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436020194369989:2049] [node 2] ICP22 created new session: [6:7574436020194370084:2049]
2025-11-19T14:00:13.437520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436020194370084:2049] [node 2] ICS09 handshake done sender: [6:7574436020194370069:2117] self: [6:7574436020194370079:0] peer: [2:7574436017217190823:0] socket: 271 qp: -1
2025-11-19T14:00:13.437526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436020194370084:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:13.437532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:13.437697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436020194370035:2091] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:13.438433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436020194369988:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.438447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:13.438453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436020194369988:2048] [node 1] ICP22 created new session: [6:7574436020194370087:2048]
2025-11-19T14:00:13.438458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436020194370087:2048] [node 1] ICS09 handshake done sender: [6:7574436020194370068:2116] self: [6:7574436020194370080:0] peer: [1:7574436016930098116:0] socket: 261 qp: -1
2025-11-19T14:00:13.438464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436020194370087:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:13.438469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:13.438968Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436016715181212:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:13.438977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436016715181212:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:13.441025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020194370089:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54594
2025-11-19T14:00:13.442716Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436016715181212:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:13.442969Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436020194370082:2122] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:13.443130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436016715181204:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:13.443149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:13.443155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436016715181204:2110] [node 6] ICP22 created new session: [5:7574436016715181239:2110]
2025-11-19T14:00:13.443161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436016715181239:2110] [node 6] ICS09 handshake done sender: [5:7574436016715181212:2115] self: [5:7574436016715181211:0] peer: [6:7574436020194370086:0] socket: 285 qp: -1
2025-11-19T14:00:13.443166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436016715181239:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:13.443175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:13.443621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436020194370035:2091] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:13.443634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:13.443640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436020194370035:2091] [node 5] ICP22 created new session: [6:7574436020194370090:2091]
2025-11-19T14:00:13.443645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436020194370090:2091] [node 5] ICS09 handshake done sender: [6:7574436020194370082:2122] self: [6:7574436020194370086:0] peer: [5:7574436016715181211:0] socket: 233 qp: -1
2025-11-19T14:00:13.443650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436020194370090:2091] [node 5] ICS10 traffic start
2025-11-19T14:00:13.443657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:13.457843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:13.457935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436017217190833:2048] [node 1] ICS01 socket: 236 reason# EndOfStream
2025-11-19T14:00:13.457944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436012922223452:2048] [node 1] ICP30 unregister session Session# [2:7574436017217190833:2048] VirtualId# [2:7574436017217190831:0]
2025-11-19T14:00:13.457953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436017217190833:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.457987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:13.458007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:13.458497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:13.458566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436018490813600:2048] [node 1] ICS01 socket: 229 reason# EndOfStream
2025-11-19T14:00:13.458572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436018490813500:2048] [node 1] ICP30 unregister session Session# [3:7574436018490813600:2048] VirtualId# [3:7574436018490813585:0]
2025-11-19T14:00:13.458576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436018490813600:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.458606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:13.458622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:13.462713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:13.462821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436017491790557:2048] [node 1] ICS01 socket: 245 reason# EndOfStream
2025-11-19T14:00:13.462830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436017491790455:2048] [node 1] ICP30 unregister session Session# [4:7574436017491790557:2048] VirtualId# [4:7574436017491790555:0]
2025-11-19T14:00:13.462840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436017491790557:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.462892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:13.462919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:13.462959Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:13.463024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436016715181227:2048] [node 1] ICS01 socket: 232 reason# EndOfStream
2025-11-19T14:00:13.463029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436016715181137:2048] [node 1] ICP30 unregister session Session# [5:7574436016715181227:2048] VirtualId# [5:7574436016715181219:0]
2025-11-19T14:00:13.463034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436016715181227:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.463062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:13.463081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:13.463111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:13.463175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436020194370087:2048] [node 1] ICS01 socket: 261 reason# EndOfStream
2025-11-19T14:00:13.463180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436020194369988:2048] [node 1] ICP30 unregister session Session# [6:7574436020194370087:2048] VirtualId# [6:7574436020194370080:0]
2025-11-19T14:00:13.463184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436020194370087:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.463215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:13.463235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:13.475872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:13.475985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436018490813587:2049] [node 2] ICS01 socket: 227 reason# EndOfStream
2025-11-19T14:00:13.475996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436018490813501:2049] [node 2] ICP30 unregister session Session# [3:7574436018490813587:2049] VirtualId# [3:7574436018490813581:0]
2025-11-19T14:00:13.476003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436018490813587:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.476076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:13.476104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:13.481938Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:13.482034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436017491790542:2049] [node 2] ICS07 socket disconnect 243 reason# EndOfStream
2025-11-19T14:00:13.482040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436017491790542:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.482079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:13.482102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436017491790542:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:13.482478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:13.482541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436016715181236:2049] [node 2] ICS01 socket: 269 reason# EndOfStream
2025-11-19T14:00:13.482546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436016715181138:2049] [node 2] ICP30 unregister session Session# [5:7574436016715181236:2049] VirtualId# [5:7574436016715181224:0]
2025-11-19T14:00:13.482551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436016715181236:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.482571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:13.482591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:13.482315Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:13.482384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436020194370084:2049] [node 2] ICS01 socket: 271 reason# EndOfStream
2025-11-19T14:00:13.482389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436020194369989:2049] [node 2] ICP30 unregister session Session# [6:7574436020194370084:2049] VirtualId# [6:7574436020194370079:0]
2025-11-19T14:00:13.482394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436020194370084:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.482421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:13.482444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:13.484163Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436017491790456:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23649) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:13.484179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436017491790542:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:13.484184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436017491790456:2049] [node 2] ICP30 unregister session Session# [4:7574436017491790542:2049] VirtualId# [4:7574436017491790538:0]
2025-11-19T14:00:13.484226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:13.484231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436017491790456:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23649) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:13.484235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23649) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:13.496572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574436017491790456:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:13.512527Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T14:00:13.512623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436017491790548:2050] [node 3] ICS01 socket: 244 reason# EndOfStream
2025-11-19T14:00:13.512634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436017491790457:2050] [node 3] ICP30 unregister session Session# [4:7574436017491790548:2050] VirtualId# [4:7574436017491790539:0]
2025-11-19T14:00:13.512643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436017491790548:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.512696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:13.512721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:13.512863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:13.512932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436020194370077:2050] [node 3] ICS01 socket: 275 reason# EndOfStream
2025-11-19T14:00:13.512937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436020194369990:2050] [node 3] ICP30 unregister session Session# [6:7574436020194370077:2050] VirtualId# [6:7574436020194370075:0]
2025-11-19T14:00:13.512944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436020194370077:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.512975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:13.512995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:13.515203Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:13.515277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436016715181230:2050] [node 3] ICS01 socket: 252 reason# EndOfStream
2025-11-19T14:00:13.515283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436016715181139:2050] [node 3] ICP30 unregister session Session# [5:7574436016715181230:2050] VirtualId# [5:7574436016715181222:0]
2025-11-19T14:00:13.515288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436016715181230:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.515345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:13.515366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:13.540137Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:13.540245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436020194370072:2094] [node 4] ICS01 socket: 259 reason# EndOfStream
2025-11-19T14:00:13.540254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436020194370042:2094] [node 4] ICP30 unregister session Session# [6:7574436020194370072:2094] VirtualId# [6:7574436020194370067:0]
2025-11-19T14:00:13.540261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436020194370072:2094] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.540324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:13.540349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:13.579137Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T14:00:13.579242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436020194370090:2091] [node 5] ICS01 socket: 233 reason# EndOfStream
2025-11-19T14:00:13.579253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436020194370035:2091] [node 5] ICP30 unregister session Session# [6:7574436020194370090:2091] VirtualId# [6:7574436020194370086:0]
2025-11-19T14:00:13.579260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436020194370090:2091] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:13.579306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T14:00:13.579329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:13.991892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436016762359012:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:13.991914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436016762359013:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:14.022416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436023795345128:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:14.022428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436023795345129:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:14.007920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436024554219572:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:14.007935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436024554219573:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:14.077560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436023586170497:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:14.077576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436023586170498:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:14.077580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436023586170499:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:14.104183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436023324537059:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:14.104195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436023324537060:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:14.104199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436023324537061:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:14.131949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436020704939358:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:14.131984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436020704939359:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:14.131990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436020704939360:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:14.184264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436023586170527:2075] [node 5] ICP01 ready to work
2025-11-19T14:00:14.184463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436023324537077:2062] [node 4] ICP01 ready to work
2025-11-19T14:00:14.184436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436021057326370:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:14.184571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436021057326371:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.188253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436023795345188:2106] [node 5] ICP01 ready to work
2025-11-19T14:00:14.188348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436023324537096:2073] [node 6] ICP01 ready to work
2025-11-19T14:00:14.189156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436023324537102:2081] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10861
2025-11-19T14:00:14.205349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436023795345193:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:14.212323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436023795345198:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:14.208376Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436021057326371:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23367797ns) {Configure:104(23362873ns) {ProcessConfigured:135(22983715ns) {StartInitialHandshake:157(22983327ns) {DropHandshakes:469(1301ns) {DropIncomingHandshake:441(187ns) DropOutgoingHandshake:456(150ns)} GenerateSessionVirtualId:988(610ns) PrepareNewSessionHandshake:476(844ns) {SwitchToState:178(365ns)}}}}}
2025-11-19T14:00:14.208549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436021057326377:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.225478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436016762359013:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(40059812ns) {Configure:104(40055881ns) {ProcessConfigured:135(39807798ns) {StartInitialHandshake:157(39807547ns) {DropHandshakes:469(195ns) {DropIncomingHandshake:441(40ns) DropOutgoingHandshake:456(32ns)} GenerateSessionVirtualId:988(355ns) PrepareNewSessionHandshake:476(1265ns) {SwitchToState:178(802ns)}}}}}
2025-11-19T14:00:14.223869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436023324537059:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(39311527ns) {Configure:104(39309642ns) {ProcessConfigured:135(38355971ns) {StartInitialHandshake:157(38354779ns) {DropHandshakes:469(248ns) {DropIncomingHandshake:441(23ns) DropOutgoingHandshake:456(23ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:476(1865ns) {SwitchToState:178(1137ns)}}}}}
2025-11-19T14:00:14.225234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436023324537126:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10861
2025-11-19T14:00:14.228106Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436024554219572:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(43552415ns) {Configure:104(43549583ns) {ProcessConfigured:135(42177873ns) {StartInitialHandshake:157(42176577ns) {DropHandshakes:469(742ns) {DropIncomingHandshake:441(152ns) DropOutgoingHandshake:456(92ns)} GenerateSessionVirtualId:988(493ns) PrepareNewSessionHandshake:476(2229ns) {SwitchToState:178(1377ns)}}}}}
2025-11-19T14:00:14.228331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436024554219638:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:14.228498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436024554219639:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:14.228633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436024554219640:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:14.230068Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436024554219573:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(45362190ns) {Configure:104(45351407ns) {ProcessConfigured:135(44032693ns) {StartInitialHandshake:157(44031436ns) {DropHandshakes:469(327ns) {DropIncomingHandshake:441(13ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(703ns) {SwitchToState:178(194ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.237357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436024554219651:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25174
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.237459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436024554219652:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25174
2025-11-19T14:00:14.238193Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436023324537102:2081] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.241729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436024554219651:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.240000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436023586170499:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(55811136ns) {Configure:104(55809363ns) {ProcessConfigured:135(45797378ns) {StartInitialHandshake:157(45796309ns) {DropHandshakes:469(1275ns) {DropIncomingHandshake:441(135ns) DropOutgoingHandshake:456(285ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:476(1880ns) {SwitchToState:178(959ns)}}}}}
2025-11-19T14:00:14.240198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436023586170565:2079] [node 6] ICP01 ready to work
2025-11-19T14:00:14.241204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436023586170566:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5900
2025-11-19T14:00:14.241284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436023586170567:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5900
2025-11-19T14:00:14.241623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436023586170498:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(56748860ns) {Configure:104(56748262ns) {ProcessConfigured:135(47291310ns) {StartInitialHandshake:157(47290929ns) {DropHandshakes:469(379ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(124ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(828ns) {SwitchToState:178(624ns)}}}}}
2025-11-19T14:00:14.241805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436023586170527:2075] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(57484696ns) {Configure:104(57484190ns) {ProcessConfigured:135(47408056ns) {StartInitialHandshake:157(47407513ns) {DropHandshakes:469(462ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(119ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:476(637ns) {SwitchToState:178(288ns)}}}}}
2025-11-19T14:00:14.241846Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436023586170566:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.243118Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436023795345129:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(30717770ns) {Configure:104(30717181ns) {ProcessConfigured:135(30386343ns) {StartInitialHandshake:157(30385740ns) {DropHandshakes:469(1093ns) {DropIncomingHandshake:441(492ns) DropOutgoingHandshake:456(155ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:476(1163ns) {SwitchToState:178(449ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.243293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436023795345204:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30785
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.243606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436024554219654:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25174
2025-11-19T14:00:14.245106Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436024554219654:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.245654Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436024554219652:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.245673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436024554219572:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.246739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436024554219657:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25174
2025-11-19T14:00:14.247044Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436024554219652:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:14.245776Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436021057326381:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.245789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436021057326381:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:14.246849Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436021057326381:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:14.248235Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436023324537061:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(23187128ns) {Configure:104(23185873ns) {ProcessConfigured:135(22627359ns) {StartInitialHandshake:157(22626352ns) {DropHandshakes:469(372ns) {DropIncomingHandshake:441(14ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:476(1988ns) {SwitchToState:178(1017ns)}}}}}
2025-11-19T14:00:14.248349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436023324537061:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:14.247471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436020704939399:2067] [node 4] ICP01 ready to work
2025-11-19T14:00:14.247992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436020704939401:2073] [node 5] ICP01 ready to work
2025-11-19T14:00:14.247879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436024554219572:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.247934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:14.247947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436024554219572:2048] [node 1] ICP22 created new session: [2:7574436024554219658:2048]
2025-11-19T14:00:14.247956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436024554219658:2048] [node 1] ICS09 handshake done sender: [2:7574436024554219652:2116] self: [2:7574436024554219656:0] peer: [1:7574436021057326380:0] socket: 243 qp: -1
2025-11-19T14:00:14.247969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436024554219658:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:14.248011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:14.255181Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436020704939358:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(70746083ns) {Configure:104(70744370ns) {ProcessConfigured:135(70180186ns) {StartInitialHandshake:157(70179220ns) {DropHandshakes:469(865ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:476(2110ns) {SwitchToState:178(1613ns)}}}}}
2025-11-19T14:00:14.255709Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436023795345198:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(23049483ns) {Configure:104(23045635ns) {ProcessConfigured:135(22543436ns) {StartInitialHandshake:157(22542868ns) {DropHandshakes:469(947ns) {DropIncomingHandshake:441(114ns) DropOutgoingHandshake:456(141ns)} GenerateSessionVirtualId:988(448ns) PrepareNewSessionHandshake:476(1016ns) {SwitchToState:178(622ns)}}}}}
2025-11-19T14:00:14.264275Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436023795345128:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(28088463ns) {Configure:104(28087550ns) {ProcessConfigured:135(27755338ns) {StartInitialHandshake:157(27754584ns) {DropHandshakes:469(665ns) {DropIncomingHandshake:441(100ns) DropOutgoingHandshake:456(110ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:476(1988ns) {SwitchToState:178(1331ns)}}}}}
2025-11-19T14:00:14.264951Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436023795345190:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.264960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436023795345190:2107] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.268819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436020704939360:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21239654ns) {Configure:104(21237972ns) {ProcessConfigured:135(20837366ns) {StartInitialHandshake:157(20836566ns) {DropHandshakes:469(660ns) {DropIncomingHandshake:441(128ns) DropOutgoingHandshake:456(123ns)} GenerateSessionVirtualId:988(368ns) PrepareNewSessionHandshake:476(1125ns) {SwitchToState:178(966ns)}}}}}
2025-11-19T14:00:14.269110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020704939436:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34425
2025-11-19T14:00:14.269158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020704939437:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34425
2025-11-19T14:00:14.269191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020704939438:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34425
2025-11-19T14:00:14.269220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020704939439:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34425
2025-11-19T14:00:14.269230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020704939440:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34425
2025-11-19T14:00:14.264651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436016762359012:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.264678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:14.264688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436016762359012:2048] [node 2] ICP22 created new session: [1:7574436021057326387:2048]
2025-11-19T14:00:14.264696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436021057326387:2048] [node 2] ICS09 handshake done sender: [1:7574436021057326381:2112] self: [1:7574436021057326380:0] peer: [2:7574436024554219656:0] socket: 237 qp: -1
2025-11-19T14:00:14.264706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436021057326387:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:14.264715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:14.268350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436021057326370:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(83670162ns) {Configure:104(83669468ns) {ProcessConfigured:135(82172451ns) {StartInitialHandshake:157(82171766ns) {DropHandshakes:469(361ns) {DropIncomingHandshake:441(22ns) DropOutgoingHandshake:456(41ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:476(2119ns) {SwitchToState:178(986ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.269349Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436023324537126:2104] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.269400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436023324537059:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:14.270098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436020704939437:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.270122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436020704939360:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:14.269525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436023795345208:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30785
2025-11-19T14:00:14.270311Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436023795345195:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.270314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436023795345195:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.271540Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436020704939436:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.271637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436020704939401:2073] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:14.271657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020704939442:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34425
2025-11-19T14:00:14.269623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436023586170578:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5900
2025-11-19T14:00:14.272054Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436023795345195:2109] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:14.271805Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436023324537100:2080] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.271817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436023324537100:2080] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:14.272661Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436020704939437:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:14.272816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436023795345193:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.272836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:14.272843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436023795345193:2108] [node 6] ICP22 created new session: [3:7574436023795345210:2108]
2025-11-19T14:00:14.272849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436023795345210:2108] [node 6] ICS09 handshake done sender: [3:7574436023795345195:2109] self: [3:7574436023795345194:0] peer: [6:7574436020704939441:0] socket: 234 qp: -1
2025-11-19T14:00:14.272856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436023795345210:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:14.272864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:14.272969Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436023324537100:2080] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:14.273086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436023324537096:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.273096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:14.273103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436023324537096:2073] [node 6] ICP22 created new session: [5:7574436023324537142:2073]
2025-11-19T14:00:14.273109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436023324537142:2073] [node 6] ICS09 handshake done sender: [5:7574436023324537100:2080] self: [5:7574436023324537099:0] peer: [6:7574436020704939443:0] socket: 232 qp: -1
2025-11-19T14:00:14.273114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436023324537142:2073] [node 6] ICS10 traffic start
2025-11-19T14:00:14.273120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:14.279853Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436021057326379:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.279884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436021057326379:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.280804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436023795345212:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30785
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.284000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436023324537145:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10861
2025-11-19T14:00:14.284112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436023586170579:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5900
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.289914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436024554219660:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25174
2025-11-19T14:00:14.290317Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436024554219572:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(42346786ns) {HandleHandshakeStatus:329(42344763ns) {DropIncomingHandshake:441(426ns) DropOutgoingHandshake:456(4056ns) SwitchToState:178(334ns) ProcessPendingSessionEvents:495(22341473ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22319886ns max# 22237964ns}}}
2025-11-19T14:00:14.291222Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436023324537102:2081] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:14.289588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436021057326391:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13891
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.290643Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436023795345190:2107] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:14.291421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436023795345188:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.291450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:14.291456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436023795345188:2106] [node 5] ICP22 created new session: [3:7574436023795345215:2106]
2025-11-19T14:00:14.291462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436023795345215:2106] [node 5] ICS09 handshake done sender: [3:7574436023795345190:2107] self: [3:7574436023795345189:0] peer: [5:7574436023324537139:0] socket: 228 qp: -1
2025-11-19T14:00:14.291469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436023795345215:2106] [node 5] ICS10 traffic start
2025-11-19T14:00:14.291478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:14.291582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436023324537061:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.291583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436023324537146:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10861
2025-11-19T14:00:14.291597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:14.291605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436023324537061:2050] [node 3] ICP22 created new session: [5:7574436023324537147:2050]
2025-11-19T14:00:14.291610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436023324537147:2050] [node 3] ICS09 handshake done sender: [5:7574436023324537102:2081] self: [5:7574436023324537139:0] peer: [3:7574436023795345189:0] socket: 230 qp: -1
2025-11-19T14:00:14.291614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436023324537147:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:14.291627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.296031Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436016762359012:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(31246508ns) {HandleHandshakeStatus:329(31245268ns) {DropIncomingHandshake:441(151ns) DropOutgoingHandshake:456(334ns) SwitchToState:178(52ns) ProcessPendingSessionEvents:495(19781996ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19763951ns max# 19687473ns}}}
2025-11-19T14:00:14.296092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436021057326392:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13891
2025-11-19T14:00:14.296112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436021057326393:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13891
2025-11-19T14:00:14.301608Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436023795345208:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.303028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436023795345128:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:14.303626Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436023795345204:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.301961Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436020704939438:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.301987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436020704939360:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.302007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:14.302016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436020704939360:2050] [node 3] ICP22 created new session: [6:7574436020704939444:2050]
2025-11-19T14:00:14.302024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436020704939444:2050] [node 3] ICS09 handshake done sender: [6:7574436020704939437:2116] self: [6:7574436020704939441:0] peer: [3:7574436023795345194:0] socket: 259 qp: -1
2025-11-19T14:00:14.302034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436020704939444:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:14.302056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.307972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436023324537149:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10861
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.306525Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436021057326385:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.306539Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436021057326385:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:14.307169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436021057326394:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13891
2025-11-19T14:00:14.308159Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436021057326379:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:14.308219Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436021057326385:2113] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:14.308340Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436021057326392:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.308363Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436021057326394:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.308373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436016762359013:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.308387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:14.308395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436016762359013:2049] [node 3] ICP22 created new session: [1:7574436021057326397:2049]
2025-11-19T14:00:14.308401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436021057326397:2049] [node 3] ICS09 handshake done sender: [1:7574436021057326385:2113] self: [1:7574436021057326373:0] peer: [3:7574436023795345218:0] socket: 239 qp: -1
2025-11-19T14:00:14.308409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436021057326397:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:14.308416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:14.308658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436016762359013:2049] [node 3] ICP18 handshake (actor [1:7574436021057326394:2118]) is too old
2025-11-19T14:00:14.308076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436023795345193:2108] [node 6] ICP35 event processing took too much time PendingConnection:278(35147349ns) {HandleHandshakeStatus:329(35146251ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(318ns) SwitchToState:178(66ns) ProcessPendingSessionEvents:495(27968125ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27945352ns max# 27872683ns}}}
2025-11-19T14:00:14.308593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436023795345220:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30785
2025-11-19T14:00:14.308866Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436020704939440:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.308935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436020704939358:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:14.309084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436020704939399:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:14.308991Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436021057326393:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.309005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436021057326377:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.309014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:14.309019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436021057326377:2109] [node 5] ICP22 created new session: [1:7574436021057326401:2109]
2025-11-19T14:00:14.309024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436021057326401:2109] [node 5] ICS09 handshake done sender: [1:7574436021057326379:2111] self: [1:7574436021057326378:0] peer: [5:7574436023324537140:0] socket: 236 qp: -1
2025-11-19T14:00:14.309028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436021057326401:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:14.309060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:14.315564Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436023586170579:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.316606Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436021057326376:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.316624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436021057326376:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:14.315418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436023795345188:2106] [node 5] ICP35 event processing took too much time PendingConnection:278(23885645ns) {HandleHandshakeStatus:329(23884525ns) {DropIncomingHandshake:441(160ns) DropOutgoingHandshake:456(261ns) SwitchToState:178(147ns) ProcessPendingSessionEvents:495(23791125ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23767355ns max# 23691897ns}}}
2025-11-19T14:00:14.317745Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436023795345128:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17206) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436023795345208:2116] held: no
2025-11-19T14:00:14.320212Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436023586170569:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.320230Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436023586170569:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:14.320755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436023586170499:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:14.322799Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436023586170569:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:14.323712Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436023586170567:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.323723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436023586170565:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.323732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:14.323740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436023586170565:2079] [node 6] ICP22 created new session: [4:7574436023586170582:2079]
2025-11-19T14:00:14.323760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436023586170582:2079] [node 6] ICS09 handshake done sender: [4:7574436023586170569:2113] self: [4:7574436023586170568:0] peer: [6:7574436020704939448:0] socket: 247 qp: -1
2025-11-19T14:00:14.323765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436023586170582:2079] [node 6] ICS10 traffic start
2025-11-19T14:00:14.323788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:14.318195Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436023795345208:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:14.321357Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436023795345206:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.321362Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436023795345206:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:14.324289Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436023324537126:2104] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:14.324707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436023324537059:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.324722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:14.324728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436023324537059:2048] [node 1] ICP22 created new session: [5:7574436023324537151:2048]
2025-11-19T14:00:14.324733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436023324537151:2048] [node 1] ICS09 handshake done sender: [5:7574436023324537126:2104] self: [5:7574436023324537140:0] peer: [1:7574436021057326378:0] socket: 238 qp: -1
2025-11-19T14:00:14.324738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436023324537151:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:14.324745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:14.325685Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436020704939439:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.325723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436020704939359:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.328978Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436023586170578:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.329015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436023586170499:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16760) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436023586170579:2118] held: no
2025-11-19T14:00:14.329205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436023586170498:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:14.329493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436023586170587:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5900
2025-11-19T14:00:14.329539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436023586170588:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5900
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.329915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436023324537150:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10861
2025-11-19T14:00:14.328919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020704939450:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34425
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.330461Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436024554219644:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.330475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436024554219644:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:14.332328Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436021057326376:2110] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:14.332223Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436024554219660:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.321790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436023795345128:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.321805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:14.321811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436023795345128:2048] [node 1] ICP22 created new session: [3:7574436023795345221:2048]
2025-11-19T14:00:14.321817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436023795345221:2048] [node 1] ICS09 handshake done sender: [3:7574436023795345208:2116] self: [3:7574436023795345218:0] peer: [1:7574436021057326373:0] socket: 263 qp: -1
2025-11-19T14:00:14.321825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436023795345221:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:14.321834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:14.322256Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436023795345206:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:14.322356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436023795345198:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.322371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:14.322376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436023795345198:2110] [node 4] ICP22 created new session: [3:7574436023795345224:2110]
2025-11-19T14:00:14.322382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436023795345224:2110] [node 4] ICS09 handshake done sender: [3:7574436023795345206:2114] self: [3:7574436023795345201:0] peer: [4:7574436023586170581:0] socket: 270 qp: -1
2025-11-19T14:00:14.322389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436023795345224:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:14.322396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:14.332594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020704939451:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34425
2025-11-19T14:00:14.332216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574436023586170565:2079] [node 6] ICP18 handshake (actor [4:7574436023586170578:2117]) is too old
2025-11-19T14:00:14.333236Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436023324537146:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.333271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436023324537060:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:14.332673Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436024554219642:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.332682Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436024554219642:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:14.333167Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436024554219644:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:14.333517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574436023795345188:2106] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:14.333824Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436023324537150:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.333851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574436023324537096:2073] [node 6] ICP18 handshake (actor [5:7574436023324537150:2120]) is too old
2025-11-19T14:00:14.333612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436021057326371:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.333637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:14.333645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436021057326371:2108] [node 6] ICP22 created new session: [1:7574436021057326405:2108]
2025-11-19T14:00:14.333652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436021057326405:2108] [node 6] ICS09 handshake done sender: [1:7574436021057326376:2110] self: [1:7574436021057326372:0] peer: [6:7574436020704939446:0] socket: 235 qp: -1
2025-11-19T14:00:14.333660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436021057326405:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:14.333669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:14.334416Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436020704939440:2119] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:14.334151Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436023586170587:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.336558Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436024554219646:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.336579Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436024554219646:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:14.337010Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436024554219642:2111] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:14.337175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436024554219639:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.337198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:14.337206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436024554219639:2108] [node 6] ICP22 created new session: [2:7574436024554219666:2108]
2025-11-19T14:00:14.337213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436024554219666:2108] [node 6] ICS09 handshake done sender: [2:7574436024554219644:2112] self: [2:7574436024554219643:0] peer: [6:7574436020704939449:0] socket: 241 qp: -1
2025-11-19T14:00:14.337220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436024554219666:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:14.337231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:14.337993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436024554219638:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.338009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:14.338028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436024554219638:2107] [node 4] ICP22 created new session: [2:7574436024554219668:2107]
2025-11-19T14:00:14.338035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436024554219668:2107] [node 4] ICS09 handshake done sender: [2:7574436024554219642:2111] self: [2:7574436024554219641:0] peer: [4:7574436023586170586:0] socket: 240 qp: -1
2025-11-19T14:00:14.338040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436024554219668:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:14.338048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:14.339009Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436024554219646:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:14.339138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436024554219640:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.339159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:14.339164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436024554219640:2109] [node 5] ICP22 created new session: [2:7574436024554219671:2109]
2025-11-19T14:00:14.339170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436024554219671:2109] [node 5] ICS09 handshake done sender: [2:7574436024554219646:2113] self: [2:7574436024554219645:0] peer: [5:7574436023324537154:0] socket: 276 qp: -1
2025-11-19T14:00:14.339175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436024554219671:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:14.339181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.339717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436024554219673:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25174
2025-11-19T14:00:14.340581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436023324537060:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22114) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436023324537146:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.341018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436023795345226:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30785
2025-11-19T14:00:14.345033Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436023586170579:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.345392Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436023795345226:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.345440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436023795345129:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.345774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436023324537155:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10861
2025-11-19T14:00:14.345820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436023324537156:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10861
2025-11-19T14:00:14.346168Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436024554219650:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.346180Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436024554219650:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:14.346207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436023586170497:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:14.347197Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436024554219650:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:14.347301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436024554219573:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.347322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:14.347329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436024554219573:2049] [node 3] ICP22 created new session: [2:7574436024554219675:2049]
2025-11-19T14:00:14.347336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436024554219675:2049] [node 3] ICS09 handshake done sender: [2:7574436024554219650:2114] self: [2:7574436024554219633:0] peer: [3:7574436023795345227:0] socket: 279 qp: -1
2025-11-19T14:00:14.347342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436024554219675:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:14.347350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:14.347049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436020704939399:2067] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3095) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436020704939440:2119] held: no
2025-11-19T14:00:14.347150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436020704939399:2067] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.347161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:14.347167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436020704939399:2067] [node 4] ICP22 created new session: [6:7574436020704939454:2067]
2025-11-19T14:00:14.347173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436020704939454:2067] [node 4] ICS09 handshake done sender: [6:7574436020704939440:2119] self: [6:7574436020704939448:0] peer: [4:7574436023586170568:0] socket: 262 qp: -1
2025-11-19T14:00:14.347181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436020704939454:2067] [node 4] ICS10 traffic start
2025-11-19T14:00:14.347191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:14.346977Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436023324537155:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.347023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436023324537077:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:14.350461Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436021057326389:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.350473Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436021057326389:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.348207Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436023586170576:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.348217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436023586170576:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:14.348728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436023586170499:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.352219Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436023324537146:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:14.348738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:14.348744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436023586170499:2050] [node 3] ICP22 created new session: [4:7574436023586170591:2050]
2025-11-19T14:00:14.348749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436023586170591:2050] [node 3] ICS09 handshake done sender: [4:7574436023586170579:2118] self: [4:7574436023586170581:0] peer: [3:7574436023795345201:0] socket: 272 qp: -1
2025-11-19T14:00:14.348755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436023586170591:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:14.352313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436023324537060:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.348764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:14.352328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:14.352335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436023324537060:2049] [node 2] ICP22 created new session: [5:7574436023324537158:2049]
2025-11-19T14:00:14.352346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436023324537158:2049] [node 2] ICS09 handshake done sender: [5:7574436023324537146:2118] self: [5:7574436023324537154:0] peer: [2:7574436024554219645:0] socket: 277 qp: -1
2025-11-19T14:00:14.352354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436023324537158:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:14.352369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:14.352941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436023586170589:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5900
2025-11-19T14:00:14.353451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436023795345228:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30785
2025-11-19T14:00:14.353664Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436021057326389:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:14.353761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436021057326370:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.353780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:14.353787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436021057326370:2107] [node 4] ICP22 created new session: [1:7574436021057326408:2107]
2025-11-19T14:00:14.353792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436021057326408:2107] [node 4] ICS09 handshake done sender: [1:7574436021057326389:2114] self: [1:7574436021057326374:0] peer: [4:7574436023586170590:0] socket: 284 qp: -1
2025-11-19T14:00:14.353798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436021057326408:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:14.353807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:14.354235Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436023586170567:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:14.354461Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436023586170576:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:14.354420Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436024554219673:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:14.354455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436024554219638:2107] [node 4] ICP18 handshake (actor [2:7574436024554219673:2120]) is too old
2025-11-19T14:00:14.354668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436023586170498:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.354687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:14.354693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436023586170498:2049] [node 2] ICP22 created new session: [4:7574436023586170595:2049]
2025-11-19T14:00:14.354698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436023586170595:2049] [node 2] ICS09 handshake done sender: [4:7574436023586170567:2112] self: [4:7574436023586170586:0] peer: [2:7574436024554219641:0] socket: 245 qp: -1
2025-11-19T14:00:14.354706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436023586170595:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:14.354713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.354773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436023324537160:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10861
2025-11-19T14:00:14.355731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436023795345129:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22114) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436023795345226:2119] held: no
2025-11-19T14:00:14.355908Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436023795345226:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:14.356114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436023795345129:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.356128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:14.356133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436023795345129:2049] [node 2] ICP22 created new session: [3:7574436023795345229:2049]
2025-11-19T14:00:14.356139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436023795345229:2049] [node 2] ICS09 handshake done sender: [3:7574436023795345226:2119] self: [3:7574436023795345227:0] peer: [2:7574436024554219633:0] socket: 229 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.356507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436023586170527:2075] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:14.356529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:14.356535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436023586170527:2075] [node 5] ICP22 created new session: [4:7574436023586170597:2075]
2025-11-19T14:00:14.356541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436023586170597:2075] [node 5] ICS09 handshake done sender: [4:7574436023586170576:2116] self: [4:7574436023586170550:0] peer: [5:7574436023324537157:0] socket: 267 qp: -1
2025-11-19T14:00:14.356545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436023586170597:2075] [node 5] ICS10 traffic start
2025-11-19T14:00:14.356552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:14.356783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436023586170594:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5900
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.359977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436023586170497:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17206) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436023586170587:2119] held: no
2025-11-19T14:00:14.357273Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436020704939436:2115] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:14.359128Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436020704939401:2073] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27946) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436020704939436:2115] held: no
2025-11-19T14:00:14.359204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436020704939401:2073] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.359213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:14.359220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436020704939401:2073] [node 5] ICP22 created new session: [6:7574436020704939456:2073]
2025-11-19T14:00:14.359226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436020704939456:2073] [node 5] ICS09 handshake done sender: [6:7574436020704939436:2115] self: [6:7574436020704939443:0] peer: [5:7574436023324537099:0] socket: 254 qp: -1
2025-11-19T14:00:14.359235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436020704939456:2073] [node 5] ICS10 traffic start
2025-11-19T14:00:14.359243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:14.359456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436020704939358:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17206) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436020704939438:2117] held: no
2025-11-19T14:00:14.359584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020704939452:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34425
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:14.360151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436020704939453:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34425
2025-11-19T14:00:14.360452Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436023586170587:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:14.360740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436023795345229:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:14.360762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:14.361132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436023586170497:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.361146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:14.361151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436023586170497:2048] [node 1] ICP22 created new session: [4:7574436023586170599:2048]
2025-11-19T14:00:14.361155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436023586170599:2048] [node 1] ICS09 handshake done sender: [4:7574436023586170587:2119] self: [4:7574436023586170590:0] peer: [1:7574436021057326374:0] socket: 242 qp: -1
2025-11-19T14:00:14.361161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436023586170599:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:14.361166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:14.361588Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436020704939359:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22114) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436020704939439:2118] held: no
2025-11-19T14:00:14.362007Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436020704939439:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:14.362463Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436020704939438:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:14.362520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436020704939359:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.362533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:14.362538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436020704939359:2049] [node 2] ICP22 created new session: [6:7574436020704939458:2049]
2025-11-19T14:00:14.362544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436020704939458:2049] [node 2] ICS09 handshake done sender: [6:7574436020704939439:2118] self: [6:7574436020704939449:0] peer: [2:7574436024554219643:0] socket: 261 qp: -1
2025-11-19T14:00:14.362549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436020704939458:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:14.362555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:14.362703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436020704939358:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.362709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:14.362716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436020704939358:2048] [node 1] ICP22 created new session: [6:7574436020704939460:2048]
2025-11-19T14:00:14.362720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436020704939460:2048] [node 1] ICS09 handshake done sender: [6:7574436020704939438:2117] self: [6:7574436020704939446:0] peer: [1:7574436021057326372:0] socket: 260 qp: -1
2025-11-19T14:00:14.362724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436020704939460:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:14.362727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:14.361037Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436023324537155:2121] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:14.368039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436023324537077:2062] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3095) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436023324537155:2121] held: no
2025-11-19T14:00:14.368144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436023324537077:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:14.368158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:14.368164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436023324537077:2062] [node 4] ICP22 created new session: [5:7574436023324537161:2062]
2025-11-19T14:00:14.368170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436023324537161:2062] [node 4] ICS09 handshake done sender: [5:7574436023324537155:2121] self: [5:7574436023324537157:0] peer: [4:7574436023586170550:0] socket: 293 qp: -1
2025-11-19T14:00:14.368176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436023324537161:2062] [node 4] ICS10 traffic start
2025-11-19T14:00:14.368185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:14.377847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:14.377937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436024554219658:2048] [node 1] ICS01 socket: 243 reason# EndOfStream
2025-11-19T14:00:14.377947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436024554219572:2048] [node 1] ICP30 unregister session Session# [2:7574436024554219658:2048] VirtualId# [2:7574436024554219656:0]
2025-11-19T14:00:14.377953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436024554219658:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:14.377982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:14.378001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:14.378390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:14.378455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436023795345221:2048] [node 1] ICS01 socket: 263 reason# EndOfStream
2025-11-19T14:00:14.378461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436023795345128:2048] [node 1] ICP30 unregister session Session# [3:7574436023795345221:2048] VirtualId# [3:7574436023795345218:0]
2025-11-19T14:00:14.378465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436023795345221:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:14.378492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:14.378508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:14.384445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:14.384549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436023586170599:2048] [node 1] ICS01 socket: 242 reason# EndOfStream
2025-11-19T14:00:14.384559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436023586170497:2048] [node 1] ICP30 unregister session Session# [4:7574436023586170599:2048] VirtualId# [4:7574436023586170590:0]
2025-11-19T14:00:14.384568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436023586170599:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:14.384635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:14.384662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:14.384705Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:14.384774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436020704939460:2048] [node 1] ICS01 socket: 260 reason# EndOfStream
2025-11-19T14:00:14.384779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436020704939358:2048] [node 1] ICP30 unregister session Session# [6:7574436020704939460:2048] VirtualId# [6:7574436020704939446:0]
2025-11-19T14:00:14.384785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436020704939460:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:14.384825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:14.384846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:14.384882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:14.384961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436023324537151:2048] [node 1] ICS01 socket: 238 reason# EndOfStream
2025-11-19T14:00:14.384965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436023324537059:2048] [node 1] ICP30 unregister session Session# [5:7574436023324537151:2048] VirtualId# [5:7574436023324537140:0]
2025-11-19T14:00:14.384971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436023324537151:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:14.384996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:14.385018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:14.390105Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:14.390200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436023795345229:2049] [node 2] ICS01 socket: 229 reason# EndOfStream
2025-11-19T14:00:14.390209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436023795345129:2049] [node 2] ICP30 unregister session Session# [3:7574436023795345229:2049] VirtualId# [3:7574436023795345227:0]
2025-11-19T14:00:14.390216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436023795345229:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:14.390263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:14.390285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:14.404288Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:14.404385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436023324537158:2049] [node 2] ICS01 socket: 277 reason# EndOfStream
2025-11-19T14:00:14.404394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436023324537060:2049] [node 2] ICP30 unregister session Session# [5:7574436023324537158:2049] VirtualId# [5:7574436023324537154:0]
2025-11-19T14:00:14.404403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436023324537158:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:14.404458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:14.404482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:14.411078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:14.411187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436023586170595:2049] [node 2] ICS01 socket: 245 reason# EndOfStream
2025-11-19T14:00:14.411197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436023586170498:2049] [node 2] ICP30 unregister session Session# [4:7574436023586170595:2049] VirtualId# [4:7574436023586170586:0]
2025-11-19T14:00:14.411204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436023586170595:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:14.411263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:14.411298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:14.412123Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:14.412202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436020704939458:2049] [node 2] ICS01 socket: 261 reason# EndOfStream
2025-11-19T14:00:14.412209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436020704939359:2049] [node 2] ICP30 unregister session Session# [6:7574436020704939458:2049] VirtualId# [6:7574436020704939449:0]
2025-11-19T14:00:14.412214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436020704939458:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:14.412251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:14.412277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:14.428600Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:14.428708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436020704939444:2050] [node 3] ICS01 socket: 259 reason# EndOfStream
2025-11-19T14:00:14.428719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436020704939360:2050] [node 3] ICP30 unregister session Session# [6:7574436020704939444:2050] VirtualId# [6:7574436020704939441:0]
2025-11-19T14:00:14.428726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436020704939444:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:14.428781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:14.428807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:14.429102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:14.429172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436023324537147:2050] [node 3] ICS01 socket: 230 reason# EndOfStream
2025-11-19T14:00:14.429177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436023324537061:2050] [node 3] ICP30 unregister session Session# [5:7574436023324537147:2050] VirtualId# [5:7574436023324537139:0]
2025-11-19T14:00:14.429184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436023324537147:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:14.429211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:14.429233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:14.450959Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:14.451064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436020704939454:2067] [node 4] ICS01 socket: 262 reason# EndOfStream
2025-11-19T14:00:14.451072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436020704939399:2067] [node 4] ICP30 unregister session Session# [6:7574436020704939454:2067] VirtualId# [6:7574436020704939448:0]
2025-11-19T14:00:14.451080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436020704939454:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:14.451147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:14.451172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:14.828298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436021940900139:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:14.828314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436021940900140:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:14.887951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436023689268927:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:14.887971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436023689268928:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:14.912072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436021573056941:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:14.912091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436021573056942:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:14.948034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436024864780247:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:14.948047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436024864780248:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:14.948052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436024864780249:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:15.000047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436024704697687:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:15.000065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436024704697688:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:15.000070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436024704697689:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:15.053842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436027296072953:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:15.053859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436027296072954:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:15.053865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436027296072955:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.102915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436025868024297:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35161
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.101438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436026235867497:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:15.101548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436026235867498:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:15.103783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436026235867508:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20783
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.103901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436028999665044:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38723
2025-11-19T14:00:15.103952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436026235867510:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.103934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436027984236293:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:15.104050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436027984236294:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:15.104123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436027984236295:2111] [node 4] ICP01 ready to work
2025-11-19T14:00:15.106300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436027984236292:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65284
2025-11-19T14:00:15.108901Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436027984236292:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.108940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436023689268927:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.112331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436029159747599:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.115072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436027984236304:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65284
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.116552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436029159747605:2106] [node 6] ICP01 ready to work
2025-11-19T14:00:15.116170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436026235867513:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20783
2025-11-19T14:00:15.119968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436025868024300:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:15.120463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436025868024304:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.121705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436027296073015:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26391
2025-11-19T14:00:15.123825Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436027984236304:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.123931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436026235867500:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.123950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436026235867500:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.127981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436024864780247:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(20695481ns) {Configure:104(20694401ns) {ProcessConfigured:135(20237885ns) {StartInitialHandshake:157(20236483ns) {DropHandshakes:469(1162ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(86ns)} GenerateSessionVirtualId:988(238ns) PrepareNewSessionHandshake:476(1797ns) {SwitchToState:178(782ns)}}}}}
2025-11-19T14:00:15.131332Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436021573056942:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23710449ns) {Configure:104(23709958ns) {ProcessConfigured:135(23290682ns) {StartInitialHandshake:157(23290131ns) {DropHandshakes:469(505ns) {DropIncomingHandshake:441(101ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(252ns) PrepareNewSessionHandshake:476(2033ns) {SwitchToState:178(1549ns)}}}}}
2025-11-19T14:00:15.131591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436025868024312:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.136115Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436026235867513:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.132779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436027984236306:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65284
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.140237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436027984236307:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65284
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.147068Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436027296073015:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.147398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436027296073028:2114] [node 4] ICP01 ready to work
2025-11-19T14:00:15.147431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436027296073026:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26391
2025-11-19T14:00:15.147839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436027296073028:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.144125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436028999665051:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:15.144227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436028999665052:2111] [node 4] ICP01 ready to work
2025-11-19T14:00:15.145376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436028999665050:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38723
2025-11-19T14:00:15.148853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436029159747604:2104] [node 5] ICP01 ready to work
2025-11-19T14:00:15.148764Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436027984236307:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.149518Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436028999665050:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.157705Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436029159747599:2102] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.157821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436029159747620:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T14:00:15.157835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436029159747621:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T14:00:15.157902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436024864780247:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.160423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436025868024316:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35161
2025-11-19T14:00:15.160445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436025868024317:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35161
2025-11-19T14:00:15.160580Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436025868024304:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(39845133ns) {Configure:104(39844199ns) {ProcessConfigured:135(39650457ns) {StartInitialHandshake:157(39649056ns) {DropHandshakes:469(220ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(22ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(1013ns) {SwitchToState:178(277ns)}}}}}
2025-11-19T14:00:15.160040Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436029159747621:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.163316Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436025868024316:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.164622Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436026235867508:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.164676Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436026235867500:2109] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:15.164821Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436025868024317:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.165339Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436029159747607:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.165351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436029159747607:2107] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.165242Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436026235867502:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.165250Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436026235867502:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.165040Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436027984236297:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.165059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436027984236297:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.167885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436027296073021:2097] [node 5] ICP01 ready to work
2025-11-19T14:00:15.167990Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436027296073026:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.171526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436026235867518:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20783
2025-11-19T14:00:15.171547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436026235867519:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20783
2025-11-19T14:00:15.171851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436029159747604:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(22921453ns) {Configure:104(22920057ns) {ProcessConfigured:135(22611013ns) {StartInitialHandshake:157(22610217ns) {DropHandshakes:469(784ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(192ns)} GenerateSessionVirtualId:988(250ns) PrepareNewSessionHandshake:476(1538ns) {SwitchToState:178(767ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.172520Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436026235867519:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.173729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436028999665044:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.173774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436024704697687:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.172286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436027296073031:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26391
2025-11-19T14:00:15.174147Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436027296073031:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.174180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436027296072953:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.176619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436024704697688:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.176669Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436024704697688:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(27083490ns) {IncomingHandshake:261(27082818ns) {DropIncomingHandshake:441(60ns) PrepareNewSessionHandshake:476(416ns) {SwitchToState:178(110ns)} GenerateSessionVirtualId:988(267ns) IssueIncomingHandshakeReply:193(27078203ns) {SwitchToState:178(1005ns)}}}
2025-11-19T14:00:15.179799Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436026235867504:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.179813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436026235867504:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.183053Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436027984236297:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:15.183257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436027984236293:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.183286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:15.183315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436027984236293:2109] [node 5] ICP22 created new session: [2:7574436027984236310:2109]
2025-11-19T14:00:15.183323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436027984236310:2109] [node 5] ICS09 handshake done sender: [2:7574436027984236297:2113] self: [2:7574436027984236296:0] peer: [5:7574436028999665063:0] socket: 244 qp: -1
2025-11-19T14:00:15.183339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436027984236310:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:15.183354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:15.181538Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436026235867512:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.181557Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436026235867512:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.183391Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436026235867502:2110] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:15.184004Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436026235867512:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:15.183129Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436024864780248:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(59101126ns) {Configure:104(59099644ns) {ProcessConfigured:135(58736897ns) {StartInitialHandshake:157(58735976ns) {DropHandshakes:469(1004ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(330ns)} GenerateSessionVirtualId:988(296ns) PrepareNewSessionHandshake:476(1226ns) {SwitchToState:178(611ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.184317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436027296073033:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26391
2025-11-19T14:00:15.184370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436027296073034:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26391
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.184546Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436029159747607:2107] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:15.184527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436028999665065:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38723
2025-11-19T14:00:15.184161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436026235867497:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.184226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:15.184279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436026235867497:2107] [node 4] ICP22 created new session: [1:7574436026235867524:2107]
2025-11-19T14:00:15.184285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436026235867524:2107] [node 4] ICS09 handshake done sender: [1:7574436026235867502:2110] self: [1:7574436026235867501:0] peer: [4:7574436029159747622:0] socket: 237 qp: -1
2025-11-19T14:00:15.184313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436026235867524:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:15.184356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:15.184568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436026235867510:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.184577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:15.184582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436026235867510:2113] [node 6] ICP22 created new session: [1:7574436026235867526:2113]
2025-11-19T14:00:15.184586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436026235867526:2113] [node 6] ICS09 handshake done sender: [1:7574436026235867512:2114] self: [1:7574436026235867511:0] peer: [6:7574436027296073032:0] socket: 272 qp: -1
2025-11-19T14:00:15.184590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436026235867526:2113] [node 6] ICS10 traffic start
2025-11-19T14:00:15.184594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.184815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436029159747605:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.184828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:15.184833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436029159747605:2106] [node 6] ICP22 created new session: [4:7574436029159747628:2106]
2025-11-19T14:00:15.184839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436029159747628:2106] [node 6] ICS09 handshake done sender: [4:7574436029159747607:2107] self: [4:7574436029159747606:0] peer: [6:7574436027296073029:0] socket: 252 qp: -1
2025-11-19T14:00:15.184449Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436025868024297:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.184844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436029159747628:2106] [node 6] ICS10 traffic start
2025-11-19T14:00:15.184488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436021573056942:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.184859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:15.185391Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436028999665050:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:15.184865Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436027984236286:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.184873Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436027984236286:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.186841Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436027984236286:2108] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:15.185752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436021940900139:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.185775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:15.185784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436021940900139:2048] [node 2] ICP22 created new session: [1:7574436026235867528:2048]
2025-11-19T14:00:15.185791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436026235867528:2048] [node 2] ICS09 handshake done sender: [1:7574436026235867500:2109] self: [1:7574436026235867499:0] peer: [2:7574436027984236303:0] socket: 236 qp: -1
2025-11-19T14:00:15.185799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436026235867528:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:15.185806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:15.187857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436027296073035:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26391
2025-11-19T14:00:15.187917Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436027296073015:2103] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:15.188122Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436027296073031:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:15.188002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436029159747627:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T14:00:15.190125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436023689268928:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.190153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:15.190162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436023689268928:2049] [node 3] ICP22 created new session: [2:7574436027984236315:2049]
2025-11-19T14:00:15.190171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436027984236315:2049] [node 3] ICS09 handshake done sender: [2:7574436027984236286:2108] self: [2:7574436027984236285:0] peer: [3:7574436025868024321:0] socket: 235 qp: -1
2025-11-19T14:00:15.190178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436027984236315:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:15.190187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:15.188653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436024864780247:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12113) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436029159747599:2102] held: no
2025-11-19T14:00:15.190187Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436029159747599:2102] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.190376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436027296073028:2114] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.190392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:15.190399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436027296073028:2114] [node 4] ICP22 created new session: [6:7574436027296073036:2114]
2025-11-19T14:00:15.190405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436027296073036:2114] [node 4] ICS09 handshake done sender: [6:7574436027296073015:2103] self: [6:7574436027296073029:0] peer: [4:7574436029159747606:0] socket: 254 qp: -1
2025-11-19T14:00:15.190411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436027296073036:2114] [node 4] ICS10 traffic start
2025-11-19T14:00:15.190419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:15.190949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436027296072953:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.190964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:15.190969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436027296072953:2048] [node 1] ICP22 created new session: [6:7574436027296073038:2048]
2025-11-19T14:00:15.190973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436027296073038:2048] [node 1] ICS09 handshake done sender: [6:7574436027296073031:2115] self: [6:7574436027296073032:0] peer: [1:7574436026235867511:0] socket: 273 qp: -1
2025-11-19T14:00:15.190978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436027296073038:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:15.190983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:15.190600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436027984236314:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65284
2025-11-19T14:00:15.192407Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436026235867504:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:15.196063Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436029159747620:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.196086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436024864780247:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.196099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:15.196106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436024864780247:2048] [node 1] ICP22 created new session: [4:7574436029159747631:2048]
2025-11-19T14:00:15.196114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436029159747631:2048] [node 1] ICS09 handshake done sender: [4:7574436029159747599:2102] self: [4:7574436029159747622:0] peer: [1:7574436026235867501:0] socket: 243 qp: -1
2025-11-19T14:00:15.196121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436029159747631:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:15.196128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:15.196584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436025868024322:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35161
2025-11-19T14:00:15.196721Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436025868024297:2107] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:15.197395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436024704697688:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.197429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:15.197439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436024704697688:2049] [node 2] ICP22 created new session: [5:7574436028999665066:2049]
2025-11-19T14:00:15.197445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436028999665066:2049] [node 2] ICS09 handshake done sender: [5:7574436028999665050:2112] self: [5:7574436028999665063:0] peer: [2:7574436027984236296:0] socket: 245 qp: -1
2025-11-19T14:00:15.197455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436028999665066:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:15.197463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.202184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436028999665069:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38723
2025-11-19T14:00:15.202659Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436027984236292:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.204109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436026235867498:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.204132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:15.204142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436026235867498:2108] [node 5] ICP22 created new session: [1:7574436026235867532:2108]
2025-11-19T14:00:15.204150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436026235867532:2108] [node 5] ICS09 handshake done sender: [1:7574436026235867504:2111] self: [1:7574436026235867503:0] peer: [5:7574436028999665064:0] socket: 240 qp: -1
2025-11-19T14:00:15.204157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436026235867532:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:15.204171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:15.204832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436024864780249:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.204920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436029159747633:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T14:00:15.205985Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436027296073035:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.206037Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436029159747633:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.206068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436024864780248:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.206690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436023689268927:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.206716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:15.206723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436023689268927:2048] [node 1] ICP22 created new session: [2:7574436027984236318:2048]
2025-11-19T14:00:15.206731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436027984236318:2048] [node 1] ICS09 handshake done sender: [2:7574436027984236292:2112] self: [2:7574436027984236303:0] peer: [1:7574436026235867499:0] socket: 241 qp: -1
2025-11-19T14:00:15.206738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436027984236318:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:15.206745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:15.209022Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436026235867518:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.207790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436021573056942:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.207821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:15.207829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436021573056942:2049] [node 2] ICP22 created new session: [3:7574436025868024323:2049]
2025-11-19T14:00:15.207836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436025868024323:2049] [node 2] ICS09 handshake done sender: [3:7574436025868024297:2107] self: [3:7574436025868024321:0] peer: [2:7574436027984236285:0] socket: 238 qp: -1
2025-11-19T14:00:15.207845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436025868024323:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:15.207865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:15.212333Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436021940900139:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(46678805ns) {HandleHandshakeStatus:329(46677647ns) {DropIncomingHandshake:441(318ns) DropOutgoingHandshake:456(563ns) SwitchToState:178(231ns) ProcessPendingSessionEvents:495(16703853ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16683372ns max# 16601358ns}}}
2025-11-19T14:00:15.212367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436021940900139:2048] [node 2] ICP18 handshake (actor [1:7574436026235867508:2112]) is too old
2025-11-19T14:00:15.211849Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436027296073021:2097] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(43742180ns) {Configure:104(43740883ns) {ProcessConfigured:135(43384746ns) {StartInitialHandshake:157(43383226ns) {DropHandshakes:469(1432ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(253ns)} GenerateSessionVirtualId:988(282ns) PrepareNewSessionHandshake:476(1760ns) {SwitchToState:178(545ns)}}}}}
2025-11-19T14:00:15.211964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436027296073021:2097] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.215945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574436027296072953:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(24918466ns) {HandleHandshakeStatus:329(24918145ns) {DropIncomingHandshake:441(400ns) DropOutgoingHandshake:456(2172ns) SwitchToState:178(53ns) ProcessPendingSessionEvents:495(20015643ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 20003799ns max# 19942751ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.218526Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436025868024303:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.218539Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436025868024303:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.219142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436025868024325:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35161
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.219351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436029159747636:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T14:00:15.219584Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436025868024303:2110] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:15.219818Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436029159747620:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:15.219963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436024864780249:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.219988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:15.219994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436024864780249:2050] [node 3] ICP22 created new session: [4:7574436029159747637:2050]
2025-11-19T14:00:15.220001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436029159747637:2050] [node 3] ICS09 handshake done sender: [4:7574436029159747620:2114] self: [4:7574436029159747634:0] peer: [3:7574436025868024302:0] socket: 256 qp: -1
2025-11-19T14:00:15.220008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436029159747637:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:15.220017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:15.221937Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436027984236301:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.221958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436027984236301:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.226365Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436027984236299:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.226380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436027984236299:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.223916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436028999665068:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38723
2025-11-19T14:00:15.224046Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436028999665054:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.224052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436028999665054:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.224859Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436028999665044:2108] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:15.225306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436024704697688:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(39700617ns) {HandleHandshakeStatus:329(39700214ns) {DropIncomingHandshake:441(1285ns) DropOutgoingHandshake:456(2333ns) SwitchToState:178(246ns) ProcessPendingSessionEvents:495(19782144ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19760637ns max# 19674972ns}}}
2025-11-19T14:00:15.225344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574436024704697688:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:15.225715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436028999665070:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38723
2025-11-19T14:00:15.226992Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436027984236306:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.227417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436023689268928:2049] [node 3] ICP18 handshake (actor [2:7574436027984236306:2117]) is too old
2025-11-19T14:00:15.227453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436027984236320:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65284
2025-11-19T14:00:15.229986Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436028999665054:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.227958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436023689268927:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(21172269ns) {HandleHandshakeStatus:329(21171720ns) {DropIncomingHandshake:441(340ns) DropOutgoingHandshake:456(3216ns) SwitchToState:178(55ns) ProcessPendingSessionEvents:495(21080136ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21057905ns max# 20988694ns}}}
2025-11-19T14:00:15.229483Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436027984236299:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:15.229588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436027984236294:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.229605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:15.229613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436027984236294:2110] [node 6] ICP22 created new session: [2:7574436027984236322:2110]
2025-11-19T14:00:15.229618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436027984236322:2110] [node 6] ICS09 handshake done sender: [2:7574436027984236299:2114] self: [2:7574436027984236298:0] peer: [6:7574436027296073040:0] socket: 278 qp: -1
2025-11-19T14:00:15.229625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436027984236322:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:15.229634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:15.232615Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436029159747623:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.232630Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436029159747623:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.230160Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436028999665068:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.230172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436028999665051:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.230180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:15.230186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436028999665051:2110] [node 6] ICP22 created new session: [5:7574436028999665073:2110]
2025-11-19T14:00:15.230190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436028999665073:2110] [node 6] ICS09 handshake done sender: [5:7574436028999665054:2113] self: [5:7574436028999665053:0] peer: [6:7574436027296073042:0] socket: 264 qp: -1
2025-11-19T14:00:15.230195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436028999665073:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:15.230200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:15.230433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436028999665072:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38723
2025-11-19T14:00:15.230451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436028999665052:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.234198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436021573056942:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(26305950ns) {HandleHandshakeStatus:329(26304940ns) {DropIncomingHandshake:441(196ns) DropOutgoingHandshake:456(1766ns) SwitchToState:178(219ns) ProcessPendingSessionEvents:495(12423630ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12396899ns max# 12319152ns}}}
2025-11-19T14:00:15.234661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436027296072954:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.234695Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574436027296072954:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(28664629ns) {IncomingHandshake:261(28664335ns) {DropIncomingHandshake:441(14ns) PrepareNewSessionHandshake:476(477ns) {SwitchToState:178(104ns)} GenerateSessionVirtualId:988(272ns) IssueIncomingHandshakeReply:193(28660792ns) {SwitchToState:178(866ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.236365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436026235867535:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20783
2025-11-19T14:00:15.236267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436024704697687:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.236289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:15.236297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436024704697687:2048] [node 1] ICP22 created new session: [5:7574436028999665079:2048]
2025-11-19T14:00:15.236305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436028999665079:2048] [node 1] ICS09 handshake done sender: [5:7574436028999665044:2108] self: [5:7574436028999665064:0] peer: [1:7574436026235867503:0] socket: 242 qp: -1
2025-11-19T14:00:15.236312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436028999665079:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:15.236330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.237041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436027296073044:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26391
2025-11-19T14:00:15.237088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436027296073045:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26391
2025-11-19T14:00:15.234744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436025868024300:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.234770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:15.234777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436025868024300:2108] [node 4] ICP22 created new session: [3:7574436025868024329:2108]
2025-11-19T14:00:15.234784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436025868024329:2108] [node 4] ICS09 handshake done sender: [3:7574436025868024303:2110] self: [3:7574436025868024302:0] peer: [4:7574436029159747634:0] socket: 253 qp: -1
2025-11-19T14:00:15.234791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436025868024329:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:15.234813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:15.237760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436027296073046:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26391
2025-11-19T14:00:15.237626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436028999665077:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38723
2025-11-19T14:00:15.238655Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436029159747623:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:15.238787Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436027296072954:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1279) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436027296073035:2118] held: no
2025-11-19T14:00:15.238884Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436027296073035:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.240107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436025868024331:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35161
2025-11-19T14:00:15.240648Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436027984236320:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.240878Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436027984236301:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.245048Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436028999665072:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.245635Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436025868024325:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.245691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436021573056941:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.248129Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436026235867515:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.248143Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436026235867515:2116] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.249356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436025868024333:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35161
2025-11-19T14:00:15.249565Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436026235867515:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:15.249819Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436025868024325:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:15.250012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436021940900140:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.250675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436021573056941:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.259705Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436028999665052:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1333) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436028999665068:2118] held: no
2025-11-19T14:00:15.261742Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436028999665068:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:15.239208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436029159747604:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.239222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:15.239228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436029159747604:2104] [node 5] ICP22 created new session: [4:7574436029159747641:2104]
2025-11-19T14:00:15.239234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436029159747641:2104] [node 5] ICS09 handshake done sender: [4:7574436029159747623:2116] self: [4:7574436029159747619:0] peer: [5:7574436028999665076:0] socket: 284 qp: -1
2025-11-19T14:00:15.239239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436029159747641:2104] [node 5] ICS10 traffic start
2025-11-19T14:00:15.239245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:15.241225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436029159747643:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13573
2025-11-19T14:00:15.241439Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436029159747633:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:15.241510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436024864780248:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.241521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:15.241528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436024864780248:2049] [node 2] ICP22 created new session: [4:7574436029159747644:2049]
2025-11-19T14:00:15.241534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436029159747644:2049] [node 2] ICS09 handshake done sender: [4:7574436029159747633:2119] self: [4:7574436029159747635:0] peer: [2:7574436027984236300:0] socket: 291 qp: -1
2025-11-19T14:00:15.241539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436029159747644:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:15.241545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:15.264243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436027296072954:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.264275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:15.264286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436027296072954:2049] [node 2] ICP22 created new session: [6:7574436027296073047:2049]
2025-11-19T14:00:15.264295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436027296073047:2049] [node 2] ICS09 handshake done sender: [6:7574436027296073035:2118] self: [6:7574436027296073040:0] peer: [2:7574436027984236298:0] socket: 281 qp: -1
2025-11-19T14:00:15.264304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436027296073047:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:15.264315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:15.264633Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436027296073044:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.264741Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436027296073026:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:15.240965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436027984236295:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.240978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:15.240984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436027984236295:2111] [node 4] ICP22 created new session: [2:7574436027984236326:2111]
2025-11-19T14:00:15.240990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436027984236326:2111] [node 4] ICS09 handshake done sender: [2:7574436027984236301:2115] self: [2:7574436027984236300:0] peer: [4:7574436029159747635:0] socket: 290 qp: -1
2025-11-19T14:00:15.240997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436027984236326:2111] [node 4] ICS10 traffic start
2025-11-19T14:00:15.241002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:15.265028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436028999665052:2111] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.265046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:15.265053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436028999665052:2111] [node 4] ICP22 created new session: [5:7574436028999665081:2111]
2025-11-19T14:00:15.265059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436028999665081:2111] [node 4] ICS09 handshake done sender: [5:7574436028999665068:2118] self: [5:7574436028999665076:0] peer: [4:7574436029159747619:0] socket: 285 qp: -1
2025-11-19T14:00:15.265066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436028999665081:2111] [node 4] ICS10 traffic start
2025-11-19T14:00:15.265073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:15.265299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436024704697689:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.250048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:15.250059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436021940900140:2049] [node 3] ICP22 created new session: [1:7574436026235867537:2049]
2025-11-19T14:00:15.250068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436026235867537:2049] [node 3] ICS09 handshake done sender: [1:7574436026235867515:2116] self: [1:7574436026235867514:0] peer: [3:7574436025868024332:0] socket: 270 qp: -1
2025-11-19T14:00:15.250078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436026235867537:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:15.250089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:15.265866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574436026235867498:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:15.266843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436021940900140:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(16705024ns) {HandleHandshakeStatus:329(16703712ns) {DropIncomingHandshake:441(5924ns) DropOutgoingHandshake:456(399ns) SwitchToState:178(246ns) ProcessPendingSessionEvents:495(16567913ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 128111ns max# 28829ns}}}
2025-11-19T14:00:15.250693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:15.250700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436021573056941:2048] [node 1] ICP22 created new session: [3:7574436025868024334:2048]
2025-11-19T14:00:15.250707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436025868024334:2048] [node 1] ICS09 handshake done sender: [3:7574436025868024325:2119] self: [3:7574436025868024332:0] peer: [1:7574436026235867514:0] socket: 288 qp: -1
2025-11-19T14:00:15.250713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436025868024334:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:15.250722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:15.268204Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436021573056941:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(17430551ns) {HandleHandshakeStatus:329(17430003ns) {DropIncomingHandshake:441(152ns) DropOutgoingHandshake:456(795ns) SwitchToState:178(213ns) ProcessPendingSessionEvents:495(17335379ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 101699ns max# 19117ns}}}
2025-11-19T14:00:15.268457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574436021573056941:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:15.265627Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436027984236295:2111] [node 4] ICP35 event processing took too much time PendingConnection:278(24550657ns) {HandleHandshakeStatus:329(24550096ns) {DropIncomingHandshake:441(2464ns) DropOutgoingHandshake:456(567ns) SwitchToState:178(68ns) ProcessPendingSessionEvents:495(24476074ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 96432ns max# 15762ns}}}
2025-11-19T14:00:15.265010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436027296072955:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.269232Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436025868024331:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.266130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436024864780248:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(24515476ns) {HandleHandshakeStatus:329(24515095ns) {DropIncomingHandshake:441(173ns) DropOutgoingHandshake:456(1105ns) SwitchToState:178(66ns) ProcessPendingSessionEvents:495(24443717ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 87156ns max# 12357ns}}}
2025-11-19T14:00:15.269728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436027296073021:2097] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.269748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:15.269755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436027296073021:2097] [node 5] ICP22 created new session: [6:7574436027296073050:2097]
2025-11-19T14:00:15.269761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436027296073050:2097] [node 5] ICS09 handshake done sender: [6:7574436027296073026:2113] self: [6:7574436027296073042:0] peer: [5:7574436028999665053:0] socket: 266 qp: -1
2025-11-19T14:00:15.269768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436027296073050:2097] [node 5] ICS10 traffic start
2025-11-19T14:00:15.269777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:15.334555Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436028999665070:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.334635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574436028999665051:2110] [node 6] ICP18 handshake (actor [5:7574436028999665070:2120]) is too old
2025-11-19T14:00:15.335781Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436025868024314:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.335810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436025868024314:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.335863Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436025868024318:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.335871Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436025868024318:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.337397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574436027296073021:2097] [node 5] ICP35 event processing took too much time PendingConnection:278(67570829ns) {HandleHandshakeStatus:329(67569606ns) {DropIncomingHandshake:441(148ns) DropOutgoingHandshake:456(1995ns) SwitchToState:178(66ns) ProcessPendingSessionEvents:495(67450586ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 67431765ns max# 67341126ns}}}
2025-11-19T14:00:15.337070Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436025868024318:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:15.338085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436025868024304:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.338108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:15.338117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436025868024304:2109] [node 5] ICP22 created new session: [3:7574436025868024338:2109]
2025-11-19T14:00:15.338124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436025868024338:2109] [node 5] ICS09 handshake done sender: [3:7574436025868024318:2117] self: [3:7574436025868024307:0] peer: [5:7574436028999665083:0] socket: 296 qp: -1
2025-11-19T14:00:15.338131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436025868024338:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:15.338138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.340951Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436024704697689:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22921) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436028999665072:2121] held: no
2025-11-19T14:00:15.341124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436028999665084:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38723
2025-11-19T14:00:15.341292Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436028999665072:2121] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:15.341355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436024704697689:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.341369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:15.341375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436024704697689:2050] [node 3] ICP22 created new session: [5:7574436028999665085:2050]
2025-11-19T14:00:15.341383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436028999665085:2050] [node 3] ICS09 handshake done sender: [5:7574436028999665072:2121] self: [5:7574436028999665083:0] peer: [3:7574436025868024307:0] socket: 299 qp: -1
2025-11-19T14:00:15.341390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436028999665085:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:15.341397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.361423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436027296073052:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26391
2025-11-19T14:00:15.361605Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436025868024314:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:15.361729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436025868024312:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.361755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:15.361763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436025868024312:2112] [node 6] ICP22 created new session: [3:7574436025868024341:2112]
2025-11-19T14:00:15.361770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436025868024341:2112] [node 6] ICS09 handshake done sender: [3:7574436025868024314:2114] self: [3:7574436025868024313:0] peer: [6:7574436027296073049:0] socket: 292 qp: -1
2025-11-19T14:00:15.361778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436025868024341:2112] [node 6] ICS10 traffic start
2025-11-19T14:00:15.361789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:15.362478Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436027296073044:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:15.362588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436027296072955:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.362605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:15.362611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436027296072955:2050] [node 3] ICP22 created new session: [6:7574436027296073053:2050]
2025-11-19T14:00:15.362617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436027296073053:2050] [node 3] ICS09 handshake done sender: [6:7574436027296073044:2120] self: [6:7574436027296073049:0] peer: [3:7574436025868024313:0] socket: 239 qp: -1
2025-11-19T14:00:15.362621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436027296073053:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:15.362627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:15.378336Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:15.378452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436027984236318:2048] [node 1] ICS07 socket disconnect 241 reason# EndOfStream
2025-11-19T14:00:15.378460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436027984236318:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.378523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:15.378546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436027984236318:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:15.378635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:15.378719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574436025868024334:2048] [node 1] ICS07 socket disconnect 288 reason# EndOfStream
2025-11-19T14:00:15.378724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436025868024334:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.378757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:15.378775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436025868024334:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:15.382891Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436021573056941:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12113) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T14:00:15.382911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436025868024334:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:15.383341Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436021573056941:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12113) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:15.383356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436025868024334:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:15.383362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436021573056941:2048] [node 1] ICP30 unregister session Session# [3:7574436025868024334:2048] VirtualId# [3:7574436025868024332:0]
2025-11-19T14:00:15.383421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:15.383428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436021573056941:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12113) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.383432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12113) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.384026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:15.384128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436028999665079:2048] [node 1] ICS01 socket: 242 reason# EndOfStream
2025-11-19T14:00:15.384136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436024704697687:2048] [node 1] ICP30 unregister session Session# [5:7574436028999665079:2048] VirtualId# [5:7574436028999665064:0]
2025-11-19T14:00:15.384143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436028999665079:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.384191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:15.384217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:15.384257Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:15.384332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436027296073038:2048] [node 1] ICS07 socket disconnect 273 reason# EndOfStream
2025-11-19T14:00:15.384336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436027296073038:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.384361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:15.384380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436027296073038:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:15.384955Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436027296072953:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12113) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:15.384966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436027296073038:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:15.384971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436027296072953:2048] [node 1] ICP30 unregister session Session# [6:7574436027296073038:2048] VirtualId# [6:7574436027296073032:0]
2025-11-19T14:00:15.385019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:15.385024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436027296072953:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12113) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.385026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12113) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.387726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:15.387864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436029159747631:2048] [node 1] ICS01 socket: 243 reason# EndOfStream
2025-11-19T14:00:15.387873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436024864780247:2048] [node 1] ICP30 unregister session Session# [4:7574436029159747631:2048] VirtualId# [4:7574436029159747622:0]
2025-11-19T14:00:15.387881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436029159747631:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.387926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:15.387951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:15.395268Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:15.395365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574436025868024323:2049] [node 2] ICS07 socket disconnect 238 reason# EndOfStream
2025-11-19T14:00:15.395374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436025868024323:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.395414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:15.395433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436025868024323:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:15.395641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574436021573056941:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:15.395896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436027296072953:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:15.398028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436021573056942:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1279) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T14:00:15.398043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436025868024323:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:15.398400Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436021573056942:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1279) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:15.398413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436025868024323:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:15.398418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436021573056942:2049] [node 2] ICP30 unregister session Session# [3:7574436025868024323:2049] VirtualId# [3:7574436025868024321:0]
2025-11-19T14:00:15.398479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:15.398486Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436021573056942:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.398489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.407400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574436021573056942:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:15.427089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:15.427200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436027296073047:2049] [node 2] ICS01 socket: 281 reason# EndOfStream
2025-11-19T14:00:15.427214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436027296072954:2049] [node 2] ICP30 unregister session Session# [6:7574436027296073047:2049] VirtualId# [6:7574436027296073040:0]
2025-11-19T14:00:15.427223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436027296073047:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.427281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:15.427305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:15.433575Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:15.433684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436029159747644:2049] [node 2] ICS07 socket disconnect 291 reason# EndOfStream
2025-11-19T14:00:15.433691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436029159747644:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.433754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:15.433778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436029159747644:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:15.434169Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:15.434239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574436028999665066:2049] [node 2] ICS07 socket disconnect 245 reason# EndOfStream
2025-11-19T14:00:15.434243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436028999665066:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.434283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:15.434306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574436028999665066:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:15.438337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436024704697688:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1279) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:15.438362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436028999665066:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:15.438368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436024704697688:2049] [node 2] ICP30 unregister session Session# [5:7574436028999665066:2049] VirtualId# [5:7574436028999665063:0]
2025-11-19T14:00:15.438431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:15.438440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574436024704697688:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.438443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.438968Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436024864780248:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1279) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:15.438988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436029159747644:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:15.438993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436024864780248:2049] [node 2] ICP30 unregister session Session# [4:7574436029159747644:2049] VirtualId# [4:7574436029159747635:0]
2025-11-19T14:00:15.439051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:15.439057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436024864780248:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.439060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.443882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:15.443969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436027296073053:2050] [node 3] ICS01 socket: 239 reason# EndOfStream
2025-11-19T14:00:15.443977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436027296072955:2050] [node 3] ICP30 unregister session Session# [6:7574436027296073053:2050] VirtualId# [6:7574436027296073049:0]
2025-11-19T14:00:15.443983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436027296073053:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.444039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:15.444064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:15.451418Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:15.451548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436028999665085:2050] [node 3] ICS01 socket: 299 reason# EndOfStream
2025-11-19T14:00:15.451558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436024704697689:2050] [node 3] ICP30 unregister session Session# [5:7574436028999665085:2050] VirtualId# [5:7574436028999665083:0]
2025-11-19T14:00:15.451567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436028999665085:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.451625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:15.451657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:15.452696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574436024704697688:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:15.479935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:15.480045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436027296073036:2114] [node 4] ICS07 socket disconnect 254 reason# EndOfStream
2025-11-19T14:00:15.480053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436027296073036:2114] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.480109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:15.480133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436027296073036:2114] [node 4] ICS15 start handshake
2025-11-19T14:00:15.480612Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436027296073028:2114] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1333) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:15.480626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436027296073036:2114] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:15.480632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436027296073028:2114] [node 4] ICP30 unregister session Session# [6:7574436027296073036:2114] VirtualId# [6:7574436027296073029:0]
2025-11-19T14:00:15.480661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:15.480668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436027296073028:2114] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1333) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.480671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1333) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.487159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436027296073028:2114] [node 4] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:15.584006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436027192883242:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:15.584024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436027192883243:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:15.595949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436028906153268:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:15.595962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436028906153269:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:15.612023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436026761913580:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:15.612036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436026761913581:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:15.627979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436026497579536:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:15.627993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436026497579537:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:15.627998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436026497579538:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:15.655884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436025825814616:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:15.655907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436025825814617:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:15.655912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436025825814618:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:15.659890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436026500661479:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:15.659903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436026500661480:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:15.659908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436026500661481:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:15.667710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436028906153300:2078] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.668850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436026497579552:2062] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26940
2025-11-19T14:00:15.668776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436026761913608:2074] [node 5] ICP01 ready to work
2025-11-19T14:00:15.669259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436026497579556:2064] [node 5] ICP01 ready to work
2025-11-19T14:00:15.670679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436027192883287:2088] [node 6] ICP01 ready to work
2025-11-19T14:00:15.670795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436027192883288:2091] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.673223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436027192883298:2092] [node 4] ICP01 ready to work
2025-11-19T14:00:15.673559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436027192883299:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18724
2025-11-19T14:00:15.670630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436026761913615:2078] [node 4] ICP01 ready to work
2025-11-19T14:00:15.671386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436026761913624:2083] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.672996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436025825814640:2067] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49525
2025-11-19T14:00:15.676396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436025825814651:2074] [node 6] ICP01 ready to work
2025-11-19T14:00:15.677502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436028906153314:2085] [node 5] ICP01 ready to work
2025-11-19T14:00:15.677630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436028906153315:2087] [node 6] ICP01 ready to work
2025-11-19T14:00:15.676564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436026761913635:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48476
2025-11-19T14:00:15.684044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436026497579562:2067] [node 6] ICP01 ready to work
2025-11-19T14:00:15.687863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436025825814635:2062] [node 4] ICP01 ready to work
2025-11-19T14:00:15.692225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436026761913624:2083] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20760232ns) {Configure:104(20759082ns) {ProcessConfigured:135(20508058ns) {StartInitialHandshake:157(20507382ns) {DropHandshakes:469(381ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(139ns) PrepareNewSessionHandshake:476(2740ns) {SwitchToState:178(1567ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.701159Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436026761913635:2092] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.704059Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436026497579556:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(34735646ns) {Configure:104(34734727ns) {ProcessConfigured:135(34468503ns) {StartInitialHandshake:157(34467591ns) {DropHandshakes:469(510ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(1082ns) {SwitchToState:178(380ns)}}}}}
2025-11-19T14:00:15.704091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436026497579582:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26940
2025-11-19T14:00:15.704998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436028906153314:2085] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(27046973ns) {Configure:104(27045901ns) {ProcessConfigured:135(26690011ns) {StartInitialHandshake:157(26689148ns) {DropHandshakes:469(445ns) {DropIncomingHandshake:441(119ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:476(1110ns) {SwitchToState:178(32ns)}}}}}
2025-11-19T14:00:15.705253Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436028906153269:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(33260359ns) {Configure:104(33260010ns) {ProcessConfigured:135(32938393ns) {StartInitialHandshake:157(32937666ns) {DropHandshakes:469(1024ns) {DropIncomingHandshake:441(89ns) DropOutgoingHandshake:456(588ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:476(589ns) {SwitchToState:178(409ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.707202Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436027192883299:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.704606Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436026500661481:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(26989675ns) {Configure:104(26988580ns) {ProcessConfigured:135(26708487ns) {StartInitialHandshake:157(26708143ns) {DropHandshakes:469(564ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(40ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:476(1126ns) {SwitchToState:178(33ns)}}}}}
2025-11-19T14:00:15.706917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436025825814686:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49525
2025-11-19T14:00:15.706944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436025825814687:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49525
2025-11-19T14:00:15.706960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436025825814688:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49525
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.710087Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436026497579582:2085] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.710123Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436026497579552:2062] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.712790Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436025825814640:2067] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.712974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436025825814618:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.708815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436026761913651:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48476
2025-11-19T14:00:15.708889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436026761913652:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48476
2025-11-19T14:00:15.708900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436026761913653:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48476
2025-11-19T14:00:15.712395Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436026761913580:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(36186994ns) {Configure:104(36185931ns) {ProcessConfigured:135(35865872ns) {StartInitialHandshake:157(35864856ns) {DropHandshakes:469(655ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(93ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(2103ns) {SwitchToState:178(956ns)}}}}}
2025-11-19T14:00:15.713249Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436026761913651:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.713281Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436026761913610:2075] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.713287Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436026761913610:2075] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.714058Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436026761913653:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.714105Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436026761913610:2075] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:15.714228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436026761913608:2074] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.714250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:15.714258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436026761913608:2074] [node 5] ICP22 created new session: [3:7574436026761913657:2074]
2025-11-19T14:00:15.714264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436026761913657:2074] [node 5] ICS09 handshake done sender: [3:7574436026761913610:2075] self: [3:7574436026761913609:0] peer: [5:7574436025825814689:0] socket: 243 qp: -1
2025-11-19T14:00:15.714271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436026761913657:2074] [node 5] ICS10 traffic start
2025-11-19T14:00:15.714314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:15.714568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436026761913581:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.714797Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436028906153330:2099] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.714810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436028906153330:2099] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.715734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436026761913661:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48476
2025-11-19T14:00:15.715839Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436028906153330:2099] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:15.716104Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436026761913653:2108] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:15.716249Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436025825814688:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.716321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436028906153269:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.716338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:15.716345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436028906153269:2049] [node 3] ICP22 created new session: [2:7574436028906153336:2049]
2025-11-19T14:00:15.716351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436028906153336:2049] [node 3] ICS09 handshake done sender: [2:7574436028906153330:2099] self: [2:7574436028906153306:0] peer: [3:7574436026761913659:0] socket: 266 qp: -1
2025-11-19T14:00:15.716357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436028906153336:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:15.716375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:15.716397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436027192883319:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18724
2025-11-19T14:00:15.716474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436026500661514:2068] [node 5] ICP01 ready to work
2025-11-19T14:00:15.720082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436026497579581:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26940
2025-11-19T14:00:15.720006Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436025825814687:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.725278Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436026497579536:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(40126249ns) {Configure:104(40124148ns) {ProcessConfigured:135(39827482ns) {StartInitialHandshake:157(39826177ns) {DropHandshakes:469(492ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:476(1698ns) {SwitchToState:178(1272ns)}}}}}
2025-11-19T14:00:15.725380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436026497579536:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.725536Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436026497579537:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(39880003ns) {Configure:104(39879590ns) {ProcessConfigured:135(39641749ns) {StartInitialHandshake:157(39641441ns) {DropHandshakes:469(558ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(171ns) PrepareNewSessionHandshake:476(601ns) {SwitchToState:178(178ns)}}}}}
2025-11-19T14:00:15.725604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436026497579537:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.726917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436028906153315:2087] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(40936620ns) {Configure:104(40936075ns) {ProcessConfigured:135(40647375ns) {StartInitialHandshake:157(40646352ns) {DropHandshakes:469(503ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(179ns) PrepareNewSessionHandshake:476(2534ns) {SwitchToState:178(1346ns)}}}}}
2025-11-19T14:00:15.727126Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436027192883301:2100] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.727138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436027192883301:2100] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.727069Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436028906153302:2079] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.727078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436028906153302:2079] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.728426Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436028906153268:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(51065429ns) {Configure:104(42079735ns) {ProcessConfigured:135(41860414ns) {StartInitialHandshake:157(41859956ns) {DropHandshakes:469(480ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(49ns)} GenerateSessionVirtualId:988(132ns) PrepareNewSessionHandshake:476(505ns) {SwitchToState:178(17ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.728711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436026497579611:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26940
2025-11-19T14:00:15.729027Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436028906153302:2079] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:15.729261Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436026497579552:2062] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:15.729421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436028906153300:2078] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.729439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:15.729447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436028906153300:2078] [node 4] ICP22 created new session: [2:7574436028906153350:2078]
2025-11-19T14:00:15.729454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436028906153350:2078] [node 4] ICS09 handshake done sender: [2:7574436028906153302:2079] self: [2:7574436028906153301:0] peer: [4:7574436026497579610:0] socket: 240 qp: -1
2025-11-19T14:00:15.729461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436028906153350:2078] [node 4] ICS10 traffic start
2025-11-19T14:00:15.729485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:15.729986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436026497579537:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.730005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:15.730010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436026497579537:2049] [node 2] ICP22 created new session: [4:7574436026497579612:2049]
2025-11-19T14:00:15.730015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436026497579612:2049] [node 2] ICS09 handshake done sender: [4:7574436026497579552:2062] self: [4:7574436026497579610:0] peer: [2:7574436028906153301:0] socket: 241 qp: -1
2025-11-19T14:00:15.730020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436026497579612:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:15.730030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:15.731640Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436025825814635:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(43599686ns) {Configure:104(43598778ns) {ProcessConfigured:135(43256018ns) {StartInitialHandshake:157(43255404ns) {DropHandshakes:469(648ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(230ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:476(1283ns) {SwitchToState:178(676ns)}}}}}
2025-11-19T14:00:15.735836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436026500661519:2065] [node 4] ICP01 ready to work
2025-11-19T14:00:15.737323Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436025825814686:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.732870Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436026761913652:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.740716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436026500661555:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3132
2025-11-19T14:00:15.740739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436026500661556:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3132
2025-11-19T14:00:15.741288Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436026497579581:2084] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.741336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436026497579538:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.743883Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436026761913617:2079] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.743899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436026761913617:2079] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.744604Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436026761913617:2079] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:15.743927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436025825814618:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23741) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436025825814640:2067] held: no
2025-11-19T14:00:15.744197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436025825814616:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.744779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436026761913615:2078] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.744803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:15.744810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436026761913615:2078] [node 4] ICP22 created new session: [3:7574436026761913673:2078]
2025-11-19T14:00:15.744816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436026761913673:2078] [node 4] ICS09 handshake done sender: [3:7574436026761913617:2079] self: [3:7574436026761913616:0] peer: [4:7574436026497579619:0] socket: 245 qp: -1
2025-11-19T14:00:15.744823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436026761913673:2078] [node 4] ICS10 traffic start
2025-11-19T14:00:15.744833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.744522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436028906153355:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61716
2025-11-19T14:00:15.744581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436028906153356:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61716
2025-11-19T14:00:15.744612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436028906153357:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61716
2025-11-19T14:00:15.744655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436028906153358:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61716
2025-11-19T14:00:15.744903Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436028906153346:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.744906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436028906153346:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.745395Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436028906153355:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.746038Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436028906153346:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:15.746134Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436028906153357:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.746156Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436028906153356:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.746164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436028906153315:2087] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.746174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:15.746179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436028906153315:2087] [node 6] ICP22 created new session: [2:7574436028906153360:2087]
2025-11-19T14:00:15.746184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436028906153360:2087] [node 6] ICS09 handshake done sender: [2:7574436028906153346:2111] self: [2:7574436028906153318:0] peer: [6:7574436026500661557:0] socket: 278 qp: -1
2025-11-19T14:00:15.746189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436028906153360:2087] [node 6] ICS10 traffic start
2025-11-19T14:00:15.746197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:15.746417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436028906153315:2087] [node 6] ICP18 handshake (actor [2:7574436028906153356:2116]) is too old
2025-11-19T14:00:15.746435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436028906153268:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.744277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436026500661556:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.744313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436026500661480:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.745932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436026500661558:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3132
2025-11-19T14:00:15.746673Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436026500661556:2114] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:15.746789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436026500661480:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.746802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:15.746806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436026500661480:2049] [node 2] ICP22 created new session: [6:7574436026500661559:2049]
2025-11-19T14:00:15.746821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436026500661559:2049] [node 2] ICS09 handshake done sender: [6:7574436026500661556:2114] self: [6:7574436026500661557:0] peer: [2:7574436028906153318:0] socket: 244 qp: -1
2025-11-19T14:00:15.746826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436026500661559:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:15.746831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.747942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436026500661514:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(31324863ns) {Configure:104(31323856ns) {ProcessConfigured:135(30989845ns) {StartInitialHandshake:157(30988288ns) {DropHandshakes:469(501ns) {DropIncomingHandshake:441(212ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:476(456ns) {SwitchToState:178(105ns)}}}}}
2025-11-19T14:00:15.750009Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436027192883297:2098] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.750022Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436027192883297:2098] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.750239Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436028906153358:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.751981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436026761913581:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.752012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:15.752024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436026761913581:2049] [node 2] ICP22 created new session: [3:7574436026761913676:2049]
2025-11-19T14:00:15.752032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436026761913676:2049] [node 2] ICS09 handshake done sender: [3:7574436026761913653:2108] self: [3:7574436026761913659:0] peer: [2:7574436028906153306:0] socket: 270 qp: -1
2025-11-19T14:00:15.752040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436026761913676:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:15.752061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:15.750800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436025825814617:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.751909Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436028906153328:2098] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.750855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436025825814617:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(29764225ns) {IncomingHandshake:261(29763640ns) {DropIncomingHandshake:441(70ns) PrepareNewSessionHandshake:476(1173ns) {SwitchToState:178(372ns)} GenerateSessionVirtualId:988(335ns) IssueIncomingHandshakeReply:193(29753125ns) {SwitchToState:178(358ns)}}}
2025-11-19T14:00:15.751929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436028906153328:2098] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.752456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436025825814704:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49525
2025-11-19T14:00:15.751944Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436026497579583:2086] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.751963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436026497579583:2086] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.752754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436026497579620:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26940
2025-11-19T14:00:15.755505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436025825814701:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49525
2025-11-19T14:00:15.755701Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436027192883295:2097] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.755716Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436027192883295:2097] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.756022Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436027192883319:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.758476Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436026497579581:2084] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:15.758793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436026497579538:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23741) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436026497579581:2084] held: no
2025-11-19T14:00:15.756325Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436025825814640:2067] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:15.759690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436026497579538:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.759715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:15.759724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436026497579538:2050] [node 3] ICP22 created new session: [4:7574436026497579621:2050]
2025-11-19T14:00:15.759733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436026497579621:2050] [node 3] ICS09 handshake done sender: [4:7574436026497579581:2084] self: [4:7574436026497579619:0] peer: [3:7574436026761913616:0] socket: 258 qp: -1
2025-11-19T14:00:15.759741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436026497579621:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:15.759782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:15.760409Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436026500661519:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24386249ns) {Configure:104(24385341ns) {ProcessConfigured:135(24044423ns) {StartInitialHandshake:157(24043910ns) {DropHandshakes:469(1099ns) {DropIncomingHandshake:441(333ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(445ns) PrepareNewSessionHandshake:476(1677ns) {SwitchToState:178(26ns)}}}}}
2025-11-19T14:00:15.760095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436026497579562:2067] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(75947867ns) {Configure:104(75946618ns) {ProcessConfigured:135(75564124ns) {StartInitialHandshake:157(75562259ns) {DropHandshakes:469(1424ns) {DropIncomingHandshake:441(195ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:476(1148ns) {SwitchToState:178(566ns)}}}}}
2025-11-19T14:00:15.760506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436025825814618:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.760525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:15.760531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436025825814618:2050] [node 3] ICP22 created new session: [5:7574436025825814705:2050]
2025-11-19T14:00:15.760539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436025825814705:2050] [node 3] ICS09 handshake done sender: [5:7574436025825814640:2067] self: [5:7574436025825814689:0] peer: [3:7574436026761913609:0] socket: 248 qp: -1
2025-11-19T14:00:15.760547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436025825814705:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:15.760564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:15.764300Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436028906153269:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(47862973ns) {HandleHandshakeStatus:329(47862161ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(358ns) SwitchToState:178(91ns) ProcessPendingSessionEvents:495(24188328ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24161914ns max# 24066685ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.764801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436027192883325:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18724
2025-11-19T14:00:15.764849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436027192883326:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18724
2025-11-19T14:00:15.764880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436027192883327:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18724
2025-11-19T14:00:15.764365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436028906153269:2049] [node 3] ICP18 handshake (actor [2:7574436028906153355:2115]) is too old
2025-11-19T14:00:15.764528Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436026500661555:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.764575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436026500661481:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.767665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436025825814635:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.767712Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436025825814635:2062] [node 4] ICP35 event processing took too much time PendingConnection:278(30285163ns) {IncomingHandshake:261(30284568ns) {DropIncomingHandshake:441(113ns) PrepareNewSessionHandshake:476(1071ns) {SwitchToState:178(430ns)} GenerateSessionVirtualId:988(219ns) IssueIncomingHandshakeReply:193(30278119ns) {SwitchToState:178(33ns)}}}
2025-11-19T14:00:15.767871Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436027192883326:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.767935Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436027192883297:2098] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:15.768044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436027192883288:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.768062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:15.768069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436027192883288:2091] [node 5] ICP22 created new session: [1:7574436027192883330:2091]
2025-11-19T14:00:15.768076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436027192883330:2091] [node 5] ICS09 handshake done sender: [1:7574436027192883297:2098] self: [1:7574436027192883296:0] peer: [5:7574436025825814702:0] socket: 252 qp: -1
2025-11-19T14:00:15.768083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436027192883330:2091] [node 5] ICS10 traffic start
2025-11-19T14:00:15.768092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.770407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436026497579625:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26940
2025-11-19T14:00:15.768699Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436026761913649:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.768720Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436026761913649:2105] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.772750Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436028906153328:2098] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:15.772862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436028906153314:2085] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.772884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:15.772891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436028906153314:2085] [node 5] ICP22 created new session: [2:7574436028906153365:2085]
2025-11-19T14:00:15.772897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436028906153365:2085] [node 5] ICS09 handshake done sender: [2:7574436028906153328:2098] self: [2:7574436028906153316:0] peer: [5:7574436025825814697:0] socket: 265 qp: -1
2025-11-19T14:00:15.772904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436028906153365:2085] [node 5] ICS10 traffic start
2025-11-19T14:00:15.772923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:15.774965Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436027192883325:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.775043Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436027192883301:2100] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:15.775431Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436026497579582:2085] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:15.775903Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436025825814704:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.775655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436027192883298:2092] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.775673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:15.775680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436027192883298:2092] [node 4] ICP22 created new session: [1:7574436027192883334:2092]
2025-11-19T14:00:15.775687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436027192883334:2092] [node 4] ICS09 handshake done sender: [1:7574436027192883301:2100] self: [1:7574436027192883300:0] peer: [4:7574436026497579608:0] socket: 253 qp: -1
2025-11-19T14:00:15.775693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436027192883334:2092] [node 4] ICS10 traffic start
2025-11-19T14:00:15.775702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:15.777357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436027192883298:2092] [node 4] ICP18 handshake (actor [1:7574436027192883325:2117]) is too old
2025-11-19T14:00:15.777595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436026497579536:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.777619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:15.777630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436026497579536:2048] [node 1] ICP22 created new session: [4:7574436026497579626:2048]
2025-11-19T14:00:15.777638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436026497579626:2048] [node 1] ICS09 handshake done sender: [4:7574436026497579582:2085] self: [4:7574436026497579608:0] peer: [1:7574436027192883300:0] socket: 262 qp: -1
2025-11-19T14:00:15.777649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436026497579626:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:15.777671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:15.778805Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436026497579583:2086] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:15.778938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436026497579556:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.778951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:15.778957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436026497579556:2064] [node 5] ICP22 created new session: [4:7574436026497579629:2064]
2025-11-19T14:00:15.778963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436026497579629:2064] [node 5] ICS09 handshake done sender: [4:7574436026497579583:2086] self: [4:7574436026497579557:0] peer: [5:7574436025825814700:0] socket: 263 qp: -1
2025-11-19T14:00:15.778969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436026497579629:2064] [node 5] ICS10 traffic start
2025-11-19T14:00:15.778976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:15.779983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436025825814617:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5361) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436025825814688:2109] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.782606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436026497579631:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26940
2025-11-19T14:00:15.782821Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436026497579631:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.782849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574436026497579556:2064] [node 5] ICP18 handshake (actor [4:7574436026497579631:2119]) is too old
2025-11-19T14:00:15.783905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436025825814635:2062] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15465) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436025825814687:2108] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.788078Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436027192883295:2097] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:15.787605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436026500661565:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3132
2025-11-19T14:00:15.787669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436026500661566:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3132
2025-11-19T14:00:15.787703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436026500661567:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3132
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.792329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436025825814709:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49525
2025-11-19T14:00:15.793057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436025825814710:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49525
2025-11-19T14:00:15.793071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436025825814711:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49525
2025-11-19T14:00:15.792630Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436026761913581:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(59634269ns) {HandleHandshakeStatus:329(59633304ns) {DropIncomingHandshake:441(887ns) DropOutgoingHandshake:456(4401ns) SwitchToState:178(284ns) ProcessPendingSessionEvents:495(24550131ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 24526964ns max# 24449678ns}}}
2025-11-19T14:00:15.793459Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436026500661565:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.793499Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436026500661566:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.793522Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436026500661567:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.793569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436026500661514:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.793582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436026500661519:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.793594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436026500661479:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.796210Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436027192883288:2091] [node 5] ICP35 event processing took too much time PendingConnection:278(28064923ns) {HandleHandshakeStatus:329(28063850ns) {DropIncomingHandshake:441(3502ns) DropOutgoingHandshake:456(208ns) SwitchToState:178(45ns) ProcessPendingSessionEvents:495(125619ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 104780ns max# 31497ns}}}
2025-11-19T14:00:15.796282Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436027192883327:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.797513Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436027192883306:2101] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.797519Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436027192883306:2101] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.799994Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436027192883306:2101] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:15.800204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436027192883287:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.800232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:15.800242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436027192883287:2088] [node 6] ICP22 created new session: [1:7574436027192883340:2088]
2025-11-19T14:00:15.800250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436027192883340:2088] [node 6] ICS09 handshake done sender: [1:7574436027192883306:2101] self: [1:7574436027192883305:0] peer: [6:7574436026500661570:0] socket: 295 qp: -1
2025-11-19T14:00:15.800259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436027192883340:2088] [node 6] ICS10 traffic start
2025-11-19T14:00:15.800270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:15.801749Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436026500661479:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9289) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436026500661567:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.802605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436026497579634:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26940
2025-11-19T14:00:15.801055Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436026761913649:2105] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.804313Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436026497579634:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.804136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436026761913679:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48476
2025-11-19T14:00:15.805300Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436026761913679:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.805330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436026761913580:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:15.804722Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436026497579623:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.804740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436026497579623:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.808039Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436027192883310:2104] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.808050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436027192883310:2104] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.807867Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436025825814655:2078] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:15.807888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436025825814655:2078] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:15.808132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436025825814616:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9289) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436025825814686:2107] held: no
2025-11-19T14:00:15.808238Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436025825814687:2108] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.809038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436028906153367:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61716
2025-11-19T14:00:15.809348Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436028906153357:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:15.809465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436028906153268:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.809488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:15.809495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436028906153268:2048] [node 1] ICP22 created new session: [2:7574436028906153368:2048]
2025-11-19T14:00:15.809501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436028906153368:2048] [node 1] ICS09 handshake done sender: [2:7574436028906153357:2117] self: [2:7574436028906153362:0] peer: [1:7574436027192883293:0] socket: 284 qp: -1
2025-11-19T14:00:15.809509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436028906153368:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:15.809517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:15.810571Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436025825814686:2107] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:15.810751Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436025825814688:2109] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:15.810812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436025825814635:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.810825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:15.810831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436025825814635:2062] [node 4] ICP22 created new session: [5:7574436025825814712:2062]
2025-11-19T14:00:15.810838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436025825814712:2062] [node 4] ICS09 handshake done sender: [5:7574436025825814687:2108] self: [5:7574436025825814700:0] peer: [4:7574436026497579557:0] socket: 267 qp: -1
2025-11-19T14:00:15.810844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436025825814712:2062] [node 4] ICS10 traffic start
2025-11-19T14:00:15.810851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:15.811027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436025825814616:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.811033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:15.811037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436025825814616:2048] [node 1] ICP22 created new session: [5:7574436025825814714:2048]
2025-11-19T14:00:15.811040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436025825814714:2048] [node 1] ICS09 handshake done sender: [5:7574436025825814686:2107] self: [5:7574436025825814702:0] peer: [1:7574436027192883296:0] socket: 261 qp: -1
2025-11-19T14:00:15.811046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436025825814714:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:15.811052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:15.811201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436025825814617:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.811207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:15.811210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436025825814617:2049] [node 2] ICP22 created new session: [5:7574436025825814716:2049]
2025-11-19T14:00:15.811214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436025825814716:2049] [node 2] ICS09 handshake done sender: [5:7574436025825814688:2109] self: [5:7574436025825814697:0] peer: [2:7574436028906153316:0] socket: 268 qp: -1
2025-11-19T14:00:15.811218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436025825814716:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:15.811222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:15.808545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436026497579536:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(30853720ns) {HandleHandshakeStatus:329(30853164ns) {DropIncomingHandshake:441(293ns) DropOutgoingHandshake:456(5096ns) SwitchToState:178(205ns) ProcessPendingSessionEvents:495(15086709ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15064923ns max# 14986563ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.812139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436026500661572:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3132
2025-11-19T14:00:15.812212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436026761913624:2083] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.812248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:15.812260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436026761913624:2083] [node 6] ICP22 created new session: [3:7574436026761913681:2083]
2025-11-19T14:00:15.812267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436026761913681:2083] [node 6] ICS09 handshake done sender: [3:7574436026761913649:2105] self: [3:7574436026761913625:0] peer: [6:7574436026500661564:0] socket: 272 qp: -1
2025-11-19T14:00:15.812274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436026761913681:2083] [node 6] ICS10 traffic start
2025-11-19T14:00:15.812300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:15.814532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436027192883242:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.814568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:15.814577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436027192883242:2048] [node 2] ICP22 created new session: [1:7574436027192883342:2048]
2025-11-19T14:00:15.814585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436027192883342:2048] [node 2] ICS09 handshake done sender: [1:7574436027192883295:2097] self: [1:7574436027192883293:0] peer: [2:7574436028906153362:0] socket: 251 qp: -1
2025-11-19T14:00:15.814591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436027192883342:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:15.814602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:15.814872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436027192883242:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(21481857ns) {HandleHandshakeStatus:329(21481144ns) {DropIncomingHandshake:441(11166ns) DropOutgoingHandshake:456(187ns) SwitchToState:178(78ns) ProcessPendingSessionEvents:495(121039ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 96202ns max# 23834ns}}}
2025-11-19T14:00:15.816724Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436026497579623:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:15.816833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436026497579562:2067] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.816863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:15.816871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436026497579562:2067] [node 6] ICP22 created new session: [4:7574436026497579637:2067]
2025-11-19T14:00:15.816878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436026497579637:2067] [node 6] ICS09 handshake done sender: [4:7574436026497579623:2117] self: [4:7574436026497579563:0] peer: [6:7574436026500661569:0] socket: 246 qp: -1
2025-11-19T14:00:15.816885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436026497579637:2067] [node 6] ICS10 traffic start
2025-11-19T14:00:15.816892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:15.817279Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436026500661567:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.823099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436026500661519:2065] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15465) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436026500661566:2119] held: no
2025-11-19T14:00:15.823212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436026500661479:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.823227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:15.823232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436026500661479:2048] [node 1] ICP22 created new session: [6:7574436026500661575:2048]
2025-11-19T14:00:15.823238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436026500661575:2048] [node 1] ICS09 handshake done sender: [6:7574436026500661567:2120] self: [6:7574436026500661570:0] peer: [1:7574436027192883305:0] socket: 300 qp: -1
2025-11-19T14:00:15.823246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436026500661575:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:15.823270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:15.823643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436026500661574:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3132
2025-11-19T14:00:15.828392Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436025825814655:2078] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:15.828555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436026500661573:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3132
2025-11-19T14:00:15.832154Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436026500661566:2119] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:15.832271Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436026500661555:2113] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:15.828907Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436026761913624:2083] [node 6] ICP35 event processing took too much time PendingConnection:278(27660303ns) {HandleHandshakeStatus:329(27659032ns) {DropIncomingHandshake:441(9353ns) DropOutgoingHandshake:456(354ns) SwitchToState:178(80ns) ProcessPendingSessionEvents:495(148891ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 111812ns max# 33509ns}}}
2025-11-19T14:00:15.832342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436026500661519:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.832364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:15.832373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436026500661519:2065] [node 4] ICP22 created new session: [6:7574436026500661577:2065]
2025-11-19T14:00:15.832380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436026500661577:2065] [node 4] ICS09 handshake done sender: [6:7574436026500661566:2119] self: [6:7574436026500661569:0] peer: [4:7574436026497579563:0] socket: 298 qp: -1
2025-11-19T14:00:15.832389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436026500661577:2065] [node 4] ICS10 traffic start
2025-11-19T14:00:15.832400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:15.832611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436026500661481:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.832619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:15.832625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436026500661481:2050] [node 3] ICP22 created new session: [6:7574436026500661579:2050]
2025-11-19T14:00:15.832630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436026500661579:2050] [node 3] ICS09 handshake done sender: [6:7574436026500661555:2113] self: [6:7574436026500661564:0] peer: [3:7574436026761913625:0] socket: 275 qp: -1
2025-11-19T14:00:15.832634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436026500661579:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:15.832638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:15.833706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436025825814651:2074] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.833745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:15.833751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436025825814651:2074] [node 6] ICP22 created new session: [5:7574436025825814719:2074]
2025-11-19T14:00:15.833757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436025825814719:2074] [node 6] ICS09 handshake done sender: [5:7574436025825814655:2078] self: [5:7574436025825814654:0] peer: [6:7574436026500661568:0] socket: 283 qp: -1
2025-11-19T14:00:15.833763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436025825814719:2074] [node 6] ICS10 traffic start
2025-11-19T14:00:15.833775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.835937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436026500661581:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3132
2025-11-19T14:00:15.840223Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436026500661514:2068] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30145) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436026500661565:2118] held: no
2025-11-19T14:00:15.840608Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436027192883310:2104] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:15.840747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436027192883243:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:15.840779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:15.840787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436027192883243:2049] [node 3] ICP22 created new session: [1:7574436027192883345:2049]
2025-11-19T14:00:15.840795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436027192883345:2049] [node 3] ICS09 handshake done sender: [1:7574436027192883310:2104] self: [1:7574436027192883309:0] peer: [3:7574436026761913680:0] socket: 255 qp: -1
2025-11-19T14:00:15.840802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436027192883345:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:15.840817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:15.841073Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436026500661565:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:15.841277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436026761913683:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48476
2025-11-19T14:00:15.841533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436026761913580:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9289) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436026761913679:2120] held: no
2025-11-19T14:00:15.841617Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436026761913679:2120] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:15.843071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436026761913580:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.843088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:15.843095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436026761913580:2048] [node 1] ICP22 created new session: [3:7574436026761913684:2048]
2025-11-19T14:00:15.843102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436026761913684:2048] [node 1] ICS09 handshake done sender: [3:7574436026761913679:2120] self: [3:7574436026761913680:0] peer: [1:7574436027192883309:0] socket: 301 qp: -1
2025-11-19T14:00:15.843110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436026761913684:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:15.843124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:15.843468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436026500661514:2068] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:15.843492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:15.843501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436026500661514:2068] [node 5] ICP22 created new session: [6:7574436026500661582:2068]
2025-11-19T14:00:15.843508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436026500661582:2068] [node 5] ICS09 handshake done sender: [6:7574436026500661565:2118] self: [6:7574436026500661568:0] peer: [5:7574436025825814654:0] socket: 292 qp: -1
2025-11-19T14:00:15.843516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436026500661582:2068] [node 5] ICS10 traffic start
2025-11-19T14:00:15.843531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:15.866065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:15.866171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436028906153368:2048] [node 1] ICS01 socket: 284 reason# EndOfStream
2025-11-19T14:00:15.866182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436028906153268:2048] [node 1] ICP30 unregister session Session# [2:7574436028906153368:2048] VirtualId# [2:7574436028906153362:0]
2025-11-19T14:00:15.866190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436028906153368:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.866224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:15.866246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:15.866682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:15.866742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436026761913684:2048] [node 1] ICS01 socket: 301 reason# EndOfStream
2025-11-19T14:00:15.866747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436026761913580:2048] [node 1] ICP30 unregister session Session# [3:7574436026761913684:2048] VirtualId# [3:7574436026761913680:0]
2025-11-19T14:00:15.866751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436026761913684:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.866777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:15.866795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:15.883029Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:15.883135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436026500661575:2048] [node 1] ICS01 socket: 300 reason# EndOfStream
2025-11-19T14:00:15.883146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436026500661479:2048] [node 1] ICP30 unregister session Session# [6:7574436026500661575:2048] VirtualId# [6:7574436026500661570:0]
2025-11-19T14:00:15.883154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436026500661575:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.883203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:15.883230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:15.883269Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:15.883333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436026497579626:2048] [node 1] ICS07 socket disconnect 262 reason# EndOfStream
2025-11-19T14:00:15.883337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436026497579626:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.883360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:15.883378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436026497579626:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:15.887443Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436026497579536:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9289) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:15.887472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436026497579626:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:15.887478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436026497579536:2048] [node 1] ICP30 unregister session Session# [4:7574436026497579626:2048] VirtualId# [4:7574436026497579608:0]
2025-11-19T14:00:15.887536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:15.887544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436026497579536:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9289) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.887549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9289) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.887900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:15.888012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436025825814714:2048] [node 1] ICS01 socket: 261 reason# EndOfStream
2025-11-19T14:00:15.888022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436025825814616:2048] [node 1] ICP30 unregister session Session# [5:7574436025825814714:2048] VirtualId# [5:7574436025825814702:0]
2025-11-19T14:00:15.888030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436025825814714:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.888078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:15.888106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:15.888539Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:15.888612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574436026761913676:2049] [node 2] ICS07 socket disconnect 270 reason# EndOfStream
2025-11-19T14:00:15.888618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436026761913676:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.888658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:15.888677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436026761913676:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:15.891434Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436026761913581:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5361) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T14:00:15.891463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436026761913676:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:15.891818Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436026761913581:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5361) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:15.891837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436026761913676:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:15.891843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436026761913581:2049] [node 2] ICP30 unregister session Session# [3:7574436026761913676:2049] VirtualId# [3:7574436026761913659:0]
2025-11-19T14:00:15.891906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:15.891914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436026761913581:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5361) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.891917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5361) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.894806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:15.894889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436026500661559:2049] [node 2] ICS01 socket: 244 reason# EndOfStream
2025-11-19T14:00:15.894897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436026500661480:2049] [node 2] ICP30 unregister session Session# [6:7574436026500661559:2049] VirtualId# [6:7574436026500661557:0]
2025-11-19T14:00:15.894904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436026500661559:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.894956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:15.894979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:15.895004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574436026497579536:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:15.895108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:15.895166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436026497579612:2049] [node 2] ICS07 socket disconnect 241 reason# EndOfStream
2025-11-19T14:00:15.895170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436026497579612:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.895201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:15.895222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436026497579612:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:15.895950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:15.896021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436025825814716:2049] [node 2] ICS01 socket: 268 reason# EndOfStream
2025-11-19T14:00:15.896026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436025825814617:2049] [node 2] ICP30 unregister session Session# [5:7574436025825814716:2049] VirtualId# [5:7574436025825814697:0]
2025-11-19T14:00:15.896032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436025825814716:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.896073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:15.896094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:15.899549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436026497579537:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5361) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:15.899570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436026497579612:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:15.899575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436026497579537:2049] [node 2] ICP30 unregister session Session# [4:7574436026497579612:2049] VirtualId# [4:7574436026497579610:0]
2025-11-19T14:00:15.899594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:15.899601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436026497579537:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5361) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.899604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5361) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:15.907960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574436026497579537:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:15.915597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:15.915666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436025825814705:2050] [node 3] ICS01 socket: 248 reason# EndOfStream
2025-11-19T14:00:15.915674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436025825814618:2050] [node 3] ICP30 unregister session Session# [5:7574436025825814705:2050] VirtualId# [5:7574436025825814689:0]
2025-11-19T14:00:15.915687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436025825814705:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.915727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:15.915867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:15.915849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:15.915920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436026500661579:2050] [node 3] ICS01 socket: 275 reason# EndOfStream
2025-11-19T14:00:15.915927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436026500661481:2050] [node 3] ICP30 unregister session Session# [6:7574436026500661579:2050] VirtualId# [6:7574436026500661564:0]
2025-11-19T14:00:15.915932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436026500661579:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.915966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:15.915989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:15.931993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:15.932111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436026500661577:2065] [node 4] ICS01 socket: 298 reason# EndOfStream
2025-11-19T14:00:15.932123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436026500661519:2065] [node 4] ICP30 unregister session Session# [6:7574436026500661577:2065] VirtualId# [6:7574436026500661569:0]
2025-11-19T14:00:15.932132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436026500661577:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:15.932203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:15.932227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:16.005759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436030391089269:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:16.005776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436030391089270:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:16.023901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436030248807677:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:16.023921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436030248807678:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:16.042921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436031033441197:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:16.042938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436031033441198:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:16.068058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436032623898241:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:16.068077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436032623898242:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:16.068083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436032623898243:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:16.069003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436030952159520:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:16.069017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436030952159521:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:16.069021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436030952159522:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:16.115347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436030633762348:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:16.115421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436030633762349:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:16.115428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436030633762350:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:16.124498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436030952159582:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:16.125713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436032623898255:2060] [node 4] ICP01 ready to work
2025-11-19T14:00:16.126787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436031033441253:2102] [node 4] ICP01 ready to work
2025-11-19T14:00:16.124506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436030248807738:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:16.128136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436030391089335:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:16.128271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436030391089336:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:16.128351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436030391089337:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:16.128977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436030248807744:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:16.129081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436030248807745:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:16.132101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436030633762364:2060] [node 5] ICP01 ready to work
2025-11-19T14:00:16.137362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436030633762393:2078] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.146734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436030248807746:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
2025-11-19T14:00:16.146815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436030248807747:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
2025-11-19T14:00:16.155966Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436032623898255:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(29440457ns) {Configure:104(29439751ns) {ProcessConfigured:135(29238501ns) {StartInitialHandshake:157(29237590ns) {DropHandshakes:469(187ns) {DropIncomingHandshake:441(47ns) DropOutgoingHandshake:456(28ns)} GenerateSessionVirtualId:988(137ns) PrepareNewSessionHandshake:476(2699ns) {SwitchToState:178(1570ns)}}}}}
2025-11-19T14:00:16.156274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436032623898301:2064] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.164012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436031033441253:2102] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(37139639ns) {Configure:104(37138779ns) {ProcessConfigured:135(36773634ns) {StartInitialHandshake:157(36772725ns) {DropHandshakes:469(471ns) {DropIncomingHandshake:441(90ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(39ns) PrepareNewSessionHandshake:476(1780ns) {SwitchToState:178(1254ns)}}}}}
2025-11-19T14:00:16.164236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436031033441263:2103] [node 6] ICP01 ready to work
2025-11-19T14:00:16.164544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436031033441266:2104] [node 5] ICP01 ready to work
2025-11-19T14:00:16.165897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436031033441264:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59728
2025-11-19T14:00:16.165911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436031033441265:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59728
2025-11-19T14:00:16.166642Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030248807746:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.166793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436030248807677:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.172683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436030952159589:2110] [node 5] ICP01 ready to work
2025-11-19T14:00:16.174157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030952159587:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6908
2025-11-19T14:00:16.174236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030952159588:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6908
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.194947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436030248807759:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
2025-11-19T14:00:16.189934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436030633762348:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(52465242ns) {Configure:104(52463576ns) {ProcessConfigured:135(52149557ns) {StartInitialHandshake:157(52147428ns) {DropHandshakes:469(1343ns) {DropIncomingHandshake:441(258ns) DropOutgoingHandshake:456(230ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:476(2663ns) {SwitchToState:178(1529ns)}}}}}
2025-11-19T14:00:16.190716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436030633762426:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10569
2025-11-19T14:00:16.190729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436030633762427:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10569
2025-11-19T14:00:16.198865Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436030952159587:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.206087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436031033441278:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59728
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.211265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032623898310:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44293
2025-11-19T14:00:16.211330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032623898311:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44293
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.213510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436031033441279:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59728
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.216380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436030952159521:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.216630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030952159598:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6908
2025-11-19T14:00:16.217125Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436031033441278:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.217188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436031033441198:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.219880Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030248807740:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.219898Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436030248807740:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.220310Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030248807759:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.220338Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030248807751:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.220343Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436030248807751:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.221161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436030248807760:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.222653Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436031033441265:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.222814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436031033441197:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.222836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436031033441281:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59728
2025-11-19T14:00:16.216941Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030391089330:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.216956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436030391089330:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.217643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436030391089348:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56132
2025-11-19T14:00:16.217679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436030391089349:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56132
2025-11-19T14:00:16.217906Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436030391089330:2107] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:16.218002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436030391089269:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.218019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:16.218027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436030391089269:2048] [node 2] ICP22 created new session: [1:7574436030391089351:2048]
2025-11-19T14:00:16.218036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436030391089351:2048] [node 2] ICS09 handshake done sender: [1:7574436030391089330:2107] self: [1:7574436030391089329:0] peer: [2:7574436030248807758:0] socket: 248 qp: -1
2025-11-19T14:00:16.218044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436030391089351:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:16.218053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.223766Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436030952159588:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.226187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436030952159522:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.226222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030952159602:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6908
2025-11-19T14:00:16.226437Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436031033441262:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.226446Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436031033441262:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.227207Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436031033441262:2110] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:16.228671Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436030248807740:2108] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:16.228847Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436030248807751:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:16.228945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436030248807738:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.228966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:16.228974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436030248807738:2107] [node 4] ICP22 created new session: [2:7574436030248807764:2107]
2025-11-19T14:00:16.228981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436030248807764:2107] [node 4] ICS09 handshake done sender: [2:7574436030248807740:2108] self: [2:7574436030248807739:0] peer: [4:7574436030952159601:0] socket: 246 qp: -1
2025-11-19T14:00:16.228990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436030248807764:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:16.229003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:16.227153Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030391089348:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.227195Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030391089341:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.227198Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436030391089341:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.229359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436030391089269:2048] [node 2] ICP18 handshake (actor [1:7574436030391089348:2115]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.231786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436031033441253:2102] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.231804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:16.231816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436031033441253:2102] [node 4] ICP22 created new session: [3:7574436031033441284:2102]
2025-11-19T14:00:16.231823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436031033441284:2102] [node 4] ICS09 handshake done sender: [3:7574436031033441262:2110] self: [3:7574436031033441254:0] peer: [4:7574436030952159603:0] socket: 260 qp: -1
2025-11-19T14:00:16.231830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436031033441284:2102] [node 4] ICS10 traffic start
2025-11-19T14:00:16.231839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:16.232549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436030248807678:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.230173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436030633762428:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10569
2025-11-19T14:00:16.230218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436030633762429:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10569
2025-11-19T14:00:16.232583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:16.232591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436030248807678:2049] [node 3] ICP22 created new session: [2:7574436030248807766:2049]
2025-11-19T14:00:16.232597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436030248807766:2049] [node 3] ICS09 handshake done sender: [2:7574436030248807751:2114] self: [2:7574436030248807750:0] peer: [3:7574436031033441280:0] socket: 264 qp: -1
2025-11-19T14:00:16.232601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436030248807766:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:16.232605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:16.232998Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436031033441264:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.233057Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436031033441278:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:16.232782Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436030248807746:2111] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:16.233757Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436030391089341:2112] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:16.233928Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436030633762427:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.233962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436030633762350:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.233942Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030391089349:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.233954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436030391089270:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.233965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:16.233972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436030391089270:2049] [node 3] ICP22 created new session: [1:7574436030391089355:2049]
2025-11-19T14:00:16.233977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436030391089355:2049] [node 3] ICS09 handshake done sender: [1:7574436030391089341:2112] self: [1:7574436030391089340:0] peer: [3:7574436031033441282:0] socket: 255 qp: -1
2025-11-19T14:00:16.233982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436030391089355:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:16.233989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:16.233781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436030248807677:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.233793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:16.233798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436030248807677:2048] [node 1] ICP22 created new session: [2:7574436030248807768:2048]
2025-11-19T14:00:16.233803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436030248807768:2048] [node 1] ICS09 handshake done sender: [2:7574436030248807746:2111] self: [2:7574436030248807758:0] peer: [1:7574436030391089329:0] socket: 249 qp: -1
2025-11-19T14:00:16.233808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436030248807768:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:16.233814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.237372Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436031033441268:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.237382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436031033441268:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.237778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436031033441198:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.237792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:16.237797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436031033441198:2049] [node 2] ICP22 created new session: [3:7574436031033441288:2049]
2025-11-19T14:00:16.237802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436031033441288:2049] [node 2] ICS09 handshake done sender: [3:7574436031033441278:2117] self: [3:7574436031033441280:0] peer: [2:7574436030248807750:0] socket: 266 qp: -1
2025-11-19T14:00:16.237808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436031033441288:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:16.237815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:16.237968Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574436031033441198:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:16.238167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436031033441286:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59728
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.238479Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436032623898311:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.238511Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436032623898310:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.238589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032623898321:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44293
2025-11-19T14:00:16.238618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032623898322:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44293
2025-11-19T14:00:16.238629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436032623898241:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.238795Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030391089339:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.238804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436030391089339:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.240174Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436030633762429:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.240202Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436030633762428:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.240227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436030633762348:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.240240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436030633762364:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.240016Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436030952159598:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.240084Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436030952159587:2111] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:16.240496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030952159604:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6908
2025-11-19T14:00:16.240625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436030952159521:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.240636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:16.240641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436030952159521:2049] [node 2] ICP22 created new session: [4:7574436030952159606:2049]
2025-11-19T14:00:16.240647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436030952159606:2049] [node 2] ICS09 handshake done sender: [4:7574436030952159587:2111] self: [4:7574436030952159601:0] peer: [2:7574436030248807739:0] socket: 250 qp: -1
2025-11-19T14:00:16.240654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436030952159606:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:16.240660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.241148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030952159605:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6908
2025-11-19T14:00:16.241222Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436030952159588:2112] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:16.241385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436030952159522:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.241398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:16.241415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436030952159522:2050] [node 3] ICP22 created new session: [4:7574436030952159609:2050]
2025-11-19T14:00:16.241419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436030952159609:2050] [node 3] ICS09 handshake done sender: [4:7574436030952159588:2112] self: [4:7574436030952159603:0] peer: [3:7574436031033441254:0] socket: 267 qp: -1
2025-11-19T14:00:16.241424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436030952159609:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:16.241429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.244610Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436032623898321:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.244647Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436032623898313:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.244651Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436032623898313:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.245358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032623898325:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44293
2025-11-19T14:00:16.245373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436032623898255:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.244285Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436030633762426:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.244328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436030633762393:2078] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.245588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436030633762434:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10569
2025-11-19T14:00:16.245953Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436030952159584:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.245963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436030952159584:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.245960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436030248807677:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:16.246238Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030248807747:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.246769Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436030391089339:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:16.246862Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436032623898313:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:16.246516Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436030952159593:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.246537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436030952159593:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.247563Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436031033441279:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.247974Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436031033441265:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:16.248167Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030391089343:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.248179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436030391089343:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.248887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436030391089335:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.248899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:16.248905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436030391089335:2108] [node 5] ICP22 created new session: [1:7574436030391089360:2108]
2025-11-19T14:00:16.248910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436030391089360:2108] [node 5] ICS09 handshake done sender: [1:7574436030391089339:2111] self: [1:7574436030391089338:0] peer: [5:7574436032623898323:0] socket: 254 qp: -1
2025-11-19T14:00:16.248917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436030391089360:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:16.248928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:16.249900Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436030633762428:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.249989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436030633762435:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10569
2025-11-19T14:00:16.250033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436030633762436:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10569
2025-11-19T14:00:16.250052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436030633762437:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10569
2025-11-19T14:00:16.250061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436030633762438:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10569
2025-11-19T14:00:16.250068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436030633762364:2060] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.250078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:16.250083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436030633762364:2060] [node 5] ICP22 created new session: [6:7574436030633762439:2060]
2025-11-19T14:00:16.250088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436030633762439:2060] [node 5] ICS09 handshake done sender: [6:7574436030633762428:2117] self: [6:7574436030633762432:0] peer: [5:7574436032623898312:0] socket: 290 qp: -1
2025-11-19T14:00:16.250093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436030633762439:2060] [node 5] ICS10 traffic start
2025-11-19T14:00:16.250099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.255997Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436030391089343:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:16.256194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436030391089336:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.256206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:16.256214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436030391089336:2109] [node 6] ICP22 created new session: [1:7574436030391089364:2109]
2025-11-19T14:00:16.256221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436030391089364:2109] [node 6] ICS09 handshake done sender: [1:7574436030391089343:2113] self: [1:7574436030391089342:0] peer: [6:7574436030633762431:0] socket: 281 qp: -1
2025-11-19T14:00:16.256226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436030391089364:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:16.256237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:16.256539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436030391089363:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56132
2025-11-19T14:00:16.256103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436032623898301:2064] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.256142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:16.256152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436032623898301:2064] [node 6] ICP22 created new session: [5:7574436032623898330:2064]
2025-11-19T14:00:16.256160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436032623898330:2064] [node 6] ICS09 handshake done sender: [5:7574436032623898313:2114] self: [5:7574436032623898312:0] peer: [6:7574436030633762432:0] socket: 274 qp: -1
2025-11-19T14:00:16.256169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436032623898330:2064] [node 6] ICS10 traffic start
2025-11-19T14:00:16.256180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:16.256744Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436030952159584:2109] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:16.257008Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436030952159593:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:16.257471Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436032623898311:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.257598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032623898328:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44293
2025-11-19T14:00:16.257639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032623898329:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44293
2025-11-19T14:00:16.257207Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436030952159605:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.257221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436030952159582:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.257230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:16.257236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436030952159582:2108] [node 6] ICP22 created new session: [4:7574436030952159614:2108]
2025-11-19T14:00:16.257241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436030952159614:2108] [node 6] ICS09 handshake done sender: [4:7574436030952159584:2109] self: [4:7574436030952159583:0] peer: [6:7574436030633762433:0] socket: 247 qp: -1
2025-11-19T14:00:16.257246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436030952159614:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:16.257254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:16.257468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436030952159589:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.257480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:16.257484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436030952159589:2110] [node 5] ICP22 created new session: [4:7574436030952159616:2110]
2025-11-19T14:00:16.257488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436030952159616:2110] [node 5] ICS09 handshake done sender: [4:7574436030952159593:2114] self: [4:7574436030952159592:0] peer: [5:7574436032623898327:0] socket: 278 qp: -1
2025-11-19T14:00:16.257492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436030952159616:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:16.257497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:16.258322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436031033441197:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.258341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:16.258347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436031033441197:2048] [node 1] ICP22 created new session: [3:7574436031033441293:2048]
2025-11-19T14:00:16.258353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436031033441293:2048] [node 1] ICS09 handshake done sender: [3:7574436031033441265:2112] self: [3:7574436031033441282:0] peer: [1:7574436030391089340:0] socket: 259 qp: -1
2025-11-19T14:00:16.258359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436031033441293:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:16.258366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:16.258511Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030391089363:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.258541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436030391089336:2109] [node 6] ICP18 handshake (actor [1:7574436030391089363:2117]) is too old
2025-11-19T14:00:16.264153Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436030633762429:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.264806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436032623898241:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.264828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:16.264842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436032623898241:2048] [node 1] ICP22 created new session: [5:7574436032623898333:2048]
2025-11-19T14:00:16.264850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436032623898333:2048] [node 1] ICS09 handshake done sender: [5:7574436032623898311:2113] self: [5:7574436032623898323:0] peer: [1:7574436030391089338:0] socket: 270 qp: -1
2025-11-19T14:00:16.264858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436032623898333:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:16.264867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:16.266315Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436030633762426:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:16.264647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436030391089366:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56132
2025-11-19T14:00:16.267943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436030952159520:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.268212Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436031033441268:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.268397Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030391089345:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.268411Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436030391089345:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.270357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436031033441263:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.270386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:16.270393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436031033441263:2103] [node 6] ICP22 created new session: [3:7574436031033441296:2103]
2025-11-19T14:00:16.270400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436031033441296:2103] [node 6] ICS09 handshake done sender: [3:7574436031033441268:2113] self: [3:7574436031033441267:0] peer: [6:7574436030633762430:0] socket: 261 qp: -1
2025-11-19T14:00:16.270408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436031033441296:2103] [node 6] ICS10 traffic start
2025-11-19T14:00:16.270417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:16.269758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030952159619:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6908
2025-11-19T14:00:16.270993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436030633762348:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.271016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:16.271022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436030633762348:2048] [node 1] ICP22 created new session: [6:7574436030633762442:2048]
2025-11-19T14:00:16.271029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436030633762442:2048] [node 1] ICS09 handshake done sender: [6:7574436030633762429:2118] self: [6:7574436030633762431:0] peer: [1:7574436030391089342:0] socket: 291 qp: -1
2025-11-19T14:00:16.271047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436030633762442:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:16.271055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:16.271209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574436030633762348:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:16.271396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436030633762393:2078] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.271406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:16.271412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436030633762393:2078] [node 4] ICP22 created new session: [6:7574436030633762444:2078]
2025-11-19T14:00:16.271417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436030633762444:2078] [node 4] ICS09 handshake done sender: [6:7574436030633762426:2115] self: [6:7574436030633762433:0] peer: [4:7574436030952159583:0] socket: 268 qp: -1
2025-11-19T14:00:16.271422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436030633762444:2078] [node 4] ICS10 traffic start
2025-11-19T14:00:16.271430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:16.271817Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436030633762435:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.272466Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436030391089345:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:16.272653Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436030952159605:2120] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:16.272713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436030952159520:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.272724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:16.272729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436030952159520:2048] [node 1] ICP22 created new session: [4:7574436030952159620:2048]
2025-11-19T14:00:16.272733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436030952159620:2048] [node 1] ICS09 handshake done sender: [4:7574436030952159605:2120] self: [4:7574436030952159618:0] peer: [1:7574436030391089344:0] socket: 295 qp: -1
2025-11-19T14:00:16.272738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436030952159620:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:16.272744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:16.276038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436032623898255:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64538) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436032623898321:2117] held: no
2025-11-19T14:00:16.276515Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436032623898328:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.276543Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436032623898322:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.276555Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436032623898321:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.276987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436032623898255:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.277000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:16.277007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436032623898255:2060] [node 4] ICP22 created new session: [5:7574436032623898335:2060]
2025-11-19T14:00:16.277012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436032623898335:2060] [node 4] ICS09 handshake done sender: [5:7574436032623898321:2117] self: [5:7574436032623898327:0] peer: [4:7574436030952159592:0] socket: 262 qp: -1
2025-11-19T14:00:16.277017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436032623898335:2060] [node 4] ICS10 traffic start
2025-11-19T14:00:16.277024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:16.278659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436032623898242:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.278677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436032623898243:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.276751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436030248807773:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54539
2025-11-19T14:00:16.278982Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030248807753:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.278988Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436030248807753:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.279583Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436030248807753:2115] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:16.279660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436030248807744:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.279669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:16.279676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436030248807744:2109] [node 5] ICP22 created new session: [2:7574436030248807775:2109]
2025-11-19T14:00:16.279682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436030248807775:2109] [node 5] ICS09 handshake done sender: [2:7574436030248807753:2115] self: [2:7574436030248807752:0] peer: [5:7574436032623898337:0] socket: 263 qp: -1
2025-11-19T14:00:16.279688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436030248807775:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:16.279697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:16.280035Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436031033441272:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.280049Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436031033441272:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.281437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436030391089337:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.281460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:16.281466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436030391089337:2110] [node 4] ICP22 created new session: [1:7574436030391089368:2110]
2025-11-19T14:00:16.281472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436030391089368:2110] [node 4] ICS09 handshake done sender: [1:7574436030391089345:2114] self: [1:7574436030391089344:0] peer: [4:7574436030952159618:0] socket: 265 qp: -1
2025-11-19T14:00:16.281477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436030391089368:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:16.281485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:16.283641Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436031033441272:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.284039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032623898339:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44293
2025-11-19T14:00:16.283772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436031033441266:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.283793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:16.283800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436031033441266:2104] [node 5] ICP22 created new session: [3:7574436031033441299:2104]
2025-11-19T14:00:16.283805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436031033441299:2104] [node 5] ICS09 handshake done sender: [3:7574436031033441272:2115] self: [3:7574436031033441271:0] peer: [5:7574436032623898338:0] socket: 271 qp: -1
2025-11-19T14:00:16.283811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436031033441299:2104] [node 5] ICS10 traffic start
2025-11-19T14:00:16.283821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:16.284161Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030248807773:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.284204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436030248807744:2109] [node 5] ICP18 handshake (actor [2:7574436030248807773:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.285191Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436030633762350:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20713) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436030633762427:2116] held: no
2025-11-19T14:00:16.285539Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436030633762427:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:16.284912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032623898340:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44293
2025-11-19T14:00:16.292106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436030633762349:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.292136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436030633762350:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.292155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:16.292162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436030633762350:2050] [node 3] ICP22 created new session: [6:7574436030633762447:2050]
2025-11-19T14:00:16.292170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436030633762447:2050] [node 3] ICS09 handshake done sender: [6:7574436030633762427:2116] self: [6:7574436030633762430:0] peer: [3:7574436031033441267:0] socket: 269 qp: -1
2025-11-19T14:00:16.292181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436030633762447:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:16.292192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:16.292329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436032623898242:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3029) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436032623898328:2120] held: no
2025-11-19T14:00:16.293501Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030248807755:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.293518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436030248807755:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.294713Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436030248807755:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:16.294822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436030248807745:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.294842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:16.294849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436030248807745:2110] [node 6] ICP22 created new session: [2:7574436030248807778:2110]
2025-11-19T14:00:16.294856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436030248807778:2110] [node 6] ICS09 handshake done sender: [2:7574436030248807755:2116] self: [2:7574436030248807754:0] peer: [6:7574436030633762446:0] socket: 299 qp: -1
2025-11-19T14:00:16.294862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436030248807778:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:16.294873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.299977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436030633762349:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3029) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436030633762435:2120] held: no
2025-11-19T14:00:16.300188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436030633762449:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10569
2025-11-19T14:00:16.304461Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436030633762435:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:16.304356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436032623898243:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20713) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436032623898322:2118] held: no
2025-11-19T14:00:16.304491Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436032623898328:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:16.304794Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436032623898322:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:16.304566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436030633762349:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.304588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:16.304595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436030633762349:2049] [node 2] ICP22 created new session: [6:7574436030633762450:2049]
2025-11-19T14:00:16.304602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436030633762450:2049] [node 2] ICS09 handshake done sender: [6:7574436030633762435:2120] self: [6:7574436030633762446:0] peer: [2:7574436030248807754:0] socket: 300 qp: -1
2025-11-19T14:00:16.304613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436030633762450:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:16.304626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:16.305218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436032623898242:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.305234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:16.305241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436032623898242:2049] [node 2] ICP22 created new session: [5:7574436032623898341:2049]
2025-11-19T14:00:16.305246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436032623898341:2049] [node 2] ICS09 handshake done sender: [5:7574436032623898328:2120] self: [5:7574436032623898337:0] peer: [2:7574436030248807752:0] socket: 305 qp: -1
2025-11-19T14:00:16.305252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436032623898341:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:16.305258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:16.305432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436032623898243:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.305438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:16.305442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436032623898243:2050] [node 3] ICP22 created new session: [5:7574436032623898343:2050]
2025-11-19T14:00:16.305445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436032623898343:2050] [node 3] ICS09 handshake done sender: [5:7574436032623898322:2118] self: [5:7574436032623898338:0] peer: [3:7574436031033441271:0] socket: 293 qp: -1
2025-11-19T14:00:16.305450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436032623898343:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:16.305455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:16.324707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:16.324821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436030248807768:2048] [node 1] ICS01 socket: 249 reason# EndOfStream
2025-11-19T14:00:16.324834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436030248807677:2048] [node 1] ICP30 unregister session Session# [2:7574436030248807768:2048] VirtualId# [2:7574436030248807758:0]
2025-11-19T14:00:16.324845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436030248807768:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.324895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:16.324918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:16.332250Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:16.332376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574436031033441293:2048] [node 1] ICS07 socket disconnect 259 reason# EndOfStream
2025-11-19T14:00:16.332389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436031033441293:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.332450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:16.332475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436031033441293:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:16.332992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436031033441197:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17627) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:16.333009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436031033441293:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:16.333015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436031033441197:2048] [node 1] ICP30 unregister session Session# [3:7574436031033441293:2048] VirtualId# [3:7574436031033441282:0]
2025-11-19T14:00:16.333078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:16.333087Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436031033441197:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17627) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:16.333091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17627) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:16.338931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:16.339044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436032623898333:2048] [node 1] ICS01 socket: 270 reason# EndOfStream
2025-11-19T14:00:16.339056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436032623898241:2048] [node 1] ICP30 unregister session Session# [5:7574436032623898333:2048] VirtualId# [5:7574436032623898323:0]
2025-11-19T14:00:16.339064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436032623898333:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.339129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:16.339159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:16.339204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:16.339266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436030952159620:2048] [node 1] ICS01 socket: 295 reason# EndOfStream
2025-11-19T14:00:16.339271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436030952159520:2048] [node 1] ICP30 unregister session Session# [4:7574436030952159620:2048] VirtualId# [4:7574436030952159618:0]
2025-11-19T14:00:16.339275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436030952159620:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.339306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:16.339328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:16.339858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:16.339943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436030633762442:2048] [node 1] ICS01 socket: 291 reason# EndOfStream
2025-11-19T14:00:16.339951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436030633762348:2048] [node 1] ICP30 unregister session Session# [6:7574436030633762442:2048] VirtualId# [6:7574436030633762431:0]
2025-11-19T14:00:16.339959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436030633762442:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.340002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:16.340029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:16.346706Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:16.346820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436031033441288:2049] [node 2] ICS01 socket: 266 reason# EndOfStream
2025-11-19T14:00:16.346830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436031033441198:2049] [node 2] ICP30 unregister session Session# [3:7574436031033441288:2049] VirtualId# [3:7574436031033441280:0]
2025-11-19T14:00:16.346838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436031033441288:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.346894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:16.346915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:16.354011Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:16.354125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436032623898341:2049] [node 2] ICS01 socket: 305 reason# EndOfStream
2025-11-19T14:00:16.354135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436032623898242:2049] [node 2] ICP30 unregister session Session# [5:7574436032623898341:2049] VirtualId# [5:7574436032623898337:0]
2025-11-19T14:00:16.354142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436032623898341:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.354194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:16.354227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:16.354265Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:16.354331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436030633762450:2049] [node 2] ICS01 socket: 300 reason# EndOfStream
2025-11-19T14:00:16.354336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436030633762349:2049] [node 2] ICP30 unregister session Session# [6:7574436030633762450:2049] VirtualId# [6:7574436030633762446:0]
2025-11-19T14:00:16.354341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436030633762450:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.354361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:16.354384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:16.354607Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:16.354668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436030952159606:2049] [node 2] ICS01 socket: 250 reason# EndOfStream
2025-11-19T14:00:16.354672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436030952159521:2049] [node 2] ICP30 unregister session Session# [4:7574436030952159606:2049] VirtualId# [4:7574436030952159601:0]
2025-11-19T14:00:16.354676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436030952159606:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.354705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:16.354739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:16.360848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T14:00:16.360966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436030952159609:2050] [node 3] ICS01 socket: 267 reason# EndOfStream
2025-11-19T14:00:16.360980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436030952159522:2050] [node 3] ICP30 unregister session Session# [4:7574436030952159609:2050] VirtualId# [4:7574436030952159603:0]
2025-11-19T14:00:16.360989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436030952159609:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.361035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:16.361062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:16.361185Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:16.361329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436030633762447:2050] [node 3] ICS01 socket: 269 reason# EndOfStream
2025-11-19T14:00:16.361335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436030633762350:2050] [node 3] ICP30 unregister session Session# [6:7574436030633762447:2050] VirtualId# [6:7574436030633762430:0]
2025-11-19T14:00:16.361339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436030633762447:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.361367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:16.361387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:16.361469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:16.361527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436032623898343:2050] [node 3] ICS01 socket: 293 reason# EndOfStream
2025-11-19T14:00:16.361531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436032623898243:2050] [node 3] ICP30 unregister session Session# [5:7574436032623898343:2050] VirtualId# [5:7574436032623898338:0]
2025-11-19T14:00:16.361535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436032623898343:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.361558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:16.361577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:16.380801Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:16.380920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436030633762444:2078] [node 4] ICS01 socket: 268 reason# EndOfStream
2025-11-19T14:00:16.380931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436030633762393:2078] [node 4] ICP30 unregister session Session# [6:7574436030633762444:2078] VirtualId# [6:7574436030633762433:0]
2025-11-19T14:00:16.380938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436030633762444:2078] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.380991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:16.381016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:16.381139Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T14:00:16.381206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436032623898335:2060] [node 4] ICS01 socket: 262 reason# EndOfStream
2025-11-19T14:00:16.381211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436032623898255:2060] [node 4] ICP30 unregister session Session# [5:7574436032623898335:2060] VirtualId# [5:7574436032623898327:0]
2025-11-19T14:00:16.381216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436032623898335:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.381240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:16.381257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:16.481902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436030253416778:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:16.481923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436030253416779:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:16.559719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436030719743142:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:16.560075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436030719743143:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:16.563864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436029991319310:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:16.563878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436029991319311:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:16.595907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436030435323281:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:16.595935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436030435323282:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:16.595942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436030435323283:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:16.628537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436032209050903:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:16.628558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436032209050904:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:16.628564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436032209050905:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:16.663022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436029479062758:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:16.663040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436029479062759:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:16.663045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436029479062760:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:16.707642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436030435323342:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:16.707999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436032209050959:2101] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.712570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436029991319370:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51551
2025-11-19T14:00:16.712399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032209050966:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44382
2025-11-19T14:00:16.711399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436030253416844:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:16.711536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436030253416845:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:16.711613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436030253416846:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.716019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436029479062788:2068] [node 5] ICP01 ready to work
2025-11-19T14:00:16.721064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436029991319371:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51551
2025-11-19T14:00:16.724937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436029479062782:2061] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.736528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436029991319373:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:16.732750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436030719743202:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28720
2025-11-19T14:00:16.732818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436030719743203:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28720
2025-11-19T14:00:16.737591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436030719743206:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:16.737664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436030719743207:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.739160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436029991319380:2111] [node 4] ICP01 ready to work
2025-11-19T14:00:16.739194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436029991319378:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51551
2025-11-19T14:00:16.739241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436029991319379:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51551
2025-11-19T14:00:16.734552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436029479062828:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30491
2025-11-19T14:00:16.745696Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436029479062782:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(20576498ns) {Configure:104(20574444ns) {ProcessConfigured:135(20181440ns) {StartInitialHandshake:157(20180711ns) {DropHandshakes:469(1289ns) {DropIncomingHandshake:441(200ns) DropOutgoingHandshake:456(201ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:476(9956ns) {SwitchToState:178(750ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.746663Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436030253416778:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(38613298ns) {Configure:104(38612717ns) {ProcessConfigured:135(38156650ns) {StartInitialHandshake:157(38155854ns) {DropHandshakes:469(644ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(86ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:476(1952ns) {SwitchToState:178(1229ns)}}}}}
2025-11-19T14:00:16.748655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436030253416857:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49169
2025-11-19T14:00:16.750991Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030719743203:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.751219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436030719743220:2115] [node 6] ICP01 ready to work
2025-11-19T14:00:16.752013Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030253416857:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.752184Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436029479062828:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.752213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436029479062758:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.755762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436029991319387:2115] [node 6] ICP01 ready to work
2025-11-19T14:00:16.755850Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436029991319378:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.755925Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436029991319371:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.756617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436029991319310:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.754266Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436032209050959:2101] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(45902579ns) {Configure:104(45901951ns) {ProcessConfigured:135(45671521ns) {StartInitialHandshake:157(45670179ns) {DropHandshakes:469(171ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(15ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:476(1647ns) {SwitchToState:178(1357ns)}}}}}
2025-11-19T14:00:16.754485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436032209050976:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.756854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436029479062838:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30491
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.761829Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436029991319370:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.761506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030435323348:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16230
2025-11-19T14:00:16.761530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030435323349:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16230
2025-11-19T14:00:16.763735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436030435323357:2114] [node 6] ICP01 ready to work
2025-11-19T14:00:16.766888Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030253416850:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.766913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436030253416850:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.767127Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030719743202:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.772443Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436032209050966:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.772603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436032209050976:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.772613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032209050982:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44382
2025-11-19T14:00:16.772632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032209050983:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44382
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.774265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436029991319395:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51551
2025-11-19T14:00:16.772760Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436030435323281:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(22357436ns) {Configure:104(22356175ns) {ProcessConfigured:135(21939831ns) {StartInitialHandshake:157(21938806ns) {DropHandshakes:469(1513ns) {DropIncomingHandshake:441(200ns) DropOutgoingHandshake:456(243ns)} GenerateSessionVirtualId:988(317ns) PrepareNewSessionHandshake:476(1177ns) {SwitchToState:178(305ns)}}}}}
2025-11-19T14:00:16.774229Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436030435323349:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.774293Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436030435323344:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.774298Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436030435323344:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.774883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436030435323283:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.776252Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436029991319395:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.780160Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030253416843:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.780180Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436030253416843:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.778124Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436029479062838:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.778186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436029479062788:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.782630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436029479062840:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30491
2025-11-19T14:00:16.784176Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436030435323344:2108] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:16.783953Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436032209050983:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.783999Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436032209050975:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.784004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436032209050975:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.784707Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436032209050982:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.784854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436032209050905:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.784866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436032209050903:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.785188Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436032209050975:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:16.785307Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436030253416850:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:16.787789Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436029991319382:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.787812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436029991319382:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.791872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436032209050959:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.791897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:16.791910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436032209050959:2101] [node 6] ICP22 created new session: [5:7574436032209050989:2101]
2025-11-19T14:00:16.791918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436032209050989:2101] [node 6] ICS09 handshake done sender: [5:7574436032209050975:2113] self: [5:7574436032209050961:0] peer: [6:7574436029479062839:0] socket: 278 qp: -1
2025-11-19T14:00:16.791929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436032209050989:2101] [node 6] ICS10 traffic start
2025-11-19T14:00:16.791943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:16.792305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032209050988:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44382
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.792456Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436030253416843:2110] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:16.793685Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436032209050966:2104] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:16.793963Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030253416848:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.793974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436030253416848:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.793095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436030719743222:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28720
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.799509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436030435323342:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.799540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:16.799553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436030435323342:2107] [node 5] ICP22 created new session: [4:7574436030435323363:2107]
2025-11-19T14:00:16.799561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436030435323363:2107] [node 5] ICS09 handshake done sender: [4:7574436030435323344:2108] self: [4:7574436030435323343:0] peer: [5:7574436032209050984:0] socket: 252 qp: -1
2025-11-19T14:00:16.799572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436030435323363:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:16.799585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:16.800202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030435323365:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16230
2025-11-19T14:00:16.802268Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436030719743220:2115] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(50974239ns) {Configure:104(50973442ns) {ProcessConfigured:135(50658564ns) {StartInitialHandshake:157(50657873ns) {DropHandshakes:469(392ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:476(1627ns) {SwitchToState:178(1328ns)}}}}}
2025-11-19T14:00:16.802581Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436029479062828:2107] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:16.804738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436030253416845:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.804763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:16.804769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436030253416845:2108] [node 6] ICP22 created new session: [1:7574436030253416863:2108]
2025-11-19T14:00:16.804774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436030253416863:2108] [node 6] ICS09 handshake done sender: [1:7574436030253416850:2112] self: [1:7574436030253416849:0] peer: [6:7574436029479062837:0] socket: 259 qp: -1
2025-11-19T14:00:16.804779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436030253416863:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:16.804786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:16.804996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436030253416779:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.805002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:16.805005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436030253416779:2049] [node 3] ICP22 created new session: [1:7574436030253416865:2049]
2025-11-19T14:00:16.805009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436030253416865:2049] [node 3] ICS09 handshake done sender: [1:7574436030253416843:2110] self: [1:7574436030253416842:0] peer: [3:7574436029991319393:0] socket: 257 qp: -1
2025-11-19T14:00:16.805012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436030253416865:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:16.805015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:16.804114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436032209050976:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.804138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:16.804145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436032209050976:2108] [node 4] ICP22 created new session: [5:7574436032209050992:2108]
2025-11-19T14:00:16.804152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436032209050992:2108] [node 4] ICS09 handshake done sender: [5:7574436032209050966:2104] self: [5:7574436032209050984:0] peer: [4:7574436030435323343:0] socket: 255 qp: -1
2025-11-19T14:00:16.804160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436032209050992:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:16.804171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.805515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436029479062758:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.805532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:16.805537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436029479062758:2048] [node 1] ICP22 created new session: [6:7574436029479062843:2048]
2025-11-19T14:00:16.805542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436029479062843:2048] [node 1] ICS09 handshake done sender: [6:7574436029479062828:2107] self: [6:7574436029479062837:0] peer: [1:7574436030253416849:0] socket: 265 qp: -1
2025-11-19T14:00:16.805546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436029479062843:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:16.805567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:16.805859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436029479062841:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30491
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.806039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436029479062842:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30491
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.807260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436030253416861:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49169
2025-11-19T14:00:16.807307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436030253416862:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49169
2025-11-19T14:00:16.807548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436030253416867:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49169
2025-11-19T14:00:16.807978Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436030435323348:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.808267Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030719743222:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.808383Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030253416861:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.811675Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436029479062842:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.811775Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436029479062838:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:16.818413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436029479062788:2068] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.818434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:16.818442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436029479062788:2068] [node 5] ICP22 created new session: [6:7574436029479062846:2068]
2025-11-19T14:00:16.818450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436029479062846:2068] [node 5] ICS09 handshake done sender: [6:7574436029479062838:2116] self: [6:7574436029479062839:0] peer: [5:7574436032209050961:0] socket: 280 qp: -1
2025-11-19T14:00:16.818459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436029479062846:2068] [node 5] ICS10 traffic start
2025-11-19T14:00:16.818469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.820266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436030719743227:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28720
2025-11-19T14:00:16.820614Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436029991319375:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.820628Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436029991319375:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.821590Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436029991319382:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:16.822079Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436030253416848:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:16.822255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436029479062782:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.822761Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030253416862:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.822788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436030253416779:2049] [node 3] ICP18 handshake (actor [1:7574436030253416862:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.824187Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436030435323359:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.824201Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436030435323359:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.824852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436030435323282:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.824889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030435323367:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16230
2025-11-19T14:00:16.824931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030435323368:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16230
2025-11-19T14:00:16.825140Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030719743227:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.825176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436030719743142:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.825280Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030253416856:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.825287Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436030253416856:2114] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.826463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032209050995:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44382
2025-11-19T14:00:16.826428Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436030435323367:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.826455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030435323369:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16230
2025-11-19T14:00:16.826494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436030435323281:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.826597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032209050996:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44382
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.826814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032209050997:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44382
2025-11-19T14:00:16.826821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436029479062850:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30491
2025-11-19T14:00:16.826976Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436030253416856:2114] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:16.826975Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436030435323359:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:16.827088Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030719743213:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.827095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436030719743213:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.827161Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436029479062842:2119] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:16.827229Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030253416867:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.827254Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436030253416852:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.827256Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436030253416852:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.827328Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436032209050982:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:16.827381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436030253416844:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.827395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:16.827402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436030253416844:2107] [node 5] ICP22 created new session: [1:7574436030253416873:2107]
2025-11-19T14:00:16.827409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436030253416873:2107] [node 5] ICS09 handshake done sender: [1:7574436030253416848:2111] self: [1:7574436030253416847:0] peer: [5:7574436032209050987:0] socket: 258 qp: -1
2025-11-19T14:00:16.827415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436030253416873:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:16.827424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:16.827537Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436029991319375:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.827698Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436029991319379:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.827762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436030719743229:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28720
2025-11-19T14:00:16.827894Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436030719743227:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:16.828354Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436030719743213:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:16.828276Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436030435323369:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.828352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436030435323357:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.828363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:16.828371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436030435323357:2114] [node 6] ICP22 created new session: [4:7574436030435323374:2114]
2025-11-19T14:00:16.828376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436030435323374:2114] [node 6] ICS09 handshake done sender: [4:7574436030435323359:2115] self: [4:7574436030435323358:0] peer: [6:7574436029479062848:0] socket: 298 qp: -1
2025-11-19T14:00:16.828382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436030435323374:2114] [node 6] ICS10 traffic start
2025-11-19T14:00:16.828390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:16.828630Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436030435323349:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:16.828439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436030719743142:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.828454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:16.828461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436030719743142:2048] [node 1] ICP22 created new session: [2:7574436030719743231:2048]
2025-11-19T14:00:16.828467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436030719743231:2048] [node 1] ICS09 handshake done sender: [2:7574436030719743227:2118] self: [2:7574436030719743228:0] peer: [1:7574436030253416838:0] socket: 302 qp: -1
2025-11-19T14:00:16.828473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436030719743231:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:16.828479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:16.828679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436030719743206:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.828689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:16.828694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436030719743206:2109] [node 4] ICP22 created new session: [2:7574436030719743233:2109]
2025-11-19T14:00:16.828698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436030719743233:2109] [node 4] ICS09 handshake done sender: [2:7574436030719743213:2113] self: [2:7574436030719743212:0] peer: [4:7574436030435323370:0] socket: 271 qp: -1
2025-11-19T14:00:16.828702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436030719743233:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:16.828707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:16.829003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436032209050903:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.829023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:16.829030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436032209050903:2048] [node 1] ICP22 created new session: [5:7574436032209050998:2048]
2025-11-19T14:00:16.829036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436032209050998:2048] [node 1] ICS09 handshake done sender: [5:7574436032209050982:2116] self: [5:7574436032209050987:0] peer: [1:7574436030253416847:0] socket: 277 qp: -1
2025-11-19T14:00:16.829042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436032209050998:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:16.829050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.829865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436030435323283:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.829878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:16.829881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436030435323283:2050] [node 3] ICP22 created new session: [4:7574436030435323376:2050]
2025-11-19T14:00:16.829886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436030435323376:2050] [node 3] ICS09 handshake done sender: [4:7574436030435323349:2111] self: [4:7574436030435323362:0] peer: [3:7574436029991319381:0] socket: 281 qp: -1
2025-11-19T14:00:16.829890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436030435323376:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:16.829897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:16.830194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574436030435323357:2114] [node 6] ICP18 handshake (actor [4:7574436030435323369:2120]) is too old
2025-11-19T14:00:16.830253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030435323373:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16230
2025-11-19T14:00:16.829425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436029479062782:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.829441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:16.829446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436029479062782:2061] [node 4] ICP22 created new session: [6:7574436029479062851:2061]
2025-11-19T14:00:16.829450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436029479062851:2061] [node 4] ICS09 handshake done sender: [6:7574436029479062842:2119] self: [6:7574436029479062848:0] peer: [4:7574436030435323358:0] socket: 299 qp: -1
2025-11-19T14:00:16.829455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436029479062851:2061] [node 4] ICS10 traffic start
2025-11-19T14:00:16.829461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:16.836975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436030253416778:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.837003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:16.837011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436030253416778:2048] [node 2] ICP22 created new session: [1:7574436030253416875:2048]
2025-11-19T14:00:16.837018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436030253416875:2048] [node 2] ICS09 handshake done sender: [1:7574436030253416856:2114] self: [1:7574436030253416838:0] peer: [2:7574436030719743228:0] socket: 301 qp: -1
2025-11-19T14:00:16.837025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436030253416875:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:16.837036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:16.838468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436029991319373:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.838491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:16.838498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436029991319373:2109] [node 5] ICP22 created new session: [3:7574436029991319401:2109]
2025-11-19T14:00:16.838504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436029991319401:2109] [node 5] ICS09 handshake done sender: [3:7574436029991319375:2110] self: [3:7574436029991319374:0] peer: [5:7574436032209050986:0] socket: 268 qp: -1
2025-11-19T14:00:16.838510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436029991319401:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:16.838517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:16.838746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436029991319310:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4544) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436029991319371:2108] held: no
2025-11-19T14:00:16.838822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436029991319380:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.838830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:16.838836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436029991319380:2111] [node 4] ICP22 created new session: [3:7574436029991319403:2111]
2025-11-19T14:00:16.838841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436029991319403:2111] [node 4] ICS09 handshake done sender: [3:7574436029991319382:2114] self: [3:7574436029991319381:0] peer: [4:7574436030435323362:0] socket: 274 qp: -1
2025-11-19T14:00:16.838855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436029991319403:2111] [node 4] ICS10 traffic start
2025-11-19T14:00:16.838859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:16.839636Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436032209050983:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:16.840204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436030253416844:2107] [node 5] ICP18 handshake (actor [1:7574436030253416867:2118]) is too old
2025-11-19T14:00:16.841290Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436030253416852:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:16.841546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436030253416846:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.841559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:16.841565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436030253416846:2109] [node 4] ICP22 created new session: [1:7574436030253416878:2109]
2025-11-19T14:00:16.841570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436030253416878:2109] [node 4] ICS09 handshake done sender: [1:7574436030253416852:2113] self: [1:7574436030253416851:0] peer: [4:7574436030435323372:0] socket: 300 qp: -1
2025-11-19T14:00:16.841577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436030253416878:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:16.841583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.842705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436029479062853:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30491
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.845225Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436032209050905:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24521) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436032209050983:2117] held: no
2025-11-19T14:00:16.845626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032209051000:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44382
2025-11-19T14:00:16.845799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436032209050905:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.845812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:16.845817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436032209050905:2050] [node 3] ICP22 created new session: [5:7574436032209051001:2050]
2025-11-19T14:00:16.845823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436032209051001:2050] [node 3] ICS09 handshake done sender: [5:7574436032209050983:2117] self: [5:7574436032209050986:0] peer: [3:7574436029991319374:0] socket: 285 qp: -1
2025-11-19T14:00:16.845829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436032209051001:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:16.845836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.846293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436030253416880:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49169
2025-11-19T14:00:16.846493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436030435323282:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12400) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436030435323348:2110] held: no
2025-11-19T14:00:16.846828Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436030435323348:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.847052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436030435323282:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.847064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:16.847068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436030435323282:2049] [node 2] ICP22 created new session: [4:7574436030435323380:2049]
2025-11-19T14:00:16.847072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436030435323380:2049] [node 2] ICS09 handshake done sender: [4:7574436030435323348:2110] self: [4:7574436030435323370:0] peer: [2:7574436030719743212:0] socket: 272 qp: -1
2025-11-19T14:00:16.847076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436030435323380:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:16.847081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:16.847263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436030435323379:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16230
2025-11-19T14:00:16.847816Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436030435323367:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:16.848039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436030435323281:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.848054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:16.848065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436030435323281:2048] [node 1] ICP22 created new session: [4:7574436030435323382:2048]
2025-11-19T14:00:16.848071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436030435323382:2048] [node 1] ICS09 handshake done sender: [4:7574436030435323367:2118] self: [4:7574436030435323372:0] peer: [1:7574436030253416851:0] socket: 303 qp: -1
2025-11-19T14:00:16.848076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436030435323382:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:16.848083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:16.847937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436029991319400:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51551
2025-11-19T14:00:16.850485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436032209051000:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.850525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436032209050904:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.850788Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030719743215:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.850799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436030719743215:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.851454Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436029479062853:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.851500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436029479062759:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.851683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436032209051004:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44382
2025-11-19T14:00:16.851811Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436030719743215:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:16.852043Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436032209051000:2122] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:16.852185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436029991319311:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.852444Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436029991319371:2108] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:16.852325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436032209050904:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.852341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:16.852347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436032209050904:2049] [node 2] ICP22 created new session: [5:7574436032209051005:2049]
2025-11-19T14:00:16.852352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436032209051005:2049] [node 2] ICS09 handshake done sender: [5:7574436032209051000:2122] self: [5:7574436032209051003:0] peer: [2:7574436030719743214:0] socket: 256 qp: -1
2025-11-19T14:00:16.852358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436032209051005:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:16.852367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.853774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436029479062855:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30491
2025-11-19T14:00:16.853918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030719743211:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.853928Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436030719743211:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.854568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436030719743207:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.854581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:16.854587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436030719743207:2110] [node 5] ICP22 created new session: [2:7574436030719743239:2110]
2025-11-19T14:00:16.854592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436030719743239:2110] [node 5] ICS09 handshake done sender: [2:7574436030719743215:2114] self: [2:7574436030719743214:0] peer: [5:7574436032209051003:0] socket: 295 qp: -1
2025-11-19T14:00:16.854598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436030719743239:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:16.854605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:16.855639Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436029479062855:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.855676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436029479062760:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:16.855181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436029991319310:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.855199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:16.855209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436029991319310:2048] [node 1] ICP22 created new session: [3:7574436029991319408:2048]
2025-11-19T14:00:16.855215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436029991319408:2048] [node 1] ICS09 handshake done sender: [3:7574436029991319371:2108] self: [3:7574436029991319393:0] peer: [1:7574436030253416842:0] socket: 260 qp: -1
2025-11-19T14:00:16.855223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436029991319408:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:16.855231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:16.856296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436029991319391:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.856303Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436029991319391:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:16.856346Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030719743224:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.856355Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436030719743224:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.861727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436030719743238:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28720
2025-11-19T14:00:16.862000Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436030719743211:2112] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:16.862135Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436030719743224:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:16.862205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436030719743143:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.862221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:16.862230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436030719743143:2049] [node 3] ICP22 created new session: [2:7574436030719743243:2049]
2025-11-19T14:00:16.862235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436030719743243:2049] [node 3] ICS09 handshake done sender: [2:7574436030719743211:2112] self: [2:7574436030719743210:0] peer: [3:7574436029991319407:0] socket: 270 qp: -1
2025-11-19T14:00:16.862242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436030719743243:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:16.862252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:16.862455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436030719743220:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.862467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:16.862471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436030719743220:2115] [node 6] ICP22 created new session: [2:7574436030719743245:2115]
2025-11-19T14:00:16.862474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436030719743245:2115] [node 6] ICS09 handshake done sender: [2:7574436030719743224:2117] self: [2:7574436030719743221:0] peer: [6:7574436029479062854:0] socket: 275 qp: -1
2025-11-19T14:00:16.862478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436030719743245:2115] [node 6] ICS10 traffic start
2025-11-19T14:00:16.862483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:16.863278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574436030253416846:2109] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:16.868643Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436029479062759:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12400) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436029479062853:2121] held: no
2025-11-19T14:00:16.868822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436029479062857:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30491
2025-11-19T14:00:16.868837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436029479062858:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30491
2025-11-19T14:00:16.868936Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436029991319391:2118] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:16.869078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436029991319411:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51551
2025-11-19T14:00:16.869272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436029991319387:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:16.869295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:16.869301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436029991319387:2115] [node 6] ICP22 created new session: [3:7574436029991319412:2115]
2025-11-19T14:00:16.869307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436029991319412:2115] [node 6] ICS09 handshake done sender: [3:7574436029991319391:2118] self: [3:7574436029991319390:0] peer: [6:7574436029479062856:0] socket: 308 qp: -1
2025-11-19T14:00:16.869314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436029991319412:2115] [node 6] ICS10 traffic start
2025-11-19T14:00:16.869325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:16.869816Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436029991319379:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:16.870008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436029991319311:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.870021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:16.870026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436029991319311:2049] [node 2] ICP22 created new session: [3:7574436029991319414:2049]
2025-11-19T14:00:16.870031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436029991319414:2049] [node 2] ICS09 handshake done sender: [3:7574436029991319379:2113] self: [3:7574436029991319407:0] peer: [2:7574436030719743210:0] socket: 273 qp: -1
2025-11-19T14:00:16.870036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436029991319414:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:16.870042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:16.871392Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436029479062853:2121] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:16.871481Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436029479062855:2122] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:16.871933Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436030719743238:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:16.872336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436030719743143:2049] [node 3] ICP18 handshake (actor [2:7574436030719743238:2120]) is too old
2025-11-19T14:00:16.871742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436029479062759:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.871768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:16.871774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436029479062759:2049] [node 2] ICP22 created new session: [6:7574436029479062859:2049]
2025-11-19T14:00:16.871779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436029479062859:2049] [node 2] ICS09 handshake done sender: [6:7574436029479062853:2121] self: [6:7574436029479062854:0] peer: [2:7574436030719743221:0] socket: 276 qp: -1
2025-11-19T14:00:16.871785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436029479062859:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:16.871794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:16.872888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436029479062760:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:16.872903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:16.872909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436029479062760:2050] [node 3] ICP22 created new session: [6:7574436029479062861:2050]
2025-11-19T14:00:16.872914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436029479062861:2050] [node 3] ICS09 handshake done sender: [6:7574436029479062855:2122] self: [6:7574436029479062856:0] peer: [3:7574436029991319390:0] socket: 284 qp: -1
2025-11-19T14:00:16.872921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436029479062861:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:16.872929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:16.873070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574436029479062760:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:16.881629Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:16.881739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436030719743231:2048] [node 1] ICS01 socket: 302 reason# EndOfStream
2025-11-19T14:00:16.881749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436030719743142:2048] [node 1] ICP30 unregister session Session# [2:7574436030719743231:2048] VirtualId# [2:7574436030719743228:0]
2025-11-19T14:00:16.881757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436030719743231:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.881793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:16.881814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:16.887919Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:16.888021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436029991319408:2048] [node 1] ICS01 socket: 260 reason# EndOfStream
2025-11-19T14:00:16.888035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436029991319310:2048] [node 1] ICP30 unregister session Session# [3:7574436029991319408:2048] VirtualId# [3:7574436029991319393:0]
2025-11-19T14:00:16.888052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436029991319408:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.888106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:16.888138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:16.890834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:16.890931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436030435323382:2048] [node 1] ICS01 socket: 303 reason# EndOfStream
2025-11-19T14:00:16.890942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436030435323281:2048] [node 1] ICP30 unregister session Session# [4:7574436030435323382:2048] VirtualId# [4:7574436030435323372:0]
2025-11-19T14:00:16.890948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436030435323382:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.891006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:16.891036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:16.891887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:16.891971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436029479062843:2048] [node 1] ICS01 socket: 265 reason# EndOfStream
2025-11-19T14:00:16.891979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436029479062758:2048] [node 1] ICP30 unregister session Session# [6:7574436029479062843:2048] VirtualId# [6:7574436029479062837:0]
2025-11-19T14:00:16.891987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436029479062843:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.892032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:16.892058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:16.895342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:16.895423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436032209050998:2048] [node 1] ICS01 socket: 277 reason# EndOfStream
2025-11-19T14:00:16.895430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436032209050903:2048] [node 1] ICP30 unregister session Session# [5:7574436032209050998:2048] VirtualId# [5:7574436032209050987:0]
2025-11-19T14:00:16.895436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436032209050998:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.895492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:16.895515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:16.905804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:16.905921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436029991319414:2049] [node 2] ICS01 socket: 273 reason# EndOfStream
2025-11-19T14:00:16.905932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436029991319311:2049] [node 2] ICP30 unregister session Session# [3:7574436029991319414:2049] VirtualId# [3:7574436029991319407:0]
2025-11-19T14:00:16.905941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436029991319414:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.905992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:16.906015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:16.928456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:16.928573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436030435323380:2049] [node 2] ICS01 socket: 272 reason# EndOfStream
2025-11-19T14:00:16.928584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436030435323282:2049] [node 2] ICP30 unregister session Session# [4:7574436030435323380:2049] VirtualId# [4:7574436030435323370:0]
2025-11-19T14:00:16.928592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436030435323380:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.928625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:16.928646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:16.934102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:16.934220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436029479062859:2049] [node 2] ICS01 socket: 276 reason# EndOfStream
2025-11-19T14:00:16.934234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436029479062759:2049] [node 2] ICP30 unregister session Session# [6:7574436029479062859:2049] VirtualId# [6:7574436029479062854:0]
2025-11-19T14:00:16.934243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436029479062859:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.934298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:16.934324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:16.934600Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:16.934667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436032209051005:2049] [node 2] ICS01 socket: 256 reason# EndOfStream
2025-11-19T14:00:16.934671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436032209050904:2049] [node 2] ICP30 unregister session Session# [5:7574436032209051005:2049] VirtualId# [5:7574436032209051003:0]
2025-11-19T14:00:16.934676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436032209051005:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.934706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:16.934730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:16.954643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:16.954756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436032209051001:2050] [node 3] ICS01 socket: 285 reason# EndOfStream
2025-11-19T14:00:16.954768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436032209050905:2050] [node 3] ICP30 unregister session Session# [5:7574436032209051001:2050] VirtualId# [5:7574436032209050986:0]
2025-11-19T14:00:16.954777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436032209051001:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.954825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:16.954850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:16.954892Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:16.954959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436029479062861:2050] [node 3] ICS01 socket: 284 reason# EndOfStream
2025-11-19T14:00:16.954964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436029479062760:2050] [node 3] ICP30 unregister session Session# [6:7574436029479062861:2050] VirtualId# [6:7574436029479062856:0]
2025-11-19T14:00:16.954969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436029479062861:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:16.954996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:16.955018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:17.000976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T14:00:17.001102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436032209050992:2108] [node 4] ICS01 socket: 255 reason# EndOfStream
2025-11-19T14:00:17.001117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436032209050976:2108] [node 4] ICP30 unregister session Session# [5:7574436032209050992:2108] VirtualId# [5:7574436032209050984:0]
2025-11-19T14:00:17.001126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436032209050992:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.001173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:17.001205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:17.007551Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:17.007685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436029479062851:2061] [node 4] ICS01 socket: 299 reason# EndOfStream
2025-11-19T14:00:17.007698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436029479062782:2061] [node 4] ICP30 unregister session Session# [6:7574436029479062851:2061] VirtualId# [6:7574436029479062848:0]
2025-11-19T14:00:17.007708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436029479062851:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.007786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:17.007815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:17.155921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436035371636316:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:17.155944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436035371636317:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:17.224707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436036629905847:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:17.224723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436036629905848:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:17.285572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436034675525469:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:17.285608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436034675525470:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:17.366599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436034620160018:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:17.366614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436034620160019:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:17.366619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436034620160020:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:17.432999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436037295459083:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:17.433020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436037295459084:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:17.433025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436037295459085:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:17.483962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436036315422756:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:17.483962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436036315422755:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:17.483975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436036315422757:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.542623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436037295459104:2068] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56389
2025-11-19T14:00:17.541539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436034620160043:2071] [node 5] ICP01 ready to work
2025-11-19T14:00:17.543000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436034620160050:2075] [node 6] ICP01 ready to work
2025-11-19T14:00:17.542325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436035371636378:2106] [node 5] ICP01 ready to work
2025-11-19T14:00:17.542436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436035371636379:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:17.541390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436036315422772:2063] [node 4] ICP01 ready to work
2025-11-19T14:00:17.549504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436037295459107:2070] [node 4] ICP01 ready to work
2025-11-19T14:00:17.549067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436036629905908:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:17.551808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436036315422786:2072] [node 5] ICP01 ready to work
2025-11-19T14:00:17.554087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436034675525530:2104] [node 4] ICP01 ready to work
2025-11-19T14:00:17.554205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436034675525531:2106] [node 6] ICP01 ready to work
2025-11-19T14:00:17.554274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436034675525532:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.567973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436036315422809:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4405
2025-11-19T14:00:17.576214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436035371636392:2113] [node 6] ICP01 ready to work
2025-11-19T14:00:17.577202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436036629905920:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.579409Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436034620160050:2075] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(36299793ns) {Configure:104(36299271ns) {ProcessConfigured:135(36018460ns) {StartInitialHandshake:157(36017529ns) {DropHandshakes:469(475ns) {DropIncomingHandshake:441(78ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(150ns) PrepareNewSessionHandshake:476(1868ns) {SwitchToState:178(961ns)}}}}}
2025-11-19T14:00:17.580097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436034620160088:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45873
2025-11-19T14:00:17.580176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436034620160089:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45873
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.583458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436036629905918:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44138
2025-11-19T14:00:17.583496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436036629905919:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44138
2025-11-19T14:00:17.584158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436034675525547:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57686
2025-11-19T14:00:17.584184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436034675525548:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57686
2025-11-19T14:00:17.584199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436034675525549:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57686
2025-11-19T14:00:17.588085Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436034620160088:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.588262Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436034675525549:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.588350Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436034620160089:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.588334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436037295459107:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(38751891ns) {Configure:104(38750886ns) {ProcessConfigured:135(38356121ns) {StartInitialHandshake:157(38355419ns) {DropHandshakes:469(504ns) {DropIncomingHandshake:441(82ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(1748ns) {SwitchToState:178(854ns)}}}}}
2025-11-19T14:00:17.588442Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436036629905919:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.588729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436036629905917:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:17.589499Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436034675525548:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.589526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436034675525470:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:17.589743Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436034675525547:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.589221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436037295459139:2076] [node 6] ICP01 ready to work
2025-11-19T14:00:17.587844Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436036315422809:2093] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.594487Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436034620160019:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(44220255ns) {Configure:104(44218826ns) {ProcessConfigured:135(43924319ns) {StartInitialHandshake:157(43923709ns) {DropHandshakes:469(634ns) {DropIncomingHandshake:441(91ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(1505ns) {SwitchToState:178(679ns)}}}}}
2025-11-19T14:00:17.594855Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436034675525534:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.594865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436034675525534:2108] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.595560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436034620160100:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45873
2025-11-19T14:00:17.595655Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436034675525534:2108] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:17.595943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436034675525530:2104] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.595965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:17.595971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436034675525530:2104] [node 4] ICP22 created new session: [3:7574436034675525553:2104]
2025-11-19T14:00:17.595978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436034675525553:2104] [node 4] ICS09 handshake done sender: [3:7574436034675525534:2108] self: [3:7574436034675525533:0] peer: [4:7574436034620160098:0] socket: 270 qp: -1
2025-11-19T14:00:17.595985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436034675525553:2104] [node 4] ICS10 traffic start
2025-11-19T14:00:17.595995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.596802Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436037295459104:2068] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.597197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436034675525469:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(29240311ns) {Configure:104(29239203ns) {ProcessConfigured:135(28865815ns) {StartInitialHandshake:157(28864901ns) {DropHandshakes:469(1635ns) {DropIncomingHandshake:441(307ns) DropOutgoingHandshake:456(204ns)} GenerateSessionVirtualId:988(348ns) PrepareNewSessionHandshake:476(579ns) {SwitchToState:178(381ns)}}}}}
2025-11-19T14:00:17.597226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436034675525469:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.597510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436037295459153:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56389
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.598035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436037295459155:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56389
2025-11-19T14:00:17.598095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436037295459156:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56389
2025-11-19T14:00:17.598158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436037295459107:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.598613Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436036629905910:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.598624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436036629905910:2108] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:17.599020Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436036629905918:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.599237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436036629905928:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44138
2025-11-19T14:00:17.599365Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436037295459153:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.600577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436034675525558:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57686
2025-11-19T14:00:17.600595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436034675525559:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57686
2025-11-19T14:00:17.601284Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436036315422755:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(49405729ns) {Configure:104(49405302ns) {ProcessConfigured:135(49077583ns) {StartInitialHandshake:157(49077463ns) {DropHandshakes:469(451ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:476(652ns) {SwitchToState:178(264ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.601582Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436035371636392:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25191691ns) {Configure:104(25190165ns) {ProcessConfigured:135(24808426ns) {StartInitialHandshake:157(24807962ns) {DropHandshakes:469(1442ns) {DropIncomingHandshake:441(252ns) DropOutgoingHandshake:456(202ns)} GenerateSessionVirtualId:988(290ns) PrepareNewSessionHandshake:476(1212ns) {SwitchToState:178(28ns)}}}}}
2025-11-19T14:00:17.601625Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436035371636383:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.601629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436035371636383:2109] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:17.601148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436036629905847:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23794929ns) {Configure:104(23794442ns) {ProcessConfigured:135(23359547ns) {StartInitialHandshake:157(23358666ns) {DropHandshakes:469(535ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:476(557ns) {SwitchToState:178(347ns)}}}}}
2025-11-19T14:00:17.601172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436036629905847:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:17.601835Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436036629905910:2108] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:17.600977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436037295459084:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:17.601011Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436037295459155:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.601056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436037295459085:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:17.602181Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436037295459156:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.602651Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436034675525538:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.602659Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436034675525538:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:17.603147Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436034675525536:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.603150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436034675525536:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:17.603846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436036629905848:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.603864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:17.603871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436036629905848:2049] [node 3] ICP22 created new session: [2:7574436036629905932:2049]
2025-11-19T14:00:17.603878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436036629905932:2049] [node 3] ICS09 handshake done sender: [2:7574436036629905910:2108] self: [2:7574436036629905909:0] peer: [3:7574436034675525551:0] socket: 263 qp: -1
2025-11-19T14:00:17.603884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436036629905932:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:17.603894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.601476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436036315422834:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4405
2025-11-19T14:00:17.603898Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436034620160045:2072] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.603912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436034620160045:2072] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:17.605041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436034620160101:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45873
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.607126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436034620160102:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45873
2025-11-19T14:00:17.607334Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436034620160045:2072] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:17.602193Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436035371636383:2109] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:17.602280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436035371636316:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.602295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:17.602312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436035371636316:2048] [node 2] ICP22 created new session: [1:7574436035371636396:2048]
2025-11-19T14:00:17.602319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436035371636396:2048] [node 2] ICS09 handshake done sender: [1:7574436035371636383:2109] self: [1:7574436035371636380:0] peer: [2:7574436036629905931:0] socket: 262 qp: -1
2025-11-19T14:00:17.602326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436035371636396:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:17.602334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:17.607934Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436036315422834:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.607977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436036315422772:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:17.607942Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436034675525548:2115] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:17.608149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436036629905912:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.608162Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436036629905912:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:17.608341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436036315422757:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:17.608396Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436035371636385:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.608409Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436035371636385:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:17.607447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436034620160043:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.607462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:17.607468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436034620160043:2071] [node 5] ICP22 created new session: [4:7574436034620160105:2071]
2025-11-19T14:00:17.607474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436034620160105:2071] [node 5] ICS09 handshake done sender: [4:7574436034620160045:2072] self: [4:7574436034620160044:0] peer: [5:7574436037295459163:0] socket: 259 qp: -1
2025-11-19T14:00:17.607495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436034620160105:2071] [node 5] ICS10 traffic start
2025-11-19T14:00:17.607502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:17.608145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436034620160020:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:17.608176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436034620160020:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(19788023ns) {IncomingHandshake:261(19787795ns) {DropIncomingHandshake:441(25ns) PrepareNewSessionHandshake:476(349ns) {SwitchToState:178(164ns)} GenerateSessionVirtualId:988(188ns) IssueIncomingHandshakeReply:193(19785038ns) {SwitchToState:178(459ns)}}}
2025-11-19T14:00:17.608211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436034620160020:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22241) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436034620160089:2112] held: no
2025-11-19T14:00:17.608310Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436034620160089:2112] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:17.608471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436037295459154:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56389
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.610021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436034675525470:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.610040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:17.610046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436034675525470:2049] [node 2] ICP22 created new session: [3:7574436034675525562:2049]
2025-11-19T14:00:17.610052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436034675525562:2049] [node 2] ICS09 handshake done sender: [3:7574436034675525548:2115] self: [3:7574436034675525551:0] peer: [2:7574436036629905909:0] socket: 280 qp: -1
2025-11-19T14:00:17.610057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436034675525562:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:17.610063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:17.609570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436035371636399:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44376
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.616264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436036629905934:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44138
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.617074Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436034675525538:2110] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:17.617236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436036629905935:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44138
2025-11-19T14:00:17.617679Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436037295459154:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.617717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436037295459083:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:17.622722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436036629905848:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:17.622788Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436036629905918:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:17.624624Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436034675525536:2109] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:17.622156Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436034620160092:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.622183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436034620160092:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.625473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436034675525532:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.625491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:17.625498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436034675525532:2107] [node 5] ICP22 created new session: [3:7574436034675525564:2107]
2025-11-19T14:00:17.625505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436034675525564:2107] [node 5] ICS09 handshake done sender: [3:7574436034675525538:2110] self: [3:7574436034675525537:0] peer: [5:7574436037295459168:0] socket: 272 qp: -1
2025-11-19T14:00:17.625511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436034675525564:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:17.625521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:17.625975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436034675525566:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57686
2025-11-19T14:00:17.626265Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436035371636385:2110] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:17.626422Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436036629905912:2109] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:17.626443Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436034675525558:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.626805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436035371636317:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.626819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:17.626826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436035371636317:2049] [node 3] ICP22 created new session: [1:7574436035371636402:2049]
2025-11-19T14:00:17.626831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436035371636402:2049] [node 3] ICS09 handshake done sender: [1:7574436035371636385:2110] self: [1:7574436035371636384:0] peer: [3:7574436034675525557:0] socket: 264 qp: -1
2025-11-19T14:00:17.626838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436035371636402:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:17.626843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:17.627079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436035371636401:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44376
2025-11-19T14:00:17.626528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436036629905847:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.626543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:17.626549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436036629905847:2048] [node 1] ICP22 created new session: [2:7574436036629905938:2048]
2025-11-19T14:00:17.626554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436036629905938:2048] [node 1] ICS09 handshake done sender: [2:7574436036629905918:2112] self: [2:7574436036629905931:0] peer: [1:7574436035371636380:0] socket: 266 qp: -1
2025-11-19T14:00:17.626560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436036629905938:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:17.626566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:17.626935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436036629905908:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.626942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:17.626946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436036629905908:2107] [node 5] ICP22 created new session: [2:7574436036629905940:2107]
2025-11-19T14:00:17.626950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436036629905940:2107] [node 5] ICS09 handshake done sender: [2:7574436036629905912:2109] self: [2:7574436036629905911:0] peer: [5:7574436037295459167:0] socket: 265 qp: -1
2025-11-19T14:00:17.626954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436036629905940:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:17.626959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:17.627403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436034675525531:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.627417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:17.627423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436034675525531:2106] [node 6] ICP22 created new session: [3:7574436034675525568:2106]
2025-11-19T14:00:17.627428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436034675525568:2106] [node 6] ICS09 handshake done sender: [3:7574436034675525536:2109] self: [3:7574436034675525535:0] peer: [6:7574436036315422833:0] socket: 271 qp: -1
2025-11-19T14:00:17.627434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436034675525568:2106] [node 6] ICS10 traffic start
2025-11-19T14:00:17.627451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:17.627698Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436034675525547:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:17.628735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574436034675525532:2107] [node 5] ICP18 handshake (actor [3:7574436034675525558:2118]) is too old
2025-11-19T14:00:17.629004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436034675525469:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.629017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:17.629024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436034675525469:2048] [node 1] ICP22 created new session: [3:7574436034675525570:2048]
2025-11-19T14:00:17.629028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436034675525570:2048] [node 1] ICS09 handshake done sender: [3:7574436034675525547:2114] self: [3:7574436034675525557:0] peer: [1:7574436035371636384:0] socket: 268 qp: -1
2025-11-19T14:00:17.629034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436034675525570:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:17.629041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:17.629892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436034620160020:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.629909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:17.629916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436034620160020:2050] [node 3] ICP22 created new session: [4:7574436034620160108:2050]
2025-11-19T14:00:17.629923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436034620160108:2050] [node 3] ICS09 handshake done sender: [4:7574436034620160089:2112] self: [4:7574436034620160098:0] peer: [3:7574436034675525533:0] socket: 276 qp: -1
2025-11-19T14:00:17.629929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436034620160108:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:17.629935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.631461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436034620160110:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45873
2025-11-19T14:00:17.632012Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436034620160102:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.632050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436034620160018:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:17.632358Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436034620160101:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.632384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574436034620160043:2071] [node 5] ICP18 handshake (actor [4:7574436034620160101:2118]) is too old
2025-11-19T14:00:17.632806Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436034620160110:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.632863Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436035371636399:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.632830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436034620160019:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:17.632924Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436036629905935:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.634690Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436035371636388:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.634710Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436035371636388:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.635461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436036315422837:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4405
2025-11-19T14:00:17.635541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436036315422838:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4405
2025-11-19T14:00:17.635572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436036315422839:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4405
2025-11-19T14:00:17.636049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436036315422840:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4405
2025-11-19T14:00:17.636686Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436036629905926:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.636700Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436036629905926:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.637972Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436035371636390:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.640284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436034620160113:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45873
2025-11-19T14:00:17.637988Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436035371636390:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:17.638510Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436035371636388:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:17.638681Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436036315422837:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.638707Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436036315422809:2093] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:17.639708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436036315422757:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.639728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:17.639736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436036315422757:2050] [node 3] ICP22 created new session: [6:7574436036315422841:2050]
2025-11-19T14:00:17.639755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436036315422841:2050] [node 3] ICS09 handshake done sender: [6:7574436036315422809:2093] self: [6:7574436036315422833:0] peer: [3:7574436034675525535:0] socket: 273 qp: -1
2025-11-19T14:00:17.639763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436036315422841:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:17.639771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:17.640058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436036315422756:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:17.640387Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436036629905923:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.640395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436036629905923:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:17.641326Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436036629905923:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:17.640580Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436036315422840:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.641425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436036629905920:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.641444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:17.641449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436036629905920:2111] [node 6] ICP22 created new session: [2:7574436036629905944:2111]
2025-11-19T14:00:17.641460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436036629905944:2111] [node 6] ICS09 handshake done sender: [2:7574436036629905923:2114] self: [2:7574436036629905922:0] peer: [6:7574436036315422843:0] socket: 282 qp: -1
2025-11-19T14:00:17.641467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436036629905944:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:17.641485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:17.641555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436036315422755:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:17.641680Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436036315422838:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.641727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436036315422786:2072] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:17.640832Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436035371636390:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:17.642128Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436035371636394:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.642136Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436035371636394:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:17.642681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436035371636379:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.642695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:17.642701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436035371636379:2108] [node 4] ICP22 created new session: [1:7574436035371636409:2108]
2025-11-19T14:00:17.642708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436035371636409:2108] [node 4] ICS09 handshake done sender: [1:7574436035371636390:2112] self: [1:7574436035371636382:0] peer: [4:7574436034620160111:0] socket: 298 qp: -1
2025-11-19T14:00:17.642713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436035371636409:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:17.642746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.643253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436035371636408:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44376
2025-11-19T14:00:17.643365Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436035371636394:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:17.643443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436035371636392:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.643456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:17.643461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436035371636392:2113] [node 6] ICP22 created new session: [1:7574436035371636412:2113]
2025-11-19T14:00:17.643467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436035371636412:2113] [node 6] ICS09 handshake done sender: [1:7574436035371636394:2114] self: [1:7574436035371636393:0] peer: [6:7574436036315422845:0] socket: 313 qp: -1
2025-11-19T14:00:17.643472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436035371636412:2113] [node 6] ICS10 traffic start
2025-11-19T14:00:17.643493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:17.643840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436035371636378:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.643851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:17.643856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436035371636378:2106] [node 5] ICP22 created new session: [1:7574436035371636414:2106]
2025-11-19T14:00:17.643862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436035371636414:2106] [node 5] ICS09 handshake done sender: [1:7574436035371636388:2111] self: [1:7574436035371636381:0] peer: [5:7574436037295459170:0] socket: 269 qp: -1
2025-11-19T14:00:17.643867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436035371636414:2106] [node 5] ICS10 traffic start
2025-11-19T14:00:17.643874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.644271Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436037295459107:2070] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12723) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436037295459104:2068] held: no
2025-11-19T14:00:17.644420Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436037295459160:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.644423Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436037295459160:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:17.645159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436037295459171:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56389
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.645464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436037295459172:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56389
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.645586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436037295459173:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56389
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.645706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436037295459174:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56389
2025-11-19T14:00:17.645971Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436037295459104:2068] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:17.646042Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436037295459153:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:17.646098Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436037295459154:2113] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:17.646194Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436037295459160:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:17.646269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436037295459107:2070] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.646280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:17.646285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436037295459107:2070] [node 4] ICP22 created new session: [5:7574436037295459177:2070]
2025-11-19T14:00:17.646290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436037295459177:2070] [node 4] ICS09 handshake done sender: [5:7574436037295459104:2068] self: [5:7574436037295459163:0] peer: [4:7574436034620160044:0] socket: 260 qp: -1
2025-11-19T14:00:17.646295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436037295459177:2070] [node 4] ICS10 traffic start
2025-11-19T14:00:17.646301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:17.646511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436037295459084:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.646519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:17.646522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436037295459084:2049] [node 2] ICP22 created new session: [5:7574436037295459179:2049]
2025-11-19T14:00:17.646527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436037295459179:2049] [node 2] ICS09 handshake done sender: [5:7574436037295459153:2112] self: [5:7574436037295459167:0] peer: [2:7574436036629905911:0] socket: 267 qp: -1
2025-11-19T14:00:17.646531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436037295459179:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:17.646536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:17.646795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436037295459083:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.646802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:17.646806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436037295459083:2048] [node 1] ICP22 created new session: [5:7574436037295459181:2048]
2025-11-19T14:00:17.646809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436037295459181:2048] [node 1] ICS09 handshake done sender: [5:7574436037295459154:2113] self: [5:7574436037295459170:0] peer: [1:7574436035371636381:0] socket: 286 qp: -1
2025-11-19T14:00:17.646813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436037295459181:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:17.646817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:17.647106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436037295459139:2076] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.647116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:17.647120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436037295459139:2076] [node 6] ICP22 created new session: [5:7574436037295459183:2076]
2025-11-19T14:00:17.647123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436037295459183:2076] [node 6] ICS09 handshake done sender: [5:7574436037295459160:2117] self: [5:7574436037295459159:0] peer: [6:7574436036315422846:0] socket: 292 qp: -1
2025-11-19T14:00:17.647126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436037295459183:2076] [node 6] ICS10 traffic start
2025-11-19T14:00:17.647132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:17.641905Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436034620160102:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.648175Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436037295459085:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22241) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436037295459155:2114] held: no
2025-11-19T14:00:17.650209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436034620160018:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22701) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436034620160102:2119] held: no
2025-11-19T14:00:17.650885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436034620160018:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.650898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:17.650905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436034620160018:2048] [node 1] ICP22 created new session: [4:7574436034620160115:2048]
2025-11-19T14:00:17.650910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436034620160115:2048] [node 1] ICS09 handshake done sender: [4:7574436034620160102:2119] self: [4:7574436034620160111:0] peer: [1:7574436035371636382:0] socket: 299 qp: -1
2025-11-19T14:00:17.650917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436034620160115:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:17.650936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.651113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436036629905946:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44138
2025-11-19T14:00:17.651413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436036315422847:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4405
2025-11-19T14:00:17.651540Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436034620160092:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.651621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436036315422848:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4405
2025-11-19T14:00:17.651599Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574436035371636317:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:17.651739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436034620160050:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.651771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:17.651776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436034620160050:2075] [node 6] ICP22 created new session: [4:7574436034620160118:2075]
2025-11-19T14:00:17.651783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436034620160118:2075] [node 6] ICS09 handshake done sender: [4:7574436034620160092:2114] self: [4:7574436034620160051:0] peer: [6:7574436036315422836:0] socket: 279 qp: -1
2025-11-19T14:00:17.651788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436034620160118:2075] [node 6] ICS10 traffic start
2025-11-19T14:00:17.651796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:17.652530Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436036629905926:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.652713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436036315422849:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4405
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:17.652904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436036629905917:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:17.652922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:17.652927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436036629905917:2110] [node 4] ICP22 created new session: [2:7574436036629905948:2110]
2025-11-19T14:00:17.652932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436036629905948:2110] [node 4] ICS09 handshake done sender: [2:7574436036629905926:2115] self: [2:7574436036629905925:0] peer: [4:7574436034620160112:0] socket: 309 qp: -1
2025-11-19T14:00:17.652936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436036629905948:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:17.652944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:17.653772Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436037295459155:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:17.654634Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436035371636408:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:17.654676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436035371636392:2113] [node 6] ICP18 handshake (actor [1:7574436035371636408:2117]) is too old
2025-11-19T14:00:17.655121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436037295459085:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.655135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:17.655141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436037295459085:2050] [node 3] ICP22 created new session: [5:7574436037295459185:2050]
2025-11-19T14:00:17.655146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436037295459185:2050] [node 3] ICS09 handshake done sender: [5:7574436037295459155:2114] self: [5:7574436037295459168:0] peer: [3:7574436034675525537:0] socket: 287 qp: -1
2025-11-19T14:00:17.655152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436037295459185:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:17.655159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:17.655431Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436036315422837:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:17.652704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436034620160117:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45873
2025-11-19T14:00:17.656475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436036315422850:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4405
2025-11-19T14:00:17.656546Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436036315422838:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:17.656624Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436036315422756:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27308) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436036315422837:2117] held: no
2025-11-19T14:00:17.656709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436036315422756:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.656718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:17.656723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436036315422756:2049] [node 2] ICP22 created new session: [6:7574436036315422851:2049]
2025-11-19T14:00:17.656728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436036315422851:2049] [node 2] ICS09 handshake done sender: [6:7574436036315422837:2117] self: [6:7574436036315422843:0] peer: [2:7574436036629905922:0] socket: 290 qp: -1
2025-11-19T14:00:17.656733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436036315422851:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:17.656741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:17.658310Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436034620160019:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27308) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436034620160110:2120] held: no
2025-11-19T14:00:17.658448Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436034620160110:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:17.659204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436034620160019:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.659219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:17.659226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436034620160019:2049] [node 2] ICP22 created new session: [4:7574436034620160120:2049]
2025-11-19T14:00:17.659232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436034620160120:2049] [node 2] ICS09 handshake done sender: [4:7574436034620160110:2120] self: [4:7574436034620160112:0] peer: [2:7574436036629905925:0] socket: 314 qp: -1
2025-11-19T14:00:17.659237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436034620160120:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:17.659244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:17.659930Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436036315422840:2120] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:17.661832Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436036315422755:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22701) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436036315422840:2120] held: no
2025-11-19T14:00:17.661907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436036315422755:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.661936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:17.661942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436036315422755:2048] [node 1] ICP22 created new session: [6:7574436036315422853:2048]
2025-11-19T14:00:17.661948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436036315422853:2048] [node 1] ICS09 handshake done sender: [6:7574436036315422840:2120] self: [6:7574436036315422845:0] peer: [1:7574436035371636393:0] socket: 317 qp: -1
2025-11-19T14:00:17.661954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436036315422853:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:17.661961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:17.662129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436036315422786:2072] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.662137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:17.662141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436036315422786:2072] [node 5] ICP22 created new session: [6:7574436036315422855:2072]
2025-11-19T14:00:17.662144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436036315422855:2072] [node 5] ICS09 handshake done sender: [6:7574436036315422838:2118] self: [6:7574436036315422846:0] peer: [5:7574436037295459159:0] socket: 315 qp: -1
2025-11-19T14:00:17.662150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436036315422855:2072] [node 5] ICS10 traffic start
2025-11-19T14:00:17.662154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:17.662283Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574436036315422786:2072] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:17.662452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436036315422772:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12723) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436036315422834:2115] held: no
2025-11-19T14:00:17.662699Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436036315422834:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:17.663528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436036315422772:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:17.663539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:17.663546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436036315422772:2063] [node 4] ICP22 created new session: [6:7574436036315422857:2063]
2025-11-19T14:00:17.663551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436036315422857:2063] [node 4] ICS09 handshake done sender: [6:7574436036315422834:2115] self: [6:7574436036315422836:0] peer: [4:7574436034620160051:0] socket: 283 qp: -1
2025-11-19T14:00:17.663557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436036315422857:2063] [node 4] ICS10 traffic start
2025-11-19T14:00:17.663565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:17.671939Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:17.672047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436036629905938:2048] [node 1] ICS07 socket disconnect 266 reason# EndOfStream
2025-11-19T14:00:17.672054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436036629905938:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.672085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:17.672114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436036629905938:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:17.674108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:17.674199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436034675525570:2048] [node 1] ICS01 socket: 268 reason# EndOfStream
2025-11-19T14:00:17.674209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436034675525469:2048] [node 1] ICP30 unregister session Session# [3:7574436034675525570:2048] VirtualId# [3:7574436034675525557:0]
2025-11-19T14:00:17.674216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436034675525570:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.674245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:17.674267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:17.675093Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436036629905847:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22701) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T14:00:17.675107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436036629905938:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:17.675427Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436036629905847:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22701) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:17.675439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436036629905938:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:17.675443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436036629905847:2048] [node 1] ICP30 unregister session Session# [2:7574436036629905938:2048] VirtualId# [2:7574436036629905931:0]
2025-11-19T14:00:17.675508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:17.675514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574436036629905847:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22701) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:17.675517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22701) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:17.684150Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:17.684212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436034620160115:2048] [node 1] ICS01 socket: 299 reason# EndOfStream
2025-11-19T14:00:17.684229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436034620160018:2048] [node 1] ICP30 unregister session Session# [4:7574436034620160115:2048] VirtualId# [4:7574436034620160111:0]
2025-11-19T14:00:17.684234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436034620160115:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.684257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:17.684280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:17.684313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:17.684370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436036315422853:2048] [node 1] ICS01 socket: 317 reason# EndOfStream
2025-11-19T14:00:17.684375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436036315422755:2048] [node 1] ICP30 unregister session Session# [6:7574436036315422853:2048] VirtualId# [6:7574436036315422845:0]
2025-11-19T14:00:17.684379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436036315422853:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.684408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:17.684427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:17.683877Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:17.683995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436037295459181:2048] [node 1] ICS01 socket: 286 reason# EndOfStream
2025-11-19T14:00:17.684007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436037295459083:2048] [node 1] ICP30 unregister session Session# [5:7574436037295459181:2048] VirtualId# [5:7574436037295459170:0]
2025-11-19T14:00:17.684016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436037295459181:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.684067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:17.684110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:17.699881Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:17.699990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436034675525562:2049] [node 2] ICS01 socket: 280 reason# EndOfStream
2025-11-19T14:00:17.700004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436034675525470:2049] [node 2] ICP30 unregister session Session# [3:7574436034675525562:2049] VirtualId# [3:7574436034675525551:0]
2025-11-19T14:00:17.700013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436034675525562:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.700053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:17.700075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:17.716078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:17.716190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436034620160120:2049] [node 2] ICS01 socket: 314 reason# EndOfStream
2025-11-19T14:00:17.716201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436034620160019:2049] [node 2] ICP30 unregister session Session# [4:7574436034620160120:2049] VirtualId# [4:7574436034620160112:0]
2025-11-19T14:00:17.716210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436034620160120:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.716262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:17.716287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:17.716454Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:17.716550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436037295459179:2049] [node 2] ICS01 socket: 267 reason# EndOfStream
2025-11-19T14:00:17.716555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436037295459084:2049] [node 2] ICP30 unregister session Session# [5:7574436037295459179:2049] VirtualId# [5:7574436037295459167:0]
2025-11-19T14:00:17.716559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436037295459179:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.716590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:17.716611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:17.719240Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:17.719338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436036315422851:2049] [node 2] ICS01 socket: 290 reason# EndOfStream
2025-11-19T14:00:17.719346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436036315422756:2049] [node 2] ICP30 unregister session Session# [6:7574436036315422851:2049] VirtualId# [6:7574436036315422843:0]
2025-11-19T14:00:17.719354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436036315422851:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.719405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:17.719429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:17.734724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:17.734833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436037295459185:2050] [node 3] ICS01 socket: 287 reason# EndOfStream
2025-11-19T14:00:17.734846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436037295459085:2050] [node 3] ICP30 unregister session Session# [5:7574436037295459185:2050] VirtualId# [5:7574436037295459168:0]
2025-11-19T14:00:17.734854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436037295459185:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.734906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:17.734934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:17.735030Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:17.735099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436036315422841:2050] [node 3] ICS01 socket: 273 reason# EndOfStream
2025-11-19T14:00:17.735105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436036315422757:2050] [node 3] ICP30 unregister session Session# [6:7574436036315422841:2050] VirtualId# [6:7574436036315422833:0]
2025-11-19T14:00:17.735111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436036315422841:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.735142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:17.735162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:17.766020Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:17.766147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436036315422857:2063] [node 4] ICS01 socket: 283 reason# EndOfStream
2025-11-19T14:00:17.766159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436036315422772:2063] [node 4] ICP30 unregister session Session# [6:7574436036315422857:2063] VirtualId# [6:7574436036315422836:0]
2025-11-19T14:00:17.766169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436036315422857:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:17.766226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:17.766256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:17.948157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436037287294220:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:17.948174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436037287294221:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:18.048003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436039092832643:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:18.048019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436039092832644:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:18.103833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436038761633319:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:18.103854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436038761633320:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:18.183876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436039734532148:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:18.183892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436039734532149:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:18.183897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436039734532150:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:18.216049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436038863046967:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:18.216060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436038863046968:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:18.216064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436038863046969:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:18.281053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436040017721302:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:18.281068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436040017721303:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:18.281074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436040017721304:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:18.308123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436038863046982:2060] [node 4] ICP01 ready to work
2025-11-19T14:00:18.307855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436041582261578:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:18.307864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436041582261579:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.310928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436040017721321:2063] [node 5] ICP01 ready to work
2025-11-19T14:00:18.310536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436039092832701:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33132
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.312819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436041582261583:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53234
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.314177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436038863047000:2075] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
2025-11-19T14:00:18.313808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436039734532202:2098] [node 6] ICP01 ready to work
2025-11-19T14:00:18.317852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436039092832710:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:18.317940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436039092832711:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:18.318172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436041582261588:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:18.318282Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436041582261583:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.320185Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436039092832701:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.327643Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436039734532149:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19031540ns) {Configure:104(19030523ns) {ProcessConfigured:135(18776931ns) {StartInitialHandshake:157(18776662ns) {DropHandshakes:469(284ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:476(944ns) {SwitchToState:178(137ns)}}}}}
2025-11-19T14:00:18.325662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436040017721331:2068] [node 4] ICP01 ready to work
2025-11-19T14:00:18.327690Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436037287294221:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19758257ns) {Configure:104(19757528ns) {ProcessConfigured:135(18770416ns) {StartInitialHandshake:157(18769648ns) {DropHandshakes:469(440ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(255ns) PrepareNewSessionHandshake:476(1259ns) {SwitchToState:178(813ns)}}}}}
2025-11-19T14:00:18.330780Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436041582261578:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(22770888ns) {Configure:104(22767632ns) {ProcessConfigured:135(21563922ns) {StartInitialHandshake:157(21563334ns) {DropHandshakes:469(340ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(87ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(1153ns) {SwitchToState:178(455ns)}}}}}
2025-11-19T14:00:18.331856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436038863046989:2065] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.336630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436039092832719:2112] [node 4] ICP01 ready to work
2025-11-19T14:00:18.337151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436039092832717:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33132
2025-11-19T14:00:18.337211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436039092832718:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33132
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.337783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436039092832710:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19787062ns) {Configure:104(19786687ns) {ProcessConfigured:135(19456380ns) {StartInitialHandshake:157(19456026ns) {DropHandshakes:469(629ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:476(724ns) {SwitchToState:178(504ns)}}}}}
2025-11-19T14:00:18.337702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436041582261596:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53234
2025-11-19T14:00:18.339551Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436039092832717:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.339590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436039092832643:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:18.339121Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436040017721302:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(27405169ns) {Configure:104(27404449ns) {ProcessConfigured:135(27141086ns) {StartInitialHandshake:157(27140627ns) {DropHandshakes:469(592ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:476(1019ns) {SwitchToState:178(638ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.339726Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436041582261585:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.339738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436041582261585:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:18.339764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436041582261597:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53234
2025-11-19T14:00:18.339382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436038863046967:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(19425006ns) {Configure:104(19423868ns) {ProcessConfigured:135(19042935ns) {StartInitialHandshake:157(19042307ns) {DropHandshakes:469(629ns) {DropIncomingHandshake:441(104ns) DropOutgoingHandshake:456(85ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:476(711ns) {SwitchToState:178(517ns)}}}}}
2025-11-19T14:00:18.339456Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436038863047000:2075] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.344018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436038863046968:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24010591ns) {Configure:104(24008953ns) {ProcessConfigured:135(23652598ns) {StartInitialHandshake:157(23651337ns) {DropHandshakes:469(617ns) {DropIncomingHandshake:441(96ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(504ns) PrepareNewSessionHandshake:476(915ns) {SwitchToState:178(841ns)}}}}}
2025-11-19T14:00:18.344137Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436041582261585:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:18.344820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436037287294220:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.344847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:18.344855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436037287294220:2048] [node 2] ICP22 created new session: [1:7574436041582261599:2048]
2025-11-19T14:00:18.344862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436041582261599:2048] [node 2] ICS09 handshake done sender: [1:7574436041582261585:2110] self: [1:7574436041582261584:0] peer: [2:7574436039092832724:0] socket: 273 qp: -1
2025-11-19T14:00:18.344869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436041582261599:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:18.344879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:18.345684Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436041582261596:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.346762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436039092832643:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62159) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436039092832717:2113] held: no
2025-11-19T14:00:18.349334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436038761633383:2104] [node 6] ICP01 ready to work
2025-11-19T14:00:18.348423Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436041582261597:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.349467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436039734532212:2103] [node 5] ICP01 ready to work
2025-11-19T14:00:18.356151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436038863046989:2065] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(24036070ns) {Configure:104(24035557ns) {ProcessConfigured:135(23658270ns) {StartInitialHandshake:157(23658046ns) {DropHandshakes:469(524ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(297ns) PrepareNewSessionHandshake:476(801ns) {SwitchToState:178(24ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.355902Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436039092832711:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(36013060ns) {Configure:104(24543721ns) {ProcessConfigured:135(24116621ns) {StartInitialHandshake:157(24114033ns) {DropHandshakes:469(511ns) {DropIncomingHandshake:441(147ns) DropOutgoingHandshake:456(104ns)} GenerateSessionVirtualId:988(366ns) PrepareNewSessionHandshake:476(1756ns) {SwitchToState:178(1184ns)}}}}}
2025-11-19T14:00:18.358578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436038761633384:2106] [node 5] ICP01 ready to work
2025-11-19T14:00:18.359195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436038761633385:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:18.362276Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436039092832718:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.363260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436041582261579:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(55328766ns) {Configure:104(55327754ns) {ProcessConfigured:135(53817107ns) {StartInitialHandshake:157(53815649ns) {DropHandshakes:469(245ns) {DropIncomingHandshake:441(46ns) DropOutgoingHandshake:456(79ns)} GenerateSessionVirtualId:988(63ns) PrepareNewSessionHandshake:476(1276ns) {SwitchToState:178(924ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.364168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436039734532225:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10495
2025-11-19T14:00:18.364227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436039734532226:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10495
2025-11-19T14:00:18.364259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436039734532227:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10495
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.364356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436038761633319:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(47483971ns) {Configure:104(47483042ns) {ProcessConfigured:135(47056902ns) {StartInitialHandshake:157(47056232ns) {DropHandshakes:469(634ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(187ns) PrepareNewSessionHandshake:476(883ns) {SwitchToState:178(250ns)}}}}}
2025-11-19T14:00:18.365524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436038761633396:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45388
2025-11-19T14:00:18.365566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436038761633397:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45388
2025-11-19T14:00:18.365595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436038761633398:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45388
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.367178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436041582261605:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53234
2025-11-19T14:00:18.366857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436040017721304:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(48361812ns) {Configure:104(48360644ns) {ProcessConfigured:135(48039712ns) {StartInitialHandshake:157(48039232ns) {DropHandshakes:469(746ns) {DropIncomingHandshake:441(44ns) DropOutgoingHandshake:456(43ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:476(919ns) {SwitchToState:178(654ns)}}}}}
2025-11-19T14:00:18.366900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436040017721379:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37671
2025-11-19T14:00:18.366946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436040017721381:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37671
2025-11-19T14:00:18.367468Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436040017721381:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.367498Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436040017721379:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.367522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436040017721302:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:18.367534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436040017721331:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:18.367767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436039092832728:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33132
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.367153Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436039734532212:2103] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17535145ns) {Configure:104(17534369ns) {ProcessConfigured:135(17070899ns) {StartInitialHandshake:157(17070496ns) {DropHandshakes:469(753ns) {DropIncomingHandshake:441(126ns) DropOutgoingHandshake:456(180ns)} GenerateSessionVirtualId:988(240ns) PrepareNewSessionHandshake:476(1009ns) {SwitchToState:178(161ns)}}}}}
2025-11-19T14:00:18.367241Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436039734532226:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.367264Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436039734532227:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.367380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436039734532229:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10495
2025-11-19T14:00:18.367389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436039734532149:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:18.368062Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436039734532225:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.368329Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436038761633396:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.368357Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436038761633398:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.368376Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436038761633397:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.368597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436038761633320:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:18.368610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436038761633319:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.372106Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436039092832717:2113] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:18.372442Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436039092832706:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.372452Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436039092832706:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:18.372517Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436039092832721:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.372523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436039092832721:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:18.372643Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436041582261591:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.372663Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436041582261591:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:18.372852Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436041582261589:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.372860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436041582261589:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:18.372715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436038863047047:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
2025-11-19T14:00:18.372812Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436039734532205:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.372825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436039734532205:2100] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.375923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436039092832730:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33132
2025-11-19T14:00:18.375961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436039092832643:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.375980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:18.375988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436039092832643:2048] [node 1] ICP22 created new session: [2:7574436039092832731:2048]
2025-11-19T14:00:18.375995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436039092832731:2048] [node 1] ICS09 handshake done sender: [2:7574436039092832717:2113] self: [2:7574436039092832724:0] peer: [1:7574436041582261584:0] socket: 275 qp: -1
2025-11-19T14:00:18.376002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436039092832731:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:18.376037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:18.376468Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436038863047047:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.376530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436038863047048:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
2025-11-19T14:00:18.376564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436038863046967:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.384897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436038761633404:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45388
2025-11-19T14:00:18.385105Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436041582261595:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.385119Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436041582261595:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:18.385819Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436038863047048:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.385905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436038863046968:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:18.387360Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436041582261595:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:18.387535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436041582261578:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.387558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:18.387564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436041582261578:2107] [node 5] ICP22 created new session: [1:7574436041582261607:2107]
2025-11-19T14:00:18.387570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436041582261607:2107] [node 5] ICS09 handshake done sender: [1:7574436041582261595:2114] self: [1:7574436041582261581:0] peer: [5:7574436038863047049:0] socket: 271 qp: -1
2025-11-19T14:00:18.387576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436041582261607:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:18.387585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:18.388284Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436041582261591:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:18.386933Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436039092832706:2107] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:18.387064Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436039092832723:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.387069Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436039092832723:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:18.389384Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436039092832721:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.389607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436038863047051:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.389702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436038863047052:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
2025-11-19T14:00:18.389868Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436038863047047:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:18.390294Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436038761633396:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:18.390470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436038863046967:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.390492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:18.390498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436038863046967:2048] [node 1] ICP22 created new session: [5:7574436038863047053:2048]
2025-11-19T14:00:18.390504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436038863047053:2048] [node 1] ICS09 handshake done sender: [5:7574436038863047047:2116] self: [5:7574436038863047049:0] peer: [1:7574436041582261581:0] socket: 290 qp: -1
2025-11-19T14:00:18.390510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436038863047053:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:18.390518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:18.391184Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436039734532205:2100] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:18.391339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436039734532202:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.391360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:18.391366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436039734532202:2098] [node 6] ICP22 created new session: [4:7574436039734532234:2098]
2025-11-19T14:00:18.391372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436039734532234:2098] [node 6] ICS09 handshake done sender: [4:7574436039734532205:2100] self: [4:7574436039734532204:0] peer: [6:7574436040017721383:0] socket: 272 qp: -1
2025-11-19T14:00:18.391376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436039734532234:2098] [node 6] ICS10 traffic start
2025-11-19T14:00:18.391384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:18.389174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436041582261588:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.389193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:18.389201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436041582261588:2111] [node 6] ICP22 created new session: [1:7574436041582261611:2111]
2025-11-19T14:00:18.389208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436041582261611:2111] [node 6] ICS09 handshake done sender: [1:7574436041582261591:2113] self: [1:7574436041582261590:0] peer: [6:7574436040017721382:0] socket: 283 qp: -1
2025-11-19T14:00:18.389216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436041582261611:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:18.389237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:18.392793Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436039092832723:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:18.393137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436039092832719:2112] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.393156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:18.393161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436039092832719:2112] [node 4] ICP22 created new session: [2:7574436039092832736:2112]
2025-11-19T14:00:18.393167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436039092832736:2112] [node 4] ICS09 handshake done sender: [2:7574436039092832721:2115] self: [2:7574436039092832720:0] peer: [4:7574436039734532231:0] socket: 285 qp: -1
2025-11-19T14:00:18.393173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436039092832736:2112] [node 4] ICS10 traffic start
2025-11-19T14:00:18.393181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:18.393381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436039092832710:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.393387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:18.393391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436039092832710:2109] [node 5] ICP22 created new session: [2:7574436039092832738:2109]
2025-11-19T14:00:18.393395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436039092832738:2109] [node 5] ICS09 handshake done sender: [2:7574436039092832723:2116] self: [2:7574436039092832712:0] peer: [5:7574436038863047050:0] socket: 266 qp: -1
2025-11-19T14:00:18.393399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436039092832738:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:18.393403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:18.393833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436038761633320:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.393852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:18.393858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436038761633320:2049] [node 2] ICP22 created new session: [3:7574436038761633407:2049]
2025-11-19T14:00:18.393864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436038761633407:2049] [node 2] ICS09 handshake done sender: [3:7574436038761633396:2113] self: [3:7574436038761633399:0] peer: [2:7574436039092832705:0] socket: 291 qp: -1
2025-11-19T14:00:18.393869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436038761633407:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:18.393876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:18.394861Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436038863047048:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:18.394987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436038863046968:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.395001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:18.395007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436038863046968:2049] [node 2] ICP22 created new session: [5:7574436038863047056:2049]
2025-11-19T14:00:18.395012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436038863047056:2049] [node 2] ICS09 handshake done sender: [5:7574436038863047048:2117] self: [5:7574436038863047050:0] peer: [2:7574436039092832712:0] socket: 303 qp: -1
2025-11-19T14:00:18.395017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436038863047056:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:18.395025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.396378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436040017721331:2068] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65320) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436040017721379:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.397822Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436041582261589:2112] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:18.397861Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436041582261605:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.397706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436040017721384:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37671
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.397920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436040017721385:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37671
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.398062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436040017721386:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37671
2025-11-19T14:00:18.398078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436040017721387:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37671
2025-11-19T14:00:18.398088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436040017721388:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37671
2025-11-19T14:00:18.398101Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436040017721381:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:18.398233Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436040017721379:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:18.398376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436040017721331:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.398390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:18.398395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436040017721331:2068] [node 4] ICP22 created new session: [6:7574436040017721389:2068]
2025-11-19T14:00:18.398401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436040017721389:2068] [node 4] ICS09 handshake done sender: [6:7574436040017721379:2114] self: [6:7574436040017721383:0] peer: [4:7574436039734532204:0] socket: 276 qp: -1
2025-11-19T14:00:18.398407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436040017721389:2068] [node 4] ICS10 traffic start
2025-11-19T14:00:18.398413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:18.396144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436038761633405:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45388
2025-11-19T14:00:18.396165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436038761633406:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45388
2025-11-19T14:00:18.398774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436040017721302:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.398786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:18.398790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436040017721302:2048] [node 1] ICP22 created new session: [6:7574436040017721391:2048]
2025-11-19T14:00:18.398795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436040017721391:2048] [node 1] ICS09 handshake done sender: [6:7574436040017721381:2116] self: [6:7574436040017721382:0] peer: [1:7574436041582261590:0] socket: 300 qp: -1
2025-11-19T14:00:18.398800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436040017721391:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:18.398806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:18.399367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436037287294221:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.399387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:18.399392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436037287294221:2049] [node 3] ICP22 created new session: [1:7574436041582261615:2049]
2025-11-19T14:00:18.399397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436041582261615:2049] [node 3] ICS09 handshake done sender: [1:7574436041582261589:2112] self: [1:7574436041582261580:0] peer: [3:7574436038761633400:0] socket: 282 qp: -1
2025-11-19T14:00:18.399402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436041582261615:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:18.399419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:18.402109Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436038761633406:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.402989Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436039734532229:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.403037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436039734532150:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:18.403177Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436038761633319:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62159) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436038761633398:2115] held: no
2025-11-19T14:00:18.407906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436039092832644:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.407941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:18.407952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436039092832644:2049] [node 3] ICP22 created new session: [2:7574436039092832741:2049]
2025-11-19T14:00:18.407961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436039092832741:2049] [node 3] ICS09 handshake done sender: [2:7574436039092832706:2107] self: [2:7574436039092832705:0] peer: [3:7574436038761633399:0] socket: 278 qp: -1
2025-11-19T14:00:18.407970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436039092832741:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:18.407982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.412984Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436040017721387:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.413027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436040017721321:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:18.413167Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436038863047045:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.413178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436038863047045:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.414028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436040017721395:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37671
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.414188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436039092832644:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:18.414270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436039092832743:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33132
2025-11-19T14:00:18.414692Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436038863047045:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:18.414863Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436040017721387:2120] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:18.411681Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436038761633392:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.411699Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436038761633392:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:18.412602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436038761633410:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45388
2025-11-19T14:00:18.412665Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436038761633398:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:18.414966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436040017721321:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.414982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:18.414990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436040017721321:2063] [node 5] ICP22 created new session: [6:7574436040017721396:2063]
2025-11-19T14:00:18.414996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436040017721396:2063] [node 5] ICS09 handshake done sender: [6:7574436040017721387:2120] self: [6:7574436040017721394:0] peer: [5:7574436038863047032:0] socket: 325 qp: -1
2025-11-19T14:00:18.415002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436040017721396:2063] [node 5] ICS10 traffic start
2025-11-19T14:00:18.415010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:18.416193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436039092832710:2109] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:18.416376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436038863046989:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.416412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:18.416419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436038863046989:2065] [node 6] ICP22 created new session: [5:7574436038863047060:2065]
2025-11-19T14:00:18.416426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436038863047060:2065] [node 6] ICS09 handshake done sender: [5:7574436038863047045:2115] self: [5:7574436038863047032:0] peer: [6:7574436040017721394:0] socket: 292 qp: -1
2025-11-19T14:00:18.416432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436038863047060:2065] [node 6] ICS10 traffic start
2025-11-19T14:00:18.416440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:18.417961Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436038761633410:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.417975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436038761633319:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.417985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:18.417990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436038761633319:2048] [node 1] ICP22 created new session: [3:7574436038761633414:2048]
2025-11-19T14:00:18.417995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436038761633414:2048] [node 1] ICS09 handshake done sender: [3:7574436038761633398:2115] self: [3:7574436038761633400:0] peer: [1:7574436041582261580:0] socket: 298 qp: -1
2025-11-19T14:00:18.418001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436038761633414:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:18.418006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.418670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436038863047062:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
2025-11-19T14:00:18.418939Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436038761633392:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:18.419079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436038761633385:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.419094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:18.419099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436038761633385:2108] [node 4] ICP22 created new session: [3:7574436038761633417:2108]
2025-11-19T14:00:18.419104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436038761633417:2108] [node 4] ICS09 handshake done sender: [3:7574436038761633392:2112] self: [3:7574436038761633391:0] peer: [4:7574436039734532237:0] socket: 296 qp: -1
2025-11-19T14:00:18.419108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436038761633417:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:18.419114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.420025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436039734532238:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10495
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.421109Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436039734532149:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27777) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436039734532227:2116] held: no
2025-11-19T14:00:18.421221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436039734532240:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10495
2025-11-19T14:00:18.424305Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436040017721386:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.424349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436040017721303:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:18.424597Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436039092832726:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.424609Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436039092832726:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.425402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436040017721399:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37671
2025-11-19T14:00:18.425477Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436039092832726:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:18.425671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436039092832711:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.425690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:18.425696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436039092832711:2110] [node 6] ICP22 created new session: [2:7574436039092832745:2110]
2025-11-19T14:00:18.425703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436039092832745:2110] [node 6] ICS09 handshake done sender: [2:7574436039092832726:2117] self: [2:7574436039092832716:0] peer: [6:7574436040017721398:0] socket: 321 qp: -1
2025-11-19T14:00:18.425708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436039092832745:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:18.425719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:18.425748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436039734532239:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10495
2025-11-19T14:00:18.426217Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436038863047062:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.426261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436038863046969:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:18.425872Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436040017721386:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:18.426676Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436040017721388:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.426706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436040017721304:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:18.426810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436040017721303:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.426826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:18.426832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436040017721303:2049] [node 2] ICP22 created new session: [6:7574436040017721401:2049]
2025-11-19T14:00:18.426837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436040017721401:2049] [node 2] ICS09 handshake done sender: [6:7574436040017721386:2119] self: [6:7574436040017721398:0] peer: [2:7574436039092832716:0] socket: 324 qp: -1
2025-11-19T14:00:18.426844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436040017721401:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:18.426849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:18.432485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436038761633401:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.432504Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436038761633401:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:18.432697Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436038761633402:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.432703Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436038761633402:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.433728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436038863047064:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.435067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436040017721403:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37671
2025-11-19T14:00:18.434545Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436038761633402:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:18.435520Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436038863047062:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:18.435286Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436038761633401:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:18.435739Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436040017721388:2121] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:18.435891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436038761633383:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.435911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:18.435917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436038761633383:2104] [node 6] ICP22 created new session: [3:7574436038761633421:2104]
2025-11-19T14:00:18.435923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436038761633421:2104] [node 6] ICS09 handshake done sender: [3:7574436038761633401:2116] self: [3:7574436038761633386:0] peer: [6:7574436040017721400:0] socket: 322 qp: -1
2025-11-19T14:00:18.435929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436038761633421:2104] [node 6] ICS10 traffic start
2025-11-19T14:00:18.435937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:18.435684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436038863046969:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.435710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:18.435717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436038863046969:2050] [node 3] ICP22 created new session: [5:7574436038863047065:2050]
2025-11-19T14:00:18.435724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436038863047065:2050] [node 3] ICS09 handshake done sender: [5:7574436038863047062:2120] self: [5:7574436038863047063:0] peer: [3:7574436038761633387:0] socket: 286 qp: -1
2025-11-19T14:00:18.435731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436038863047065:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:18.435757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:18.437043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436040017721304:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.437065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:18.437073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436040017721304:2050] [node 3] ICP22 created new session: [6:7574436040017721404:2050]
2025-11-19T14:00:18.437080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436040017721404:2050] [node 3] ICS09 handshake done sender: [6:7574436040017721388:2121] self: [6:7574436040017721400:0] peer: [3:7574436038761633386:0] socket: 326 qp: -1
2025-11-19T14:00:18.437087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436040017721404:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:18.437098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:18.437797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436038761633384:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.437822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:18.437828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436038761633384:2106] [node 5] ICP22 created new session: [3:7574436038761633423:2106]
2025-11-19T14:00:18.437833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436038761633423:2106] [node 5] ICS09 handshake done sender: [3:7574436038761633402:2117] self: [3:7574436038761633387:0] peer: [5:7574436038863047063:0] socket: 280 qp: -1
2025-11-19T14:00:18.437838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436038761633423:2106] [node 5] ICS10 traffic start
2025-11-19T14:00:18.437845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:18.438528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574436040017721304:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:18.444905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436039734532150:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19633) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436039734532229:2118] held: no
2025-11-19T14:00:18.445052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436039734532239:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.445075Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436039734532229:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:18.445283Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436039734532227:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.448820Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436041582261603:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.448832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436041582261603:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:18.449465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436038863047067:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
2025-11-19T14:00:18.450120Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436041582261603:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:18.450408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436041582261579:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.450428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:18.450437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436041582261579:2108] [node 4] ICP22 created new session: [1:7574436041582261618:2108]
2025-11-19T14:00:18.450443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436041582261618:2108] [node 4] ICS09 handshake done sender: [1:7574436041582261603:2117] self: [1:7574436041582261582:0] peer: [4:7574436039734532244:0] socket: 311 qp: -1
2025-11-19T14:00:18.450449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436041582261618:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:18.450457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.447866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436039734532150:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.447889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:18.447896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436039734532150:2050] [node 3] ICP22 created new session: [4:7574436039734532242:2050]
2025-11-19T14:00:18.447901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436039734532242:2050] [node 3] ICS09 handshake done sender: [4:7574436039734532229:2118] self: [4:7574436039734532237:0] peer: [3:7574436038761633391:0] socket: 302 qp: -1
2025-11-19T14:00:18.447910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436039734532242:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:18.447919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:18.448300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436039734532148:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:18.448429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436039734532149:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.448438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:18.448443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436039734532149:2049] [node 2] ICP22 created new session: [4:7574436039734532245:2049]
2025-11-19T14:00:18.448447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436039734532245:2049] [node 2] ICS09 handshake done sender: [4:7574436039734532227:2116] self: [4:7574436039734532231:0] peer: [2:7574436039092832720:0] socket: 293 qp: -1
2025-11-19T14:00:18.448452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436039734532245:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:18.448458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:18.451047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436039734532247:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10495
2025-11-19T14:00:18.451260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436039734532148:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62159) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436039734532239:2120] held: no
2025-11-19T14:00:18.451351Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436039734532239:2120] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:18.451761Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436038863047067:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.451814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436038863046982:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:18.451938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436039734532228:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:18.451947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436039734532228:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:18.452215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436039734532148:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.452227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:18.452237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436039734532148:2048] [node 1] ICP22 created new session: [4:7574436039734532248:2048]
2025-11-19T14:00:18.452243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436039734532248:2048] [node 1] ICS09 handshake done sender: [4:7574436039734532239:2120] self: [4:7574436039734532244:0] peer: [1:7574436041582261582:0] socket: 268 qp: -1
2025-11-19T14:00:18.452253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436039734532248:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:18.452261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:18.452796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436038863047069:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
2025-11-19T14:00:18.452993Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436039734532228:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:18.453162Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436038863047067:2122] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:18.453268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436038863046982:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:18.453284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:18.453289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436038863046982:2060] [node 4] ICP22 created new session: [5:7574436038863047070:2060]
2025-11-19T14:00:18.453293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436038863047070:2060] [node 4] ICS09 handshake done sender: [5:7574436038863047067:2122] self: [5:7574436038863047068:0] peer: [4:7574436039734532224:0] socket: 279 qp: -1
2025-11-19T14:00:18.453298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436038863047070:2060] [node 4] ICS10 traffic start
2025-11-19T14:00:18.453307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:18.454163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436039734532212:2103] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:18.454182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:18.454188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436039734532212:2103] [node 5] ICP22 created new session: [4:7574436039734532251:2103]
2025-11-19T14:00:18.454193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436039734532251:2103] [node 5] ICS09 handshake done sender: [4:7574436039734532228:2117] self: [4:7574436039734532224:0] peer: [5:7574436038863047068:0] socket: 265 qp: -1
2025-11-19T14:00:18.454198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436039734532251:2103] [node 5] ICS10 traffic start
2025-11-19T14:00:18.454203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:18.479394Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:18.479513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436039092832731:2048] [node 1] ICS01 socket: 275 reason# EndOfStream
2025-11-19T14:00:18.479523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436039092832643:2048] [node 1] ICP30 unregister session Session# [2:7574436039092832731:2048] VirtualId# [2:7574436039092832724:0]
2025-11-19T14:00:18.479532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436039092832731:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:18.479592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:18.479619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:18.480339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:18.480420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436038761633414:2048] [node 1] ICS01 socket: 298 reason# EndOfStream
2025-11-19T14:00:18.480427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436038761633319:2048] [node 1] ICP30 unregister session Session# [3:7574436038761633414:2048] VirtualId# [3:7574436038761633400:0]
2025-11-19T14:00:18.480433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436038761633414:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:18.480463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:18.480482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:18.490785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:18.490881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436039734532248:2048] [node 1] ICS01 socket: 268 reason# EndOfStream
2025-11-19T14:00:18.490890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436039734532148:2048] [node 1] ICP30 unregister session Session# [4:7574436039734532248:2048] VirtualId# [4:7574436039734532244:0]
2025-11-19T14:00:18.490897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436039734532248:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:18.490953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:18.490984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:18.491025Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:18.491085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436040017721391:2048] [node 1] ICS01 socket: 300 reason# EndOfStream
2025-11-19T14:00:18.491089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436040017721302:2048] [node 1] ICP30 unregister session Session# [6:7574436040017721391:2048] VirtualId# [6:7574436040017721382:0]
2025-11-19T14:00:18.491095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436040017721391:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:18.491122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:18.491146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:18.491460Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:18.491541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436038863047053:2048] [node 1] ICS01 socket: 290 reason# EndOfStream
2025-11-19T14:00:18.491546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436038863046967:2048] [node 1] ICP30 unregister session Session# [5:7574436038863047053:2048] VirtualId# [5:7574436038863047049:0]
2025-11-19T14:00:18.491550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436038863047053:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:18.491586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:18.491606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:18.493216Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:18.493292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436038761633407:2049] [node 2] ICS01 socket: 291 reason# EndOfStream
2025-11-19T14:00:18.493298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436038761633320:2049] [node 2] ICP30 unregister session Session# [3:7574436038761633407:2049] VirtualId# [3:7574436038761633399:0]
2025-11-19T14:00:18.493305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436038761633407:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:18.493333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:18.493350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:18.504012Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:18.504106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436038863047056:2049] [node 2] ICS01 socket: 303 reason# EndOfStream
2025-11-19T14:00:18.504114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436038863046968:2049] [node 2] ICP30 unregister session Session# [5:7574436038863047056:2049] VirtualId# [5:7574436038863047050:0]
2025-11-19T14:00:18.504123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436038863047056:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:18.504177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:18.504203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:18.504243Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:18.504308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436040017721401:2049] [node 2] ICS01 socket: 324 reason# EndOfStream
2025-11-19T14:00:18.504313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436040017721303:2049] [node 2] ICP30 unregister session Session# [6:7574436040017721401:2049] VirtualId# [6:7574436040017721398:0]
2025-11-19T14:00:18.504318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436040017721401:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:18.504344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:18.504364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:18.504394Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:18.504455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436039734532245:2049] [node 2] ICS01 socket: 293 reason# EndOfStream
2025-11-19T14:00:18.504459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436039734532149:2049] [node 2] ICP30 unregister session Session# [4:7574436039734532245:2049] VirtualId# [4:7574436039734532231:0]
2025-11-19T14:00:18.504463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436039734532245:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:18.504488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:18.504507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:18.534972Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:18.535073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436038863047065:2050] [node 3] ICS01 socket: 286 reason# EndOfStream
2025-11-19T14:00:18.535082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436038863046969:2050] [node 3] ICP30 unregister session Session# [5:7574436038863047065:2050] VirtualId# [5:7574436038863047063:0]
2025-11-19T14:00:18.535090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436038863047065:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:18.535149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:18.535171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:18.538530Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:18.538622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436040017721404:2050] [node 3] ICS07 socket disconnect 326 reason# EndOfStream
2025-11-19T14:00:18.538628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436040017721404:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:18.538683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:18.538708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436040017721404:2050] [node 3] ICS15 start handshake
2025-11-19T14:00:18.543918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436040017721304:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19633) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:18.543940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436040017721404:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:18.543946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436040017721304:2050] [node 3] ICP30 unregister session Session# [6:7574436040017721404:2050] VirtualId# [6:7574436040017721400:0]
2025-11-19T14:00:18.543990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:18.543996Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436040017721304:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19633) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:18.543999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19633) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:18.554123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436040017721304:2050] [node 3] ICP33 wake up from error state
2025-11-19T14:00:18.566894Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:18.566993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436040017721389:2068] [node 4] ICS01 socket: 276 reason# EndOfStream
2025-11-19T14:00:18.567003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436040017721331:2068] [node 4] ICP30 unregister session Session# [6:7574436040017721389:2068] VirtualId# [6:7574436040017721383:0]
2025-11-19T14:00:18.567011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436040017721389:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:18.567070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:18.567095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:18.848352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436039475058518:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:18.848364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436039475058519:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:18.953846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436040338310471:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:18.953864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436040338310472:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:19.004179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436038650588867:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:19.004195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436038650588868:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:19.047949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436042522376615:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:19.047962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436042522376616:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:19.047966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436042522376617:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:19.100000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436045968410596:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:19.100020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436045968410597:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:19.100025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436045968410598:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:19.183258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436045398832222:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:19.195799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436045398832223:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:19.196203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436045398832224:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:19.208044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436043770025876:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:19.208754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436045968410610:2060] [node 6] ICP01 ready to work
2025-11-19T14:00:19.208297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436045398832235:2059] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.209417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436042945556223:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16141
2025-11-19T14:00:19.209248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436042522376670:2092] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.210085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436043770025879:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:19.211223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436043770025887:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8710
2025-11-19T14:00:19.213189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436045398832262:2077] [node 4] ICP01 ready to work
2025-11-19T14:00:19.213988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436043770025888:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:19.211987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436044633277832:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:19.212651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436044633277836:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:19.216744Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436042945556223:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.216947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436042945556225:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:19.217352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436042945556225:2108] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:19.211377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436045968410623:2065] [node 4] ICP01 ready to work
2025-11-19T14:00:19.217527Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436043770025887:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.218032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436042945556228:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-11-19T14:00:19.218667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436042522376676:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54581
2025-11-19T14:00:19.218676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436042522376677:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54581
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.219455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436043770025893:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8710
2025-11-19T14:00:19.219545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436042945556233:2111] [node 4] ICP01 ready to work
2025-11-19T14:00:19.220121Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436045398832244:2065] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.220135Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574436045398832244:2065] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:19.220789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436044633277842:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.221457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436043770025894:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8710
2025-11-19T14:00:19.221780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436042522376665:2094] [node 6] ICP01 ready to work
2025-11-19T14:00:19.222453Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436045398832244:2065] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.228133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436040338310472:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(19362077ns) {Configure:104(19361065ns) {ProcessConfigured:135(18961001ns) {StartInitialHandshake:157(18959954ns) {DropHandshakes:469(272ns) {DropIncomingHandshake:441(40ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:476(1488ns) {SwitchToState:178(843ns)}}}}}
2025-11-19T14:00:19.228391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436042945556241:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16141
2025-11-19T14:00:19.228426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436042945556242:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16141
2025-11-19T14:00:19.228441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436042945556243:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16141
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.229164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436042945556244:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16141
2025-11-19T14:00:19.230191Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436042945556223:2107] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:19.230320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436042945556225:2108] [node 6] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.230336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:19.230355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436042945556225:2108] [node 6] ICP22 created new session: [3:7574436042945556245:2108]
2025-11-19T14:00:19.230361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436042945556245:2108] [node 6] ICS09 handshake done sender: [3:7574436042945556223:2107] self: [3:7574436042945556226:0] peer: [6:7574436045398832243:0] socket: 275 qp: -1
2025-11-19T14:00:19.230369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436042945556245:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:19.230393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:19.231178Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436042945556244:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.237169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436044633277846:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7430
2025-11-19T14:00:19.237190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436044633277847:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7430
2025-11-19T14:00:19.237244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436044633277848:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7430
2025-11-19T14:00:19.237254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436044633277849:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7430
2025-11-19T14:00:19.237264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436044633277850:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7430
2025-11-19T14:00:19.237681Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436042522376677:2103] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.237715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436042522376616:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:19.237830Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436044633277835:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.237836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436044633277835:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:19.238400Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436044633277835:2110] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:19.238598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436044633277832:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.238613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:19.238620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436044633277832:2108] [node 4] ICP22 created new session: [2:7574436044633277852:2108]
2025-11-19T14:00:19.238628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436044633277852:2108] [node 4] ICS09 handshake done sender: [2:7574436044633277835:2110] self: [2:7574436044633277834:0] peer: [4:7574436042522376693:0] socket: 283 qp: -1
2025-11-19T14:00:19.238636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436044633277852:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:19.238647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.240203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436042522376694:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54581
2025-11-19T14:00:19.240661Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436042522376677:2103] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:19.240835Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436044633277847:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.241074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436042522376616:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.241092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:19.241097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436042522376616:2049] [node 2] ICP22 created new session: [4:7574436042522376695:2049]
2025-11-19T14:00:19.241103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436042522376695:2049] [node 2] ICS09 handshake done sender: [4:7574436042522376677:2103] self: [4:7574436042522376693:0] peer: [2:7574436044633277834:0] socket: 288 qp: -1
2025-11-19T14:00:19.241110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436042522376695:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:19.241127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:19.240375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436045398832297:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29982
2025-11-19T14:00:19.245857Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436042522376676:2102] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.249356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436043770025879:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(39194060ns) {Configure:104(39193195ns) {ProcessConfigured:135(38867979ns) {StartInitialHandshake:157(38867513ns) {DropHandshakes:469(518ns) {DropIncomingHandshake:441(86ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:476(1231ns) {SwitchToState:178(585ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.250072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436045398832299:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29982
2025-11-19T14:00:19.252643Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436045398832299:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.252696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436045398832262:2077] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:19.253089Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436044633277848:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.253296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436042522376692:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.253307Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436042522376692:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.253972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436045398832301:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29982
2025-11-19T14:00:19.254369Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436044633277832:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:19.254395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436040338310471:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:19.254521Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436042522376692:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:19.254694Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436045398832299:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:19.254886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436045398832262:2077] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.254904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:19.254910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436045398832262:2077] [node 4] ICP22 created new session: [6:7574436045398832302:2077]
2025-11-19T14:00:19.254915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436045398832302:2077] [node 4] ICS09 handshake done sender: [6:7574436045398832299:2112] self: [6:7574436045398832300:0] peer: [4:7574436042522376682:0] socket: 312 qp: -1
2025-11-19T14:00:19.254922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436045398832302:2077] [node 4] ICS10 traffic start
2025-11-19T14:00:19.254933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:19.255426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436042522376665:2094] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.255440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:19.255446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436042522376665:2094] [node 6] ICP22 created new session: [4:7574436042522376699:2094]
2025-11-19T14:00:19.255450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436042522376699:2094] [node 6] ICS09 handshake done sender: [4:7574436042522376692:2114] self: [4:7574436042522376682:0] peer: [6:7574436045398832300:0] socket: 311 qp: -1
2025-11-19T14:00:19.255455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436042522376699:2094] [node 6] ICS10 traffic start
2025-11-19T14:00:19.255460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:19.255363Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436043770025882:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.255375Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436043770025882:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:19.256706Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436044633277849:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.257750Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436043770025882:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:19.257929Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436043770025893:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.257974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436039475058518:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.257989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:19.257995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436039475058518:2048] [node 2] ICP22 created new session: [1:7574436043770025897:2048]
2025-11-19T14:00:19.258000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436043770025897:2048] [node 2] ICS09 handshake done sender: [1:7574436043770025882:2110] self: [1:7574436043770025881:0] peer: [2:7574436044633277856:0] socket: 280 qp: -1
2025-11-19T14:00:19.258004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436043770025897:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:19.258011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:19.256320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574436045398832224:2050] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.256340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:19.256351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436045398832224:2050] [node 3] ICP22 created new session: [6:7574436045398832304:2050]
2025-11-19T14:00:19.256357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436045398832304:2050] [node 3] ICS09 handshake done sender: [6:7574436045398832244:2065] self: [6:7574436045398832243:0] peer: [3:7574436042945556226:0] socket: 272 qp: -1
2025-11-19T14:00:19.256363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436045398832304:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:19.256388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:19.260945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436043770025876:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(52814710ns) {Configure:104(52813308ns) {ProcessConfigured:135(52363329ns) {StartInitialHandshake:157(52362324ns) {DropHandshakes:469(995ns) {DropIncomingHandshake:441(126ns) DropOutgoingHandshake:456(124ns)} GenerateSessionVirtualId:988(183ns) PrepareNewSessionHandshake:476(926ns) {SwitchToState:178(469ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.263909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436043770025902:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8710
2025-11-19T14:00:19.264474Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436043770025902:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.264897Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436042945556241:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.268111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436045398832310:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29982
2025-11-19T14:00:19.268411Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436045398832297:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.268447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436045398832223:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:19.268501Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436045398832310:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.268517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436045398832222:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.270021Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436044633277844:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.270033Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436044633277844:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:19.270963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436040338310471:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1570) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436044633277848:2117] held: no
2025-11-19T14:00:19.271087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436044633277858:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7430
2025-11-19T14:00:19.269462Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436042522376616:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(28285832ns) {HandleHandshakeStatus:329(28285181ns) {DropIncomingHandshake:441(256ns) DropOutgoingHandshake:456(1527ns) SwitchToState:178(72ns) ProcessPendingSessionEvents:495(12875574ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 12859476ns max# 12783761ns}}}
2025-11-19T14:00:19.271215Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436042945556242:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.272195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436045398832313:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29982
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.272264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436045968410674:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49767
2025-11-19T14:00:19.272319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436045968410675:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49767
2025-11-19T14:00:19.272350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436045968410676:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49767
2025-11-19T14:00:19.272376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436045968410677:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49767
2025-11-19T14:00:19.272696Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436043770025894:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.272960Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436045968410674:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.273003Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436045968410675:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.273021Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436045968410677:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.273059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436045968410623:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:19.273067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436045968410598:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:19.272605Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436042945556233:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(52444128ns) {Configure:104(52443536ns) {ProcessConfigured:135(52229712ns) {StartInitialHandshake:157(52228947ns) {DropHandshakes:469(263ns) {DropIncomingHandshake:441(26ns) DropOutgoingHandshake:456(21ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:476(571ns) {SwitchToState:178(42ns)}}}}}
2025-11-19T14:00:19.272828Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436044633277844:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:19.273653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436044633277842:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.273673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:19.273678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436044633277842:2112] [node 6] ICP22 created new session: [2:7574436044633277860:2112]
2025-11-19T14:00:19.273684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436044633277860:2112] [node 6] ICS09 handshake done sender: [2:7574436044633277844:2113] self: [2:7574436044633277843:0] peer: [6:7574436045398832311:0] socket: 308 qp: -1
2025-11-19T14:00:19.273689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436044633277860:2112] [node 6] ICS10 traffic start
2025-11-19T14:00:19.273696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:19.273911Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436045968410676:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.273940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436045968410597:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.274141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436045968410682:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49767
2025-11-19T14:00:19.274307Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436044633277838:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.274314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436044633277838:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-11-19T14:00:19.275267Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436044633277838:2111] [node 5] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-11-19T14:00:19.275308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436045968410683:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49767
2025-11-19T14:00:19.275427Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436045968410675:2116] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:19.273279Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436042522376673:2100] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.273289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436042522376673:2100] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:19.273865Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436042522376673:2100] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:19.274129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436042522376670:2092] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.274140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:19.274145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436042522376670:2092] [node 5] ICP22 created new session: [4:7574436042522376703:2092]
2025-11-19T14:00:19.274151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436042522376703:2092] [node 5] ICS09 handshake done sender: [4:7574436042522376673:2100] self: [4:7574436042522376672:0] peer: [5:7574436045968410679:0] socket: 276 qp: -1
2025-11-19T14:00:19.274156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436042522376703:2092] [node 5] ICS10 traffic start
2025-11-19T14:00:19.274186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:19.275574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436044633277836:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.275589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:19.275594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436044633277836:2109] [node 5] ICP22 created new session: [2:7574436044633277864:2109]
2025-11-19T14:00:19.275600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436044633277864:2109] [node 5] ICS09 handshake done sender: [2:7574436044633277838:2111] self: [2:7574436044633277837:0] peer: [5:7574436045968410681:0] socket: 284 qp: -1
2025-11-19T14:00:19.275605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436044633277864:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:19.275611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:19.275900Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436044633277848:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:19.275795Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436045968410676:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:19.273492Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436045398832297:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:19.275880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436045398832223:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.275892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:19.275900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436045398832223:2049] [node 2] ICP22 created new session: [6:7574436045398832314:2049]
2025-11-19T14:00:19.275906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436045398832314:2049] [node 2] ICS09 handshake done sender: [6:7574436045398832297:2110] self: [6:7574436045398832311:0] peer: [2:7574436044633277843:0] socket: 310 qp: -1
2025-11-19T14:00:19.275912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436045398832314:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:19.275919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:19.276044Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436043770025884:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.276064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436043770025884:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:19.276220Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436043770025895:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.276228Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436043770025895:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:19.276476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436045968410597:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.276491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:19.276496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436045968410597:2049] [node 2] ICP22 created new session: [5:7574436045968410684:2049]
2025-11-19T14:00:19.276501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436045968410684:2049] [node 2] ICS09 handshake done sender: [5:7574436045968410676:2117] self: [5:7574436045968410681:0] peer: [2:7574436044633277837:0] socket: 319 qp: -1
2025-11-19T14:00:19.276505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436045968410684:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:19.276524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:19.281114Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436039475058518:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(23044090ns) {HandleHandshakeStatus:329(23043501ns) {DropIncomingHandshake:441(3075ns) DropOutgoingHandshake:456(108ns) SwitchToState:178(73ns) ProcessPendingSessionEvents:495(15366008ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 15343028ns max# 15277281ns}}}
2025-11-19T14:00:19.281073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574436045398832224:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(58365451ns) {HandleHandshakeStatus:329(58364875ns) {DropIncomingHandshake:441(453ns) DropOutgoingHandshake:456(540ns) SwitchToState:178(98ns) ProcessPendingSessionEvents:495(15390397ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 15370283ns max# 15301721ns}}}
2025-11-19T14:00:19.280516Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436042522376615:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(53145979ns) {Configure:104(53144068ns) {ProcessConfigured:135(52743932ns) {StartInitialHandshake:157(52743200ns) {DropHandshakes:469(824ns) {DropIncomingHandshake:441(199ns) DropOutgoingHandshake:456(144ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:476(1396ns) {SwitchToState:178(611ns)}}}}}
2025-11-19T14:00:19.280598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436042522376615:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.283902Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436043770025890:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.283921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436043770025890:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:19.284346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436043770025906:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8710
2025-11-19T14:00:19.287842Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436043770025890:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:19.287874Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436042945556230:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.287891Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436042945556230:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:19.288029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436042522376665:2094] [node 6] ICP35 event processing took too much time PendingConnection:278(32509508ns) {HandleHandshakeStatus:329(32508968ns) {DropIncomingHandshake:441(24ns) DropOutgoingHandshake:456(147ns) SwitchToState:178(41ns) ProcessPendingSessionEvents:495(28021081ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 28002745ns max# 27941727ns}}}
2025-11-19T14:00:19.288439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436038650588867:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:19.288467Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436038650588867:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(23501299ns) {IncomingHandshake:261(23500921ns) {DropIncomingHandshake:441(48ns) PrepareNewSessionHandshake:476(420ns) {SwitchToState:178(102ns)} GenerateSessionVirtualId:988(181ns) IssueIncomingHandshakeReply:193(23495739ns) {SwitchToState:178(382ns)}}}
2025-11-19T14:00:19.292039Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436043770025906:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.292229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436043770025888:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.292253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:19.292261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436043770025888:2107] [node 4] ICP22 created new session: [1:7574436043770025908:2107]
2025-11-19T14:00:19.292267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436043770025908:2107] [node 4] ICS09 handshake done sender: [1:7574436043770025890:2113] self: [1:7574436043770025889:0] peer: [4:7574436042522376706:0] socket: 282 qp: -1
2025-11-19T14:00:19.292276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436043770025908:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:19.292285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:19.299923Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436043770025884:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:19.300284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436039475058519:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.300310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:19.300317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436039475058519:2049] [node 3] ICP22 created new session: [1:7574436043770025912:2049]
2025-11-19T14:00:19.300323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436043770025912:2049] [node 3] ICS09 handshake done sender: [1:7574436043770025884:2111] self: [1:7574436043770025883:0] peer: [3:7574436042945556247:0] socket: 281 qp: -1
2025-11-19T14:00:19.300331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436043770025912:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:19.300349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:19.299983Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436044633277850:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.300005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436040338310471:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.300019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:19.300030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436040338310471:2048] [node 1] ICP22 created new session: [2:7574436044633277866:2048]
2025-11-19T14:00:19.300041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436044633277866:2048] [node 1] ICS09 handshake done sender: [2:7574436044633277848:2117] self: [2:7574436044633277856:0] peer: [1:7574436043770025881:0] socket: 304 qp: -1
2025-11-19T14:00:19.300048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436044633277866:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:19.300055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.301706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436042945556251:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16141
2025-11-19T14:00:19.301727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436042945556252:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16141
2025-11-19T14:00:19.302642Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436042945556241:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:19.302819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436045968410623:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.302849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:19.302856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436045968410623:2065] [node 4] ICP22 created new session: [5:7574436045968410686:2065]
2025-11-19T14:00:19.302863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436045968410686:2065] [node 4] ICS09 handshake done sender: [5:7574436045968410675:2116] self: [5:7574436045968410679:0] peer: [4:7574436042522376672:0] socket: 318 qp: -1
2025-11-19T14:00:19.302869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436045968410686:2065] [node 4] ICS10 traffic start
2025-11-19T14:00:19.302890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:19.306261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436038650588867:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.306291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:19.306300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436038650588867:2048] [node 1] ICP22 created new session: [3:7574436042945556253:2048]
2025-11-19T14:00:19.306308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436042945556253:2048] [node 1] ICS09 handshake done sender: [3:7574436042945556241:2114] self: [3:7574436042945556247:0] peer: [1:7574436043770025883:0] socket: 285 qp: -1
2025-11-19T14:00:19.306315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436042945556253:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:19.306327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:19.306450Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574436045398832223:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(30455511ns) {HandleHandshakeStatus:329(30454936ns) {DropIncomingHandshake:441(173ns) DropOutgoingHandshake:456(1488ns) SwitchToState:178(241ns) ProcessPendingSessionEvents:495(30369849ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 30346204ns max# 30271219ns}}}
2025-11-19T14:00:19.306989Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436043770025895:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:19.307652Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436042945556251:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.309153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436038650588868:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.309671Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436044633277845:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.309683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436044633277845:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:19.310517Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436044633277845:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:19.310813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436040338310472:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.310843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:19.310851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436040338310472:2049] [node 3] ICP22 created new session: [2:7574436044633277870:2049]
2025-11-19T14:00:19.310856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436044633277870:2049] [node 3] ICS09 handshake done sender: [2:7574436044633277845:2114] self: [2:7574436044633277827:0] peer: [3:7574436042945556257:0] socket: 323 qp: -1
2025-11-19T14:00:19.310863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436044633277870:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:19.310870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:19.308313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436042522376708:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54581
2025-11-19T14:00:19.309455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436042522376709:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54581
2025-11-19T14:00:19.309505Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436042522376676:2102] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:19.311468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436042522376615:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.311485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:19.311492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436042522376615:2048] [node 1] ICP22 created new session: [4:7574436042522376710:2048]
2025-11-19T14:00:19.311510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436042522376710:2048] [node 1] ICS09 handshake done sender: [4:7574436042522376676:2102] self: [4:7574436042522376706:0] peer: [1:7574436043770025889:0] socket: 287 qp: -1
2025-11-19T14:00:19.311518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436042522376710:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:19.311527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:19.311874Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436042522376709:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.312002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436042522376617:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:19.308430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436045968410688:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49767
2025-11-19T14:00:19.308482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436045968410689:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49767
2025-11-19T14:00:19.309664Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436045968410688:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.309686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436045968410596:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:19.316158Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436043770025899:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.316173Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436043770025899:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:19.316312Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436042945556249:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.316324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436042945556249:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:19.320163Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436042945556230:2110] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:19.321138Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436045968410677:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:19.321321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436045968410598:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.321340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:19.321348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436045968410598:2050] [node 3] ICP22 created new session: [5:7574436045968410692:2050]
2025-11-19T14:00:19.321355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436045968410692:2050] [node 3] ICS09 handshake done sender: [5:7574436045968410677:2118] self: [5:7574436045968410680:0] peer: [3:7574436042945556229:0] socket: 320 qp: -1
2025-11-19T14:00:19.321363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436045968410692:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:19.321372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.323385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436038650588868:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1565) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436042945556251:2119] held: no
2025-11-19T14:00:19.323756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436042945556258:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16141
2025-11-19T14:00:19.323977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436042945556228:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.324001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:19.324009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436042945556228:2109] [node 5] ICP22 created new session: [3:7574436042945556259:2109]
2025-11-19T14:00:19.324016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436042945556259:2109] [node 5] ICS09 handshake done sender: [3:7574436042945556230:2110] self: [3:7574436042945556229:0] peer: [5:7574436045968410680:0] socket: 286 qp: -1
2025-11-19T14:00:19.324022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436042945556259:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:19.324032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:19.325247Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436042945556251:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:19.325336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436038650588868:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.325348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:19.325354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436038650588868:2049] [node 2] ICP22 created new session: [3:7574436042945556261:2049]
2025-11-19T14:00:19.325360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436042945556261:2049] [node 2] ICS09 handshake done sender: [3:7574436042945556251:2119] self: [3:7574436042945556257:0] peer: [2:7574436044633277827:0] socket: 303 qp: -1
2025-11-19T14:00:19.325365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436042945556261:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:19.325370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:19.332340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436043770025879:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.332394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:19.332404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436043770025879:2109] [node 6] ICP22 created new session: [1:7574436043770025915:2109]
2025-11-19T14:00:19.332412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436043770025915:2109] [node 6] ICS09 handshake done sender: [1:7574436043770025895:2116] self: [1:7574436043770025880:0] peer: [6:7574436045398832312:0] socket: 315 qp: -1
2025-11-19T14:00:19.332419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436043770025915:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:19.332433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:19.332681Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436043770025879:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(17952903ns) {HandleHandshakeStatus:329(17951963ns) {DropIncomingHandshake:441(15191ns) DropOutgoingHandshake:456(239ns) SwitchToState:178(111ns) ProcessPendingSessionEvents:495(101219ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 79368ns max# 20200ns}}}
2025-11-19T14:00:19.332995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436045968410623:2065] [node 4] ICP35 event processing took too much time PendingConnection:278(54627207ns) {HandleHandshakeStatus:329(54626619ns) {DropIncomingHandshake:441(683ns) DropOutgoingHandshake:456(6856ns) SwitchToState:178(70ns) ProcessPendingSessionEvents:495(16951747ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16931762ns max# 16868318ns}}}
2025-11-19T14:00:19.335824Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436043770025899:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.336306Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436042945556249:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:19.336288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436043770025876:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.336316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:19.336324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436043770025876:2108] [node 5] ICP22 created new session: [1:7574436043770025918:2108]
2025-11-19T14:00:19.336330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436043770025918:2108] [node 5] ICS09 handshake done sender: [1:7574436043770025899:2117] self: [1:7574436043770025877:0] peer: [5:7574436045968410690:0] socket: 329 qp: -1
2025-11-19T14:00:19.336336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436043770025918:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:19.336344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:19.337255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436042945556233:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.337280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:19.337287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436042945556233:2111] [node 4] ICP22 created new session: [3:7574436042945556264:2111]
2025-11-19T14:00:19.337297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436042945556264:2111] [node 4] ICS09 handshake done sender: [3:7574436042945556249:2118] self: [3:7574436042945556238:0] peer: [4:7574436042522376712:0] socket: 335 qp: -1
2025-11-19T14:00:19.337303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436042945556264:2111] [node 4] ICS10 traffic start
2025-11-19T14:00:19.337313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.338301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436045968410694:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49767
2025-11-19T14:00:19.338173Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436045398832222:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1570) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436045398832310:2118] held: no
2025-11-19T14:00:19.347934Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436045968410688:2121] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:19.348060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436045398832316:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29982
2025-11-19T14:00:19.348112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436045968410596:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.348132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:19.348141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436045968410596:2048] [node 1] ICP22 created new session: [5:7574436045968410695:2048]
2025-11-19T14:00:19.348149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436045968410695:2048] [node 1] ICS09 handshake done sender: [5:7574436045968410688:2121] self: [5:7574436045968410690:0] peer: [1:7574436043770025877:0] socket: 330 qp: -1
2025-11-19T14:00:19.348158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436045968410695:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:19.348173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:19.348263Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436045398832310:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:19.348437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436042522376617:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3391) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436042522376709:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.349321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436042522376713:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54581
2025-11-19T14:00:19.353394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574436045968410596:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:19.352652Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436042522376709:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:19.354119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436042522376617:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.354136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:19.354144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436042522376617:2050] [node 3] ICP22 created new session: [4:7574436042522376714:2050]
2025-11-19T14:00:19.354151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436042522376714:2050] [node 3] ICS09 handshake done sender: [4:7574436042522376709:2119] self: [4:7574436042522376712:0] peer: [3:7574436042945556238:0] socket: 337 qp: -1
2025-11-19T14:00:19.354158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436042522376714:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:19.354168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.358554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436045398832317:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29982
2025-11-19T14:00:19.358831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436045398832222:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.358849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:19.358859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436045398832222:2048] [node 1] ICP22 created new session: [6:7574436045398832318:2048]
2025-11-19T14:00:19.358865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436045398832318:2048] [node 1] ICS09 handshake done sender: [6:7574436045398832310:2118] self: [6:7574436045398832312:0] peer: [1:7574436043770025880:0] socket: 316 qp: -1
2025-11-19T14:00:19.358873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436045398832318:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:19.358899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:19.360234Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436045398832317:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.360281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436045398832235:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:19.360427Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436045968410650:2092] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:19.360440Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436045968410650:2092] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:19.361481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436045398832321:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29982
2025-11-19T14:00:19.361671Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436045968410650:2092] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:19.361854Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436045398832317:2121] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:19.361964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436045398832235:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:19.361981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:19.361985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436045398832235:2059] [node 5] ICP22 created new session: [6:7574436045398832322:2059]
2025-11-19T14:00:19.361991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436045398832322:2059] [node 5] ICS09 handshake done sender: [6:7574436045398832317:2121] self: [6:7574436045398832320:0] peer: [5:7574436045968410649:0] socket: 295 qp: -1
2025-11-19T14:00:19.361996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436045398832322:2059] [node 5] ICS10 traffic start
2025-11-19T14:00:19.362006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:19.363254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436045968410610:2060] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:19.363277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:19.363282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436045968410610:2060] [node 6] ICP22 created new session: [5:7574436045968410698:2060]
2025-11-19T14:00:19.363287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436045968410698:2060] [node 6] ICS09 handshake done sender: [5:7574436045968410650:2092] self: [5:7574436045968410649:0] peer: [6:7574436045398832320:0] socket: 278 qp: -1
2025-11-19T14:00:19.363292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436045968410698:2060] [node 6] ICS10 traffic start
2025-11-19T14:00:19.363301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:19.391854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:19.391965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436044633277866:2048] [node 1] ICS01 socket: 304 reason# EndOfStream
2025-11-19T14:00:19.391975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436040338310471:2048] [node 1] ICP30 unregister session Session# [2:7574436044633277866:2048] VirtualId# [2:7574436044633277856:0]
2025-11-19T14:00:19.391984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436044633277866:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:19.392063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:19.392091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:19.392194Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:19.392265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436042945556253:2048] [node 1] ICS01 socket: 285 reason# EndOfStream
2025-11-19T14:00:19.392271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436038650588867:2048] [node 1] ICP30 unregister session Session# [3:7574436042945556253:2048] VirtualId# [3:7574436042945556247:0]
2025-11-19T14:00:19.392277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436042945556253:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:19.392310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:19.392338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:19.403977Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:19.404081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436042522376710:2048] [node 1] ICS01 socket: 287 reason# EndOfStream
2025-11-19T14:00:19.404092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436042522376615:2048] [node 1] ICP30 unregister session Session# [4:7574436042522376710:2048] VirtualId# [4:7574436042522376706:0]
2025-11-19T14:00:19.404100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436042522376710:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:19.404161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:19.404188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:19.404587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:19.404656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436045398832318:2048] [node 1] ICS01 socket: 316 reason# EndOfStream
2025-11-19T14:00:19.404662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436045398832222:2048] [node 1] ICP30 unregister session Session# [6:7574436045398832318:2048] VirtualId# [6:7574436045398832312:0]
2025-11-19T14:00:19.404667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436045398832318:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:19.404694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:19.404718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:19.404225Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:19.404300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574436045968410695:2048] [node 1] ICS07 socket disconnect 330 reason# EndOfStream
2025-11-19T14:00:19.404305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436045968410695:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:19.404335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:19.404358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574436045968410695:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:19.407140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436045968410596:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1570) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:19.407150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436045968410695:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:19.407155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436045968410596:2048] [node 1] ICP30 unregister session Session# [5:7574436045968410695:2048] VirtualId# [5:7574436045968410690:0]
2025-11-19T14:00:19.407193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:19.407198Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574436045968410596:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1570) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:19.407201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1570) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:19.416123Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:19.416213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436042945556261:2049] [node 2] ICS01 socket: 303 reason# EndOfStream
2025-11-19T14:00:19.416222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436038650588868:2049] [node 2] ICP30 unregister session Session# [3:7574436042945556261:2049] VirtualId# [3:7574436042945556257:0]
2025-11-19T14:00:19.416229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436042945556261:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:19.416273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:19.416294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:19.419923Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:19.420004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436042522376695:2049] [node 2] ICS07 socket disconnect 288 reason# EndOfStream
2025-11-19T14:00:19.420009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436042522376695:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:19.420062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:19.420082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436042522376695:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:19.420380Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:19.420449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436045398832314:2049] [node 2] ICS07 socket disconnect 310 reason# EndOfStream
2025-11-19T14:00:19.420454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436045398832314:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:19.420490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:19.420512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436045398832314:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:19.422950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436045398832223:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1565) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:19.422964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436045398832314:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:19.422969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436045398832223:2049] [node 2] ICP30 unregister session Session# [6:7574436045398832314:2049] VirtualId# [6:7574436045398832311:0]
2025-11-19T14:00:19.423006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:19.423012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436045398832223:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:19.423015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:19.423224Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436042522376616:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1565) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:19.423236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436042522376695:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:19.423240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436042522376616:2049] [node 2] ICP30 unregister session Session# [4:7574436042522376695:2049] VirtualId# [4:7574436042522376693:0]
2025-11-19T14:00:19.423272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:19.423276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436042522376616:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:19.423278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:19.420150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574436045968410596:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:19.420184Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:19.420258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436045968410684:2049] [node 2] ICS01 socket: 319 reason# EndOfStream
2025-11-19T14:00:19.420264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436045968410597:2049] [node 2] ICP30 unregister session Session# [5:7574436045968410684:2049] VirtualId# [5:7574436045968410681:0]
2025-11-19T14:00:19.420269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436045968410684:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:19.420318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:19.420340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:19.432877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574436042522376616:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:19.433214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436045398832223:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:19.439321Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:19.439412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436045398832304:2050] [node 3] ICS07 socket disconnect 272 reason# EndOfStream
2025-11-19T14:00:19.439418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436045398832304:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:19.439481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:19.439514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436045398832304:2050] [node 3] ICS15 start handshake
2025-11-19T14:00:19.439777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:19.439842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436045968410692:2050] [node 3] ICS01 socket: 320 reason# EndOfStream
2025-11-19T14:00:19.439848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436045968410598:2050] [node 3] ICP30 unregister session Session# [5:7574436045968410692:2050] VirtualId# [5:7574436045968410680:0]
2025-11-19T14:00:19.439854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436045968410692:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:19.439888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:19.439914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:19.446993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436045398832224:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3391) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:19.447014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436045398832304:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:19.447021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436045398832224:2050] [node 3] ICP30 unregister session Session# [6:7574436045398832304:2050] VirtualId# [6:7574436045398832243:0]
2025-11-19T14:00:19.447059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:19.447064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436045398832224:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3391) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:19.447069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3391) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:19.457993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:19.458092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436045398832302:2077] [node 4] ICS01 socket: 312 reason# EndOfStream
2025-11-19T14:00:19.458102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436045398832262:2077] [node 4] ICP30 unregister session Session# [6:7574436045398832302:2077] VirtualId# [6:7574436045398832300:0]
2025-11-19T14:00:19.458110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436045398832302:2077] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:19.458176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:19.458203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:19.458489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436045398832224:2050] [node 3] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:19.659909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436044191271235:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:19.659938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436044191271236:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:19.721008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436042430817433:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:19.721023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436042430817434:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:19.797888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436044200430853:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:19.797908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436044200430854:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:19.846673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436045734619839:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:19.846688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436045734619840:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:19.846693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436045734619841:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:19.919913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436042985972483:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:19.919933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436042985972484:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:19.919938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436042985972485:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:19.963736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436044243283303:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:19.963775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436044243283304:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:19.963780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436044243283305:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:20.036112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436048486238594:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:20.036213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436048486238595:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:20.036309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436048486238596:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:20.036037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436048538250632:2060] [node 4] ICP01 ready to work
2025-11-19T14:00:20.036218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436048538250633:2074] [node 5] ICP01 ready to work
2025-11-19T14:00:20.040035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436047280939840:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.052424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436044243283303:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16127458ns) {Configure:104(16126370ns) {ProcessConfigured:135(15755553ns) {StartInitialHandshake:157(15754180ns) {DropHandshakes:469(498ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(92ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:476(2727ns) {SwitchToState:178(1652ns)}}}}}
2025-11-19T14:00:20.060041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436044200430854:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23157156ns) {Configure:104(23156182ns) {ProcessConfigured:135(22785482ns) {StartInitialHandshake:157(22784737ns) {DropHandshakes:469(469ns) {DropIncomingHandshake:441(86ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(1777ns) {SwitchToState:178(832ns)}}}}}
2025-11-19T14:00:20.058535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436048486238594:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(22050908ns) {Configure:104(22050123ns) {ProcessConfigured:135(21638576ns) {StartInitialHandshake:157(21638007ns) {DropHandshakes:469(472ns) {DropIncomingHandshake:441(82ns) DropOutgoingHandshake:456(64ns)} GenerateSessionVirtualId:988(173ns) PrepareNewSessionHandshake:476(1795ns) {SwitchToState:178(1056ns)}}}}}
2025-11-19T14:00:20.059789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436048486238597:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53525
2025-11-19T14:00:20.060240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436048495398213:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:20.060390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436048495398214:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:20.072909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436050029587202:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.073031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436050029587203:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:20.073676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436050029587200:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36114
2025-11-19T14:00:20.073745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436050029587201:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36114
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.074176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436046725784797:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:20.074318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436046725784798:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:20.074468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436046725784799:2111] [node 5] ICP01 ready to work
2025-11-19T14:00:20.076742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436046725784800:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36593
2025-11-19T14:00:20.076786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436046725784801:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36593
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.078284Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436050029587200:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.077056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436048486238611:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53525
2025-11-19T14:00:20.081905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436048495398213:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(21408552ns) {Configure:104(21407909ns) {ProcessConfigured:135(21114786ns) {StartInitialHandshake:157(21113831ns) {DropHandshakes:469(320ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:476(851ns) {SwitchToState:178(285ns)}}}}}
2025-11-19T14:00:20.082676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436048495398222:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:20.083138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436048495398223:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22888
2025-11-19T14:00:20.083185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436048495398224:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22888
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.084910Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436046725784800:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.087217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436048538250677:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8990
2025-11-19T14:00:20.087094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436047280939840:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(46979649ns) {Configure:104(46978801ns) {ProcessConfigured:135(46568949ns) {StartInitialHandshake:157(46568201ns) {DropHandshakes:469(791ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:476(1205ns) {SwitchToState:178(301ns)}}}}}
2025-11-19T14:00:20.088179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436048538250678:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8990
2025-11-19T14:00:20.090653Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048486238611:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.095899Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436048538250678:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.095940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436044243283304:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.096608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436050029587216:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36114
2025-11-19T14:00:20.098171Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436048538250677:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.098211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436048538250632:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.100084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436046725784811:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36593
2025-11-19T14:00:20.103457Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436050029587205:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.103486Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436050029587205:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.103497Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436048495398223:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.107099Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436046725784803:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.107123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436046725784803:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.107642Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436046725784811:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.109037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436048538250681:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8990
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.108256Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436048495398224:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.108295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436044200430854:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.112067Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048486238597:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.116339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436047280939847:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:20.117584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436047280939845:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5696
2025-11-19T14:00:20.117651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436047280939846:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5696
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.118258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436048538250682:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8990
2025-11-19T14:00:20.119696Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436046725784795:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.119712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436046725784795:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.120762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436048486238612:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53525
2025-11-19T14:00:20.120159Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436050029587216:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.120226Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436050029587201:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.120370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436045734619841:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.120380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436045734619839:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.121066Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436050029587205:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:20.123874Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436047280939846:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.123912Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436047280939845:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.125579Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436046725784801:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.125768Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436046725784803:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:20.127311Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436048495398217:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.127335Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436048495398217:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.127573Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048486238599:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.127585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436048486238599:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.128189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436050029587202:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.128223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:20.128234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436050029587202:2108] [node 6] ICP22 created new session: [4:7574436050029587220:2108]
2025-11-19T14:00:20.128242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436050029587220:2108] [node 6] ICS09 handshake done sender: [4:7574436050029587205:2113] self: [4:7574436050029587204:0] peer: [6:7574436048538250680:0] socket: 288 qp: -1
2025-11-19T14:00:20.128251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436050029587220:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:20.128264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:20.129013Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436048538250678:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.129098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436048538250683:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8990
2025-11-19T14:00:20.129204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436044243283304:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.129220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:20.129225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436044243283304:2049] [node 2] ICP22 created new session: [6:7574436048538250684:2049]
2025-11-19T14:00:20.129231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436048538250684:2049] [node 2] ICS09 handshake done sender: [6:7574436048538250678:2116] self: [6:7574436048538250679:0] peer: [2:7574436046725784802:0] socket: 295 qp: -1
2025-11-19T14:00:20.129236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436048538250684:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:20.129241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.130442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436048486238615:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53525
2025-11-19T14:00:20.132183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436046725784797:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.132212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:20.132219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436046725784797:2108] [node 6] ICP22 created new session: [2:7574436046725784817:2108]
2025-11-19T14:00:20.132226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436046725784817:2108] [node 6] ICS09 handshake done sender: [2:7574436046725784803:2114] self: [2:7574436046725784802:0] peer: [6:7574436048538250679:0] socket: 293 qp: -1
2025-11-19T14:00:20.132231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436046725784817:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:20.132238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:20.132652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436042430817433:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.133280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436050029587223:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36114
2025-11-19T14:00:20.133309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436050029587224:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36114
2025-11-19T14:00:20.133483Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436048486238599:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.133866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436046725784814:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36593
2025-11-19T14:00:20.133928Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436046725784795:2110] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:20.133690Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048486238612:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.133705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436048486238594:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.133715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:20.133721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436048486238594:2107] [node 4] ICP22 created new session: [1:7574436048486238617:2107]
2025-11-19T14:00:20.133727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436048486238617:2107] [node 4] ICS09 handshake done sender: [1:7574436048486238599:2111] self: [1:7574436048486238598:0] peer: [4:7574436050029587219:0] socket: 281 qp: -1
2025-11-19T14:00:20.133732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436048486238617:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:20.133738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:20.134275Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436048538250677:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.134116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436042430817434:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.134131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:20.134137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436042430817434:2049] [node 3] ICP22 created new session: [2:7574436046725784821:2049]
2025-11-19T14:00:20.134141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436046725784821:2049] [node 3] ICS09 handshake done sender: [2:7574436046725784795:2110] self: [2:7574436046725784792:0] peer: [3:7574436048495398231:0] socket: 292 qp: -1
2025-11-19T14:00:20.134146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436046725784821:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:20.134151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:20.134364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436046725784816:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36593
2025-11-19T14:00:20.135296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048486238603:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.135304Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436048486238603:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.137666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436048538250632:2060] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.137689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:20.137695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436048538250632:2060] [node 4] ICP22 created new session: [6:7574436048538250687:2060]
2025-11-19T14:00:20.137705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436048538250687:2060] [node 4] ICS09 handshake done sender: [6:7574436048538250677:2115] self: [6:7574436048538250680:0] peer: [4:7574436050029587204:0] socket: 294 qp: -1
2025-11-19T14:00:20.137711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436048538250687:2060] [node 4] ICS10 traffic start
2025-11-19T14:00:20.137732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:20.138154Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436048486238603:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:20.138305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436044191271235:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.138323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:20.138327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436044191271235:2048] [node 2] ICP22 created new session: [1:7574436048486238622:2048]
2025-11-19T14:00:20.138332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436048486238622:2048] [node 2] ICS09 handshake done sender: [1:7574436048486238603:2113] self: [1:7574436048486238602:0] peer: [2:7574436046725784819:0] socket: 283 qp: -1
2025-11-19T14:00:20.138336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436048486238622:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:20.138342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:20.144386Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436048495398217:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.145337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574436048538250632:2060] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:20.145837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436048538250686:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8990
2025-11-19T14:00:20.146911Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436050029587201:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:20.147043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436045734619839:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.147062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:20.147069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436045734619839:2048] [node 1] ICP22 created new session: [4:7574436050029587226:2048]
2025-11-19T14:00:20.147075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436050029587226:2048] [node 1] ICS09 handshake done sender: [4:7574436050029587201:2112] self: [4:7574436050029587219:0] peer: [1:7574436048486238598:0] socket: 286 qp: -1
2025-11-19T14:00:20.147081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436050029587226:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:20.147088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:20.147392Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436050029587216:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.150296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436046725784816:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.150337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436042430817433:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5585) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436046725784801:2113] held: no
2025-11-19T14:00:20.150503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436046725784824:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36593
2025-11-19T14:00:20.150676Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436046725784801:2113] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:20.150742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436042430817433:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.150756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:20.150761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436042430817433:2048] [node 1] ICP22 created new session: [2:7574436046725784826:2048]
2025-11-19T14:00:20.150766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436046725784826:2048] [node 1] ICS09 handshake done sender: [2:7574436046725784801:2113] self: [2:7574436046725784819:0] peer: [1:7574436048486238602:0] socket: 291 qp: -1
2025-11-19T14:00:20.150773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436046725784826:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:20.150781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:20.151160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436048495398213:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.151183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:20.151187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436048495398213:2107] [node 4] ICP22 created new session: [3:7574436048495398236:2107]
2025-11-19T14:00:20.151192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436048495398236:2107] [node 4] ICS09 handshake done sender: [3:7574436048495398217:2111] self: [3:7574436048495398215:0] peer: [4:7574436050029587218:0] socket: 299 qp: -1
2025-11-19T14:00:20.151197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436048495398236:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:20.151204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:20.151420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436044200430854:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61838) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436048495398224:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.151890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436048495398233:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22888
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.154255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436048495398234:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22888
2025-11-19T14:00:20.154277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436048495398235:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22888
2025-11-19T14:00:20.154459Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436048495398224:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.155406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436050029587225:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36114
2025-11-19T14:00:20.155646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436045734619841:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.155659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:20.155665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436045734619841:2050] [node 3] ICP22 created new session: [4:7574436050029587228:2050]
2025-11-19T14:00:20.155671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436050029587228:2050] [node 3] ICS09 handshake done sender: [4:7574436050029587216:2117] self: [4:7574436050029587218:0] peer: [3:7574436048495398215:0] socket: 304 qp: -1
2025-11-19T14:00:20.155677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436050029587228:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:20.155684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:20.156902Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574436045734619841:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:20.156951Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436050029587225:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.156978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436045734619840:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.157091Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436046725784805:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.157102Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436046725784805:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.157838Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436046725784805:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:20.157921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436046725784798:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.157940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:20.157946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436046725784798:2109] [node 4] ICP22 created new session: [2:7574436046725784829:2109]
2025-11-19T14:00:20.157951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436046725784829:2109] [node 4] ICS09 handshake done sender: [2:7574436046725784805:2115] self: [2:7574436046725784804:0] peer: [4:7574436050029587230:0] socket: 280 qp: -1
2025-11-19T14:00:20.157956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436046725784829:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:20.157964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:20.158592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436044200430854:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.158607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:20.158612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436044200430854:2049] [node 2] ICP22 created new session: [3:7574436048495398240:2049]
2025-11-19T14:00:20.158616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436048495398240:2049] [node 2] ICS09 handshake done sender: [3:7574436048495398224:2115] self: [3:7574436048495398231:0] peer: [2:7574436046725784792:0] socket: 297 qp: -1
2025-11-19T14:00:20.158621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436048495398240:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:20.158626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.159079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436050029587231:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36114
2025-11-19T14:00:20.159258Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436050029587225:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:20.160380Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436048495398234:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.172796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436047280939847:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.172834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436042985972483:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.176605Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048486238601:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.176621Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436048486238601:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.176405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436047280939861:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5696
2025-11-19T14:00:20.176450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436047280939862:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5696
2025-11-19T14:00:20.176482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436047280939863:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5696
2025-11-19T14:00:20.177342Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436048486238601:2112] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:20.177445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436048486238595:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.177464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:20.177471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436048486238595:2108] [node 5] ICP22 created new session: [1:7574436048486238626:2108]
2025-11-19T14:00:20.177478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436048486238626:2108] [node 5] ICS09 handshake done sender: [1:7574436048486238601:2112] self: [1:7574436048486238600:0] peer: [5:7574436047280939860:0] socket: 282 qp: -1
2025-11-19T14:00:20.177486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436048486238626:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:20.177498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.177983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436047280939864:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5696
2025-11-19T14:00:20.178063Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048486238615:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.178088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436048486238595:2108] [node 5] ICP18 handshake (actor [1:7574436048486238615:2118]) is too old
2025-11-19T14:00:20.179086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436042985972483:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5585) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436047280939845:2111] held: no
2025-11-19T14:00:20.179189Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436047280939845:2111] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:20.179324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436042985972483:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.179339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:20.179345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436042985972483:2048] [node 1] ICP22 created new session: [5:7574436047280939865:2048]
2025-11-19T14:00:20.179351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436047280939865:2048] [node 1] ICS09 handshake done sender: [5:7574436047280939845:2111] self: [5:7574436047280939860:0] peer: [1:7574436048486238600:0] socket: 301 qp: -1
2025-11-19T14:00:20.179358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436047280939865:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:20.179365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.181382Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436048495398235:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.181420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436044200430853:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.181773Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436048538250686:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.181892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436048538250691:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8990
2025-11-19T14:00:20.181921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436044243283303:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.182323Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436048538250691:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.182341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436044243283305:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.182422Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436048495398227:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.182429Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436048495398227:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.183022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436048538250694:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8990
2025-11-19T14:00:20.183108Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436048495398227:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:20.183287Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436048538250691:2121] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:20.183346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436044243283305:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.183362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:20.183369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436044243283305:2050] [node 3] ICP22 created new session: [6:7574436048538250695:2050]
2025-11-19T14:00:20.183375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436048538250695:2050] [node 3] ICS09 handshake done sender: [6:7574436048538250691:2121] self: [6:7574436048538250693:0] peer: [3:7574436048495398226:0] socket: 322 qp: -1
2025-11-19T14:00:20.183380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436048538250695:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:20.183388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:20.184079Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436047280939862:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.184153Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436047280939861:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.185205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436042985972484:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.185685Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048486238607:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.185700Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436048486238607:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.186216Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048486238605:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.186219Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436048486238605:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.185566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436045734619840:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.185591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:20.185597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436045734619840:2049] [node 2] ICP22 created new session: [4:7574436050029587232:2049]
2025-11-19T14:00:20.185603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436050029587232:2049] [node 2] ICS09 handshake done sender: [4:7574436050029587225:2120] self: [4:7574436050029587230:0] peer: [2:7574436046725784804:0] socket: 326 qp: -1
2025-11-19T14:00:20.185611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436050029587232:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:20.185619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:20.187325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574436045734619840:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:20.187597Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436050029587207:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.187601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436050029587207:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.187944Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436046725784814:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.188008Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436046725784807:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.188014Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436046725784807:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.188478Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436048486238607:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.189613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436048495398222:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.189634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:20.189641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436048495398222:2110] [node 6] ICP22 created new session: [3:7574436048495398246:2110]
2025-11-19T14:00:20.189647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436048495398246:2110] [node 6] ICS09 handshake done sender: [3:7574436048495398227:2116] self: [3:7574436048495398226:0] peer: [6:7574436048538250693:0] socket: 296 qp: -1
2025-11-19T14:00:20.189652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436048495398246:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:20.189661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:20.190332Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436048538250682:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.190375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436050029587234:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36114
2025-11-19T14:00:20.190452Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436050029587207:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:20.190632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436050029587203:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.190644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:20.190649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436050029587203:2109] [node 5] ICP22 created new session: [4:7574436050029587236:2109]
2025-11-19T14:00:20.190653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436050029587236:2109] [node 5] ICS09 handshake done sender: [4:7574436050029587207:2114] self: [4:7574436050029587206:0] peer: [5:7574436047280939859:0] socket: 289 qp: -1
2025-11-19T14:00:20.190659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436050029587236:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:20.190666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:20.189395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436047280939870:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5696
2025-11-19T14:00:20.189414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436047280939871:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5696
2025-11-19T14:00:20.190949Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436047280939863:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.191175Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436047280939846:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:20.191237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436042985972485:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.191244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436047280939847:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.191253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:20.191259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436047280939847:2109] [node 4] ICP22 created new session: [5:7574436047280939873:2109]
2025-11-19T14:00:20.191264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436047280939873:2109] [node 4] ICS09 handshake done sender: [5:7574436047280939846:2112] self: [5:7574436047280939859:0] peer: [4:7574436050029587206:0] socket: 305 qp: -1
2025-11-19T14:00:20.191269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436047280939873:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:20.191274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:20.192046Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436048486238605:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:20.192025Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436050029587234:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.192284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574436050029587203:2109] [node 5] ICP18 handshake (actor [4:7574436050029587234:2122]) is too old
2025-11-19T14:00:20.192164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436044191271236:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.192185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:20.192191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436044191271236:2049] [node 3] ICP22 created new session: [1:7574436048486238630:2049]
2025-11-19T14:00:20.192197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436048486238630:2049] [node 3] ICS09 handshake done sender: [1:7574436048486238607:2115] self: [1:7574436048486238606:0] peer: [3:7574436048495398242:0] socket: 277 qp: -1
2025-11-19T14:00:20.192202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436048486238630:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:20.192210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:20.192413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436048486238596:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.192421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:20.192424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436048486238596:2109] [node 6] ICP22 created new session: [1:7574436048486238632:2109]
2025-11-19T14:00:20.192428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436048486238632:2109] [node 6] ICS09 handshake done sender: [1:7574436048486238605:2114] self: [1:7574436048486238604:0] peer: [6:7574436048538250692:0] socket: 290 qp: -1
2025-11-19T14:00:20.192432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436048486238632:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:20.192436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.192881Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436046725784807:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:20.192678Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436048495398221:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.192688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436048495398221:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.193391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436048495398245:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22888
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.196407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436048538250633:2074] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.196447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436048538250697:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8990
2025-11-19T14:00:20.197866Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436048538250686:2120] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:20.198301Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436047280939844:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.198315Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436047280939844:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.198843Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436047280939861:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:20.199031Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436044200430853:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5585) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436048495398234:2118] held: no
2025-11-19T14:00:20.199225Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436048495398234:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:20.199464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436042985972484:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.199483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:20.199490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436042985972484:2049] [node 2] ICP22 created new session: [5:7574436047280939878:2049]
2025-11-19T14:00:20.199496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436047280939878:2049] [node 2] ICS09 handshake done sender: [5:7574436047280939861:2117] self: [5:7574436047280939869:0] peer: [2:7574436046725784806:0] socket: 330 qp: -1
2025-11-19T14:00:20.199502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436047280939878:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:20.199509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:20.199965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436044243283303:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.199984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:20.199991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436044243283303:2048] [node 1] ICP22 created new session: [6:7574436048538250700:2048]
2025-11-19T14:00:20.199997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436048538250700:2048] [node 1] ICS09 handshake done sender: [6:7574436048538250686:2120] self: [6:7574436048538250692:0] peer: [1:7574436048486238604:0] socket: 279 qp: -1
2025-11-19T14:00:20.200002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436048538250700:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:20.200008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:20.200624Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436047280939844:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.200743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436047280939840:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.200757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:20.200761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436047280939840:2107] [node 6] ICP22 created new session: [5:7574436047280939880:2107]
2025-11-19T14:00:20.200766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436047280939880:2107] [node 6] ICS09 handshake done sender: [5:7574436047280939844:2110] self: [5:7574436047280939841:0] peer: [6:7574436048538250698:0] socket: 308 qp: -1
2025-11-19T14:00:20.200769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436047280939880:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:20.200775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:20.200994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436047280939877:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5696
2025-11-19T14:00:20.203867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436046725784799:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.203900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:20.203908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436046725784799:2111] [node 5] ICP22 created new session: [2:7574436046725784833:2111]
2025-11-19T14:00:20.203915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436046725784833:2111] [node 5] ICS09 handshake done sender: [2:7574436046725784807:2116] self: [2:7574436046725784806:0] peer: [5:7574436047280939869:0] socket: 302 qp: -1
2025-11-19T14:00:20.203924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436046725784833:2111] [node 5] ICS10 traffic start
2025-11-19T14:00:20.203935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.204462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436044200430853:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.204480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:20.204490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436044200430853:2048] [node 1] ICP22 created new session: [3:7574436048495398250:2048]
2025-11-19T14:00:20.204495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436048495398250:2048] [node 1] ICS09 handshake done sender: [3:7574436048495398234:2118] self: [3:7574436048495398242:0] peer: [1:7574436048486238606:0] socket: 328 qp: -1
2025-11-19T14:00:20.204500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436048495398250:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:20.204506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:20.204791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436048495398249:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22888
2025-11-19T14:00:20.206992Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436048495398221:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:20.207112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436048495398214:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.207124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:20.207130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436048495398214:2108] [node 5] ICP22 created new session: [3:7574436048495398252:2108]
2025-11-19T14:00:20.207135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436048495398252:2108] [node 5] ICS09 handshake done sender: [3:7574436048495398221:2113] self: [3:7574436048495398220:0] peer: [5:7574436047280939872:0] socket: 276 qp: -1
2025-11-19T14:00:20.207141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436048495398252:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:20.207148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:20.208598Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436048495398249:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.208629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574436048495398214:2108] [node 5] ICP18 handshake (actor [3:7574436048495398249:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.209124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436048538250699:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8990
2025-11-19T14:00:20.223972Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436042985972485:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26713) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436047280939863:2119] held: no
2025-11-19T14:00:20.224103Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436047280939863:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:20.224932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436048538250633:2074] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16406) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436048538250682:2118] held: no
2025-11-19T14:00:20.225154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436042985972485:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.225175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:20.225183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436042985972485:2050] [node 3] ICP22 created new session: [5:7574436047280939882:2050]
2025-11-19T14:00:20.225190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436047280939882:2050] [node 3] ICS09 handshake done sender: [5:7574436047280939863:2119] self: [5:7574436047280939872:0] peer: [3:7574436048495398220:0] socket: 332 qp: -1
2025-11-19T14:00:20.225199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436047280939882:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:20.225210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:20.225099Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436048538250682:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:20.225667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436048538250633:2074] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.225677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:20.225684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436048538250633:2074] [node 5] ICP22 created new session: [6:7574436048538250702:2074]
2025-11-19T14:00:20.225690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436048538250702:2074] [node 5] ICS09 handshake done sender: [6:7574436048538250682:2118] self: [6:7574436048538250698:0] peer: [5:7574436047280939841:0] socket: 311 qp: -1
2025-11-19T14:00:20.225696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436048538250702:2074] [node 5] ICS10 traffic start
2025-11-19T14:00:20.225705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:20.230477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:20.230586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436046725784826:2048] [node 1] ICS01 socket: 291 reason# EndOfStream
2025-11-19T14:00:20.230597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436042430817433:2048] [node 1] ICP30 unregister session Session# [2:7574436046725784826:2048] VirtualId# [2:7574436046725784819:0]
2025-11-19T14:00:20.230606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436046725784826:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.230652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:20.230675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:20.231130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:20.231207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436048495398250:2048] [node 1] ICS01 socket: 328 reason# EndOfStream
2025-11-19T14:00:20.231213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436044200430853:2048] [node 1] ICP30 unregister session Session# [3:7574436048495398250:2048] VirtualId# [3:7574436048495398242:0]
2025-11-19T14:00:20.231218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436048495398250:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.231256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:20.231278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:20.248946Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:20.249067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436050029587226:2048] [node 1] ICS01 socket: 286 reason# EndOfStream
2025-11-19T14:00:20.249078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436045734619839:2048] [node 1] ICP30 unregister session Session# [4:7574436050029587226:2048] VirtualId# [4:7574436050029587219:0]
2025-11-19T14:00:20.249087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436050029587226:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.249154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:20.249180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:20.254480Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:20.254603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436047280939865:2048] [node 1] ICS01 socket: 301 reason# EndOfStream
2025-11-19T14:00:20.254615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436042985972483:2048] [node 1] ICP30 unregister session Session# [5:7574436047280939865:2048] VirtualId# [5:7574436047280939860:0]
2025-11-19T14:00:20.254624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436047280939865:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.254683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:20.254714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:20.254763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:20.254834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436048538250700:2048] [node 1] ICS01 socket: 279 reason# EndOfStream
2025-11-19T14:00:20.254840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436044243283303:2048] [node 1] ICP30 unregister session Session# [6:7574436048538250700:2048] VirtualId# [6:7574436048538250692:0]
2025-11-19T14:00:20.254845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436048538250700:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.254886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:20.254912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:20.256807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:20.256914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436048495398240:2049] [node 2] ICS01 socket: 297 reason# EndOfStream
2025-11-19T14:00:20.256923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436044200430854:2049] [node 2] ICP30 unregister session Session# [3:7574436048495398240:2049] VirtualId# [3:7574436048495398231:0]
2025-11-19T14:00:20.256931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436048495398240:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.256962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:20.256982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:20.271892Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:20.272018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436048538250684:2049] [node 2] ICS01 socket: 295 reason# EndOfStream
2025-11-19T14:00:20.272029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436044243283304:2049] [node 2] ICP30 unregister session Session# [6:7574436048538250684:2049] VirtualId# [6:7574436048538250679:0]
2025-11-19T14:00:20.272037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436048538250684:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.272097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:20.272124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:20.272163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:20.272228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436050029587232:2049] [node 2] ICS01 socket: 326 reason# EndOfStream
2025-11-19T14:00:20.272233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436045734619840:2049] [node 2] ICP30 unregister session Session# [4:7574436050029587232:2049] VirtualId# [4:7574436050029587230:0]
2025-11-19T14:00:20.272237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436050029587232:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.272263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:20.272283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:20.272314Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:20.272373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436047280939878:2049] [node 2] ICS01 socket: 330 reason# EndOfStream
2025-11-19T14:00:20.272378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436042985972484:2049] [node 2] ICP30 unregister session Session# [5:7574436047280939878:2049] VirtualId# [5:7574436047280939869:0]
2025-11-19T14:00:20.272381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436047280939878:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.272404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:20.272423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:20.283986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:20.284097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436048538250695:2050] [node 3] ICS01 socket: 322 reason# EndOfStream
2025-11-19T14:00:20.284108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436044243283305:2050] [node 3] ICP30 unregister session Session# [6:7574436048538250695:2050] VirtualId# [6:7574436048538250693:0]
2025-11-19T14:00:20.284116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436048538250695:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.284174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:20.284199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:20.284291Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:20.284357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436047280939882:2050] [node 3] ICS01 socket: 332 reason# EndOfStream
2025-11-19T14:00:20.284361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436042985972485:2050] [node 3] ICP30 unregister session Session# [5:7574436047280939882:2050] VirtualId# [5:7574436047280939872:0]
2025-11-19T14:00:20.284365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436047280939882:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.284390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:20.284409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:20.302183Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:20.302302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436048538250687:2060] [node 4] ICS01 socket: 294 reason# EndOfStream
2025-11-19T14:00:20.302312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436048538250632:2060] [node 4] ICP30 unregister session Session# [6:7574436048538250687:2060] VirtualId# [6:7574436048538250680:0]
2025-11-19T14:00:20.302320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436048538250687:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.302383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:20.302410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:20.395965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436048611440780:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:20.395985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436048611440781:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:20.429715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436050399572059:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:20.429737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436050399572060:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:20.483388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436049963809922:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:20.483408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436049963809923:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:20.499891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436046892451213:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:20.499910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436046892451214:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:20.499915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436046892451215:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:20.507847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436049911506159:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:20.507859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436049911506160:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:20.507864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436049911506161:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:20.535123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436049963809940:2064] [node 4] ICP01 ready to work
2025-11-19T14:00:20.535514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436046892451232:2065] [node 5] ICP01 ready to work
2025-11-19T14:00:20.535005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436049911506180:2067] [node 6] ICP01 ready to work
2025-11-19T14:00:20.536162Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436049911506180:2067] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:20.536186Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574436049911506180:2067] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:20.536189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:20.536328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436049911506187:2071] [node 4] ICP01 ready to work
2025-11-19T14:00:20.536570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436046892451243:2070] [node 6] ICP01 ready to work
2025-11-19T14:00:20.536641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436048611440828:2087] [node 6] ICP01 ready to work
2025-11-19T14:00:20.536731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436048611440829:2090] [node 4] ICP01 ready to work
2025-11-19T14:00:20.536048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436049963809948:2069] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.539967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436050399572101:2080] [node 4] ICP01 ready to work
2025-11-19T14:00:20.540076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436050399572102:2082] [node 6] ICP01 ready to work
2025-11-19T14:00:20.540278Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436048611440828:2087] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:20.540342Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [1:7574436048611440828:2087] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:20.540345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:20.540444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436048611440842:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24122
2025-11-19T14:00:20.543931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436048611440827:2082] [node 5] ICP01 ready to work
2025-11-19T14:00:20.538000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436049963809957:2072] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.551515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436046892451243:2070] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address incoming: [0:0:0] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.551986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436050399572095:2078] [node 5] ICP01 ready to work
2025-11-19T14:00:20.550397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436049963809958:2077] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4199
2025-11-19T14:00:20.550435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436049963809959:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4199
2025-11-19T14:00:20.551608Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436046892451243:2070] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:20.551617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:20.557180Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436050399572059:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17037969ns) {Configure:104(17036862ns) {ProcessConfigured:135(16646983ns) {StartInitialHandshake:157(16646327ns) {DropHandshakes:469(496ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:476(1373ns) {SwitchToState:178(785ns)}}}}}
2025-11-19T14:00:20.559649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [1:7574436048611440828:2087] [node 6] ICP33 wake up from error state
2025-11-19T14:00:20.559501Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436049911506161:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(22839251ns) {Configure:104(22838872ns) {ProcessConfigured:135(22636162ns) {StartInitialHandshake:157(22635834ns) {DropHandshakes:469(146ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(15ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(1259ns) {SwitchToState:178(587ns)}}}}}
2025-11-19T14:00:20.559596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574436049911506180:2067] [node 6] ICP33 wake up from error state
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.564026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436046892451258:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47110
2025-11-19T14:00:20.565708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436050399572102:2082] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:20.565781Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574436050399572102:2082] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:20.565786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:20.571807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436046892451259:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47110
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.582477Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436049911506180:2067] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21394572ns) {Configure:104(21394259ns) {ProcessConfigured:135(21387015ns) {StartInitialHandshake:157(21386726ns) {DropHandshakes:469(1201ns) {DropIncomingHandshake:441(380ns) DropOutgoingHandshake:456(344ns)} GenerateSessionVirtualId:988(167ns) PrepareNewSessionHandshake:476(910ns) {SwitchToState:178(265ns)}}}}}
2025-11-19T14:00:20.583102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436049911506225:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8712
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.583501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436049963809998:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4199
2025-11-19T14:00:20.582308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436048611440860:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24122
2025-11-19T14:00:20.582335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436048611440861:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24122
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.586738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574436046892451243:2070] [node 6] ICP33 wake up from error state
2025-11-19T14:00:20.586821Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436046892451259:2083] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.587904Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436049963809959:2078] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.591985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436048611440827:2082] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(47934882ns) {Configure:104(47933584ns) {ProcessConfigured:135(47603502ns) {StartInitialHandshake:157(47602438ns) {DropHandshakes:469(935ns) {DropIncomingHandshake:441(184ns) DropOutgoingHandshake:456(44ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:476(1535ns) {SwitchToState:178(694ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.592389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7574436050399572102:2082] [node 6] ICP33 wake up from error state
2025-11-19T14:00:20.595416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436050399572139:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47987
2025-11-19T14:00:20.597436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436046892451293:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47110
2025-11-19T14:00:20.597481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436046892451294:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47110
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.601771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436049911506241:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8712
2025-11-19T14:00:20.603197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436049911506160:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(28971067ns) {Configure:104(28970186ns) {ProcessConfigured:135(28657777ns) {StartInitialHandshake:157(28657096ns) {DropHandshakes:469(478ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:476(1312ns) {SwitchToState:178(679ns)}}}}}
2025-11-19T14:00:20.604102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436050399572133:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47987
2025-11-19T14:00:20.606628Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436046892451293:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.606663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436046892451214:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.608429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436049963809922:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.609418Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436046892451294:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.609447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436046892451213:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.609620Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436049963809998:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.609671Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436049963809958:2077] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.610284Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436046892451258:2082] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.610315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436046892451215:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.605645Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048611440861:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.609338Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048611440842:2099] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.609771Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436049911506225:2105] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.609821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436049911506161:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.613975Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436049963809961:2079] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.613990Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436049963809961:2079] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.614654Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436049963809961:2079] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.613856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048611440834:2095] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.613882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436048611440834:2095] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.613942Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048611440836:2096] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.613948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436048611440836:2096] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.618500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436049963809957:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.618527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:20.618535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436049963809957:2072] [node 5] ICP22 created new session: [3:7574436049963810008:2072]
2025-11-19T14:00:20.618540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436049963810008:2072] [node 5] ICS09 handshake done sender: [3:7574436049963809961:2079] self: [3:7574436049963809960:0] peer: [5:7574436049911506242:0] socket: 282 qp: -1
2025-11-19T14:00:20.618547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436049963810008:2072] [node 5] ICS10 traffic start
2025-11-19T14:00:20.618558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:20.614895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436049911506243:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8712
2025-11-19T14:00:20.615147Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436049911506225:2105] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:20.615229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436049911506161:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.615248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:20.615257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436049911506161:2050] [node 3] ICP22 created new session: [5:7574436049911506244:2050]
2025-11-19T14:00:20.615264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436049911506244:2050] [node 3] ICS09 handshake done sender: [5:7574436049911506225:2105] self: [5:7574436049911506242:0] peer: [3:7574436049963809960:0] socket: 296 qp: -1
2025-11-19T14:00:20.615274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436049911506244:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:20.615282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:20.616793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436049911506180:2067] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:20.616871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574436049911506180:2067] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:20.616875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61469) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.623667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436046892451300:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47110
2025-11-19T14:00:20.622848Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436050399572133:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.622881Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436050399572139:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.622907Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436050399572123:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.622911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436050399572123:2106] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.623551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436050399572143:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47987
2025-11-19T14:00:20.623922Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436049963809944:2067] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.623940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436049963809944:2067] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.626593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436050399572144:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47987
2025-11-19T14:00:20.626682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436050399572059:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.626747Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436050399572143:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.632815Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436050399572123:2106] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:20.630512Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048611440860:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.630661Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048611440826:2092] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.630667Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436048611440826:2092] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.632420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436048611440867:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24122
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.633439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436049963810011:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4199
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.633663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436046892451301:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47110
2025-11-19T14:00:20.633715Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436050399572144:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.633765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436050399572101:2080] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.633785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:20.633793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436050399572101:2080] [node 4] ICP22 created new session: [2:7574436050399572150:2080]
2025-11-19T14:00:20.633799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436050399572150:2080] [node 4] ICS09 handshake done sender: [2:7574436050399572123:2106] self: [2:7574436050399572122:0] peer: [4:7574436046892451297:0] socket: 293 qp: -1
2025-11-19T14:00:20.633808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436050399572150:2080] [node 4] ICS10 traffic start
2025-11-19T14:00:20.633834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:20.633883Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436046892451293:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:20.634076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436046892451214:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.634094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:20.634099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436046892451214:2049] [node 2] ICP22 created new session: [4:7574436046892451302:2049]
2025-11-19T14:00:20.634106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436046892451302:2049] [node 2] ICS09 handshake done sender: [4:7574436046892451293:2117] self: [4:7574436046892451297:0] peer: [2:7574436050399572122:0] socket: 294 qp: -1
2025-11-19T14:00:20.634113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436046892451302:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:20.634119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:20.623979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436049813691124:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:20.624027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436049813691125:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:20.624142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436049813691123:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:20.624330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436049813691186:2062] [node 5] ICP01 ready to work
2025-11-19T14:00:20.624462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436049813691187:2075] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.634497Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436049911506241:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.634534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436049911506160:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.634702Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436048611440834:2095] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:20.634880Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436049963809959:2078] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:20.634950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436049963809922:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.634965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:20.634970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436049963809922:2048] [node 1] ICP22 created new session: [3:7574436049963810012:2048]
2025-11-19T14:00:20.634975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436049963810012:2048] [node 1] ICS09 handshake done sender: [3:7574436049963809959:2078] self: [3:7574436049963810004:0] peer: [1:7574436048611440833:0] socket: 289 qp: -1
2025-11-19T14:00:20.634981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436049963810012:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:20.634988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:20.636640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436050399572149:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47987
2025-11-19T14:00:20.636717Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436050399572127:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.636721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436050399572127:2108] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.637715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436049911506249:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8712
2025-11-19T14:00:20.637871Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436049963809944:2067] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:20.638001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436049963809940:2064] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.638023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:20.638029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436049963809940:2064] [node 4] ICP22 created new session: [3:7574436049963810016:2064]
2025-11-19T14:00:20.638036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436049963810016:2064] [node 4] ICS09 handshake done sender: [3:7574436049963809944:2067] self: [3:7574436049963809943:0] peer: [4:7574436046892451299:0] socket: 279 qp: -1
2025-11-19T14:00:20.638042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436049963810016:2064] [node 4] ICS10 traffic start
2025-11-19T14:00:20.638050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:20.640066Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436050399572127:2108] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:20.640233Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436049911506241:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.640928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436049813691195:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7664
2025-11-19T14:00:20.640991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436049813691196:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7664
2025-11-19T14:00:20.640377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436050399572095:2078] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.640397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:20.640405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436050399572095:2078] [node 5] ICP22 created new session: [2:7574436050399572155:2078]
2025-11-19T14:00:20.640412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436050399572155:2078] [node 5] ICS09 handshake done sender: [2:7574436050399572127:2108] self: [2:7574436050399572126:0] peer: [5:7574436049911506248:0] socket: 303 qp: -1
2025-11-19T14:00:20.640419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436050399572155:2078] [node 5] ICS10 traffic start
2025-11-19T14:00:20.640428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:20.642043Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436048611440826:2092] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:20.642123Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436049813691125:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17517772ns) {Configure:104(17516842ns) {ProcessConfigured:135(17123078ns) {StartInitialHandshake:157(17122125ns) {DropHandshakes:469(1265ns) {DropIncomingHandshake:441(305ns) DropOutgoingHandshake:456(140ns)} GenerateSessionVirtualId:988(272ns) PrepareNewSessionHandshake:476(1035ns) {SwitchToState:178(276ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.642061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436046892451215:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26384) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436046892451258:2082] held: no
2025-11-19T14:00:20.642242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436046892451304:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47110
2025-11-19T14:00:20.642305Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436048611440836:2096] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.642420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436046892451305:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47110
2025-11-19T14:00:20.642608Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436046892451258:2082] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:20.642736Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436046892451294:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:20.642410Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048611440867:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.642422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436048611440781:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.642439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:20.642445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436048611440781:2049] [node 3] ICP22 created new session: [1:7574436048611440872:2049]
2025-11-19T14:00:20.642450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436048611440872:2049] [node 3] ICS09 handshake done sender: [1:7574436048611440834:2095] self: [1:7574436048611440833:0] peer: [3:7574436049963810004:0] socket: 285 qp: -1
2025-11-19T14:00:20.642455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436048611440872:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:20.642462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:20.642854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436048611440780:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.642864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:20.642868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436048611440780:2048] [node 2] ICP22 created new session: [1:7574436048611440874:2048]
2025-11-19T14:00:20.642872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436048611440874:2048] [node 2] ICS09 handshake done sender: [1:7574436048611440826:2092] self: [1:7574436048611440825:0] peer: [2:7574436050399572146:0] socket: 283 qp: -1
2025-11-19T14:00:20.642875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436048611440874:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:20.642881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:20.643052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436048611440829:2090] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.643060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:20.643064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436048611440829:2090] [node 4] ICP22 created new session: [1:7574436048611440876:2090]
2025-11-19T14:00:20.643068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436048611440876:2090] [node 4] ICS09 handshake done sender: [1:7574436048611440836:2096] self: [1:7574436048611440835:0] peer: [4:7574436046892451298:0] socket: 295 qp: -1
2025-11-19T14:00:20.643071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436048611440876:2090] [node 4] ICS10 traffic start
2025-11-19T14:00:20.643076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.643960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436049911506160:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.643980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:20.643985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436049911506160:2049] [node 2] ICP22 created new session: [5:7574436049911506250:2049]
2025-11-19T14:00:20.643990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436049911506250:2049] [node 2] ICS09 handshake done sender: [5:7574436049911506241:2117] self: [5:7574436049911506248:0] peer: [2:7574436050399572126:0] socket: 306 qp: -1
2025-11-19T14:00:20.643995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436049911506250:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:20.644001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:20.645026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436049911506252:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8712
2025-11-19T14:00:20.645039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436049911506253:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8712
2025-11-19T14:00:20.644106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436046892451215:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.644124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:20.644130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436046892451215:2050] [node 3] ICP22 created new session: [4:7574436046892451307:2050]
2025-11-19T14:00:20.644135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436046892451307:2050] [node 3] ICS09 handshake done sender: [4:7574436046892451258:2082] self: [4:7574436046892451299:0] peer: [3:7574436049963809943:0] socket: 286 qp: -1
2025-11-19T14:00:20.644142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436046892451307:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:20.644148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:20.644349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436046892451213:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.644356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:20.644360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436046892451213:2048] [node 1] ICP22 created new session: [4:7574436046892451309:2048]
2025-11-19T14:00:20.644364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436046892451309:2048] [node 1] ICS09 handshake done sender: [4:7574436046892451294:2118] self: [4:7574436046892451298:0] peer: [1:7574436048611440835:0] socket: 304 qp: -1
2025-11-19T14:00:20.644369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436046892451309:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:20.644373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.648257Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436049911506253:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.648353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436049911506187:2071] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.648370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436049911506254:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8712
2025-11-19T14:00:20.648516Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436046892451279:2103] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.648523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436046892451279:2103] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.651966Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436049963809971:2087] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.651982Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436049963809971:2087] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.647880Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436049813691195:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.648013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436049813691125:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.653718Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436049963809971:2087] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:20.653834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436049963809948:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.653852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:20.653861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436049963809948:2069] [node 6] ICP22 created new session: [3:7574436049963810019:2069]
2025-11-19T14:00:20.653868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436049963810019:2069] [node 6] ICS09 handshake done sender: [3:7574436049963809971:2087] self: [3:7574436049963809953:0] peer: [6:7574436049813691203:0] socket: 324 qp: -1
2025-11-19T14:00:20.653875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436049963810019:2069] [node 6] ICS10 traffic start
2025-11-19T14:00:20.653885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.654571Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436050399572059:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14942) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436050399572133:2110] held: no
2025-11-19T14:00:20.654689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436050399572157:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47987
2025-11-19T14:00:20.654528Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436049911506254:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.654562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436049911506159:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.654965Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436050399572133:2110] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:20.654719Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048611440864:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.654730Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436048611440864:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.655473Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436048611440864:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:20.656951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436048611440827:2082] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.656970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:20.656975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436048611440827:2082] [node 5] ICP22 created new session: [1:7574436048611440882:2082]
2025-11-19T14:00:20.656981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436048611440882:2082] [node 5] ICS09 handshake done sender: [1:7574436048611440864:2118] self: [1:7574436048611440849:0] peer: [5:7574436049911506256:0] socket: 298 qp: -1
2025-11-19T14:00:20.656986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436048611440882:2082] [node 5] ICS10 traffic start
2025-11-19T14:00:20.656993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:20.655358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436049911506257:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8712
2025-11-19T14:00:20.655665Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436049911506254:2122] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:20.655738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436049911506159:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.655763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:20.655768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436049911506159:2048] [node 1] ICP22 created new session: [5:7574436049911506258:2048]
2025-11-19T14:00:20.655773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436049911506258:2048] [node 1] ICS09 handshake done sender: [5:7574436049911506254:2122] self: [5:7574436049911506256:0] peer: [1:7574436048611440849:0] socket: 334 qp: -1
2025-11-19T14:00:20.655778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436049911506258:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:20.655786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.660501Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436049813691196:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.660665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436049813691204:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7664
2025-11-19T14:00:20.660704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436049813691205:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7664
2025-11-19T14:00:20.661136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436049813691124:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.663951Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436049911506252:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.663973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574436049911506180:2067] [node 6] ICP33 wake up from error state
2025-11-19T14:00:20.664067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436049911506180:2067] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.662001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436049963810021:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4199
2025-11-19T14:00:20.663271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436050399572059:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.663300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:20.663308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436050399572059:2048] [node 1] ICP22 created new session: [2:7574436050399572159:2048]
2025-11-19T14:00:20.663317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436050399572159:2048] [node 1] ICS09 handshake done sender: [2:7574436050399572133:2110] self: [2:7574436050399572146:0] peer: [1:7574436048611440825:0] socket: 291 qp: -1
2025-11-19T14:00:20.663325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436050399572159:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:20.663336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:20.663637Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436050399572141:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.663642Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436050399572141:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.664715Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436050399572141:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:20.665042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436050399572102:2082] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.665056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:20.665064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436050399572102:2082] [node 6] ICP22 created new session: [2:7574436050399572162:2082]
2025-11-19T14:00:20.665069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436050399572162:2082] [node 6] ICS09 handshake done sender: [2:7574436050399572141:2117] self: [2:7574436050399572140:0] peer: [6:7574436049813691206:0] socket: 325 qp: -1
2025-11-19T14:00:20.665076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436050399572162:2082] [node 6] ICS10 traffic start
2025-11-19T14:00:20.665096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:20.665360Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436049963810021:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.665395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436049963809923:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.665619Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436049813691204:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.665733Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436049813691190:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.665736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574436049813691190:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.666622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436049813691123:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.666643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436049813691207:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7664
2025-11-19T14:00:20.664818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436049911506261:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8712
2025-11-19T14:00:20.665542Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436046892451279:2103] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.667917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436049963810023:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4199
2025-11-19T14:00:20.666821Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436050399572129:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.666829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436050399572129:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.667933Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436046892451305:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.667975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436046892451243:2070] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:20.668197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436046892451232:2065] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.667490Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436050399572157:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.667587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436050399572102:2082] [node 6] ICP18 handshake (actor [2:7574436050399572157:2121]) is too old
2025-11-19T14:00:20.668221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:20.668230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436046892451232:2065] [node 5] ICP22 created new session: [4:7574436046892451313:2065]
2025-11-19T14:00:20.668237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436046892451313:2065] [node 5] ICS09 handshake done sender: [4:7574436046892451279:2103] self: [4:7574436046892451234:0] peer: [5:7574436049911506255:0] socket: 305 qp: -1
2025-11-19T14:00:20.668244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436046892451313:2065] [node 5] ICS10 traffic start
2025-11-19T14:00:20.668252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:20.668404Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436049911506253:2121] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:20.668481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436049911506262:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8712
2025-11-19T14:00:20.669051Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436050399572129:2109] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:20.669227Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436049963810021:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:20.669341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436050399572060:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.669356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:20.669361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436050399572060:2049] [node 3] ICP22 created new session: [2:7574436050399572165:2049]
2025-11-19T14:00:20.669367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436050399572165:2049] [node 3] ICS09 handshake done sender: [2:7574436050399572129:2109] self: [2:7574436050399572128:0] peer: [3:7574436049963810022:0] socket: 290 qp: -1
2025-11-19T14:00:20.669372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436050399572165:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:20.669379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:20.669870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436049911506187:2071] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1720) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436049911506253:2121] held: no
2025-11-19T14:00:20.669959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436049911506187:2071] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.669968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:20.669972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436049911506187:2071] [node 4] ICP22 created new session: [5:7574436049911506263:2071]
2025-11-19T14:00:20.669978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436049911506263:2071] [node 4] ICS09 handshake done sender: [5:7574436049911506253:2121] self: [5:7574436049911506255:0] peer: [4:7574436046892451234:0] socket: 333 qp: -1
2025-11-19T14:00:20.669983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436049911506263:2071] [node 4] ICS10 traffic start
2025-11-19T14:00:20.669991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:20.670362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436049963809923:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.670379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:20.670384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436049963809923:2049] [node 2] ICP22 created new session: [3:7574436049963810024:2049]
2025-11-19T14:00:20.670389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436049963810024:2049] [node 2] ICS09 handshake done sender: [3:7574436049963810021:2119] self: [3:7574436049963810022:0] peer: [2:7574436050399572128:0] socket: 292 qp: -1
2025-11-19T14:00:20.670394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436049963810024:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:20.670401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:20.675068Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436049813691190:2110] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:20.676023Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436049911506252:2120] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:20.676235Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048611440863:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.676247Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436048611440863:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.676185Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436049813691192:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.676194Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574436049813691192:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:20.678269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436049813691124:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29627) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436049813691196:2114] held: no
2025-11-19T14:00:20.678384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574436049813691186:2062] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.678409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:20.678415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436049813691186:2062] [node 5] ICP22 created new session: [6:7574436049813691211:2062]
2025-11-19T14:00:20.678422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436049813691211:2062] [node 5] ICS09 handshake done sender: [6:7574436049813691190:2110] self: [6:7574436049813691189:0] peer: [5:7574436049911506260:0] socket: 328 qp: -1
2025-11-19T14:00:20.678431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436049813691211:2062] [node 5] ICS10 traffic start
2025-11-19T14:00:20.678439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:20.678662Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436049813691196:2114] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:20.679302Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436049813691195:2113] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:20.678833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436049911506180:2067] [node 6] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.678853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:20.678860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436049911506180:2067] [node 6] ICP22 created new session: [5:7574436049911506265:2067]
2025-11-19T14:00:20.678866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436049911506265:2067] [node 6] ICS09 handshake done sender: [5:7574436049911506252:2120] self: [5:7574436049911506260:0] peer: [6:7574436049813691189:0] socket: 281 qp: -1
2025-11-19T14:00:20.678871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436049911506265:2067] [node 6] ICS10 traffic start
2025-11-19T14:00:20.678878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:20.679542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436049813691124:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.679556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:20.679562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436049813691124:2049] [node 2] ICP22 created new session: [6:7574436049813691213:2049]
2025-11-19T14:00:20.679567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436049813691213:2049] [node 2] ICS09 handshake done sender: [6:7574436049813691196:2114] self: [6:7574436049813691206:0] peer: [2:7574436050399572140:0] socket: 288 qp: -1
2025-11-19T14:00:20.679572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436049813691213:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:20.679577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:20.679683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436049813691125:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.679690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:20.679694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436049813691125:2050] [node 3] ICP22 created new session: [6:7574436049813691215:2050]
2025-11-19T14:00:20.679698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436049813691215:2050] [node 3] ICS09 handshake done sender: [6:7574436049813691195:2113] self: [6:7574436049813691203:0] peer: [3:7574436049963809953:0] socket: 284 qp: -1
2025-11-19T14:00:20.679703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436049813691215:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:20.679708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:20.680824Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436048611440863:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:20.680928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436048611440828:2087] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.680951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:20.680957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436048611440828:2087] [node 6] ICP22 created new session: [1:7574436048611440885:2087]
2025-11-19T14:00:20.680962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436048611440885:2087] [node 6] ICS09 handshake done sender: [1:7574436048611440863:2117] self: [1:7574436048611440862:0] peer: [6:7574436049813691209:0] socket: 300 qp: -1
2025-11-19T14:00:20.680968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436048611440885:2087] [node 6] ICS10 traffic start
2025-11-19T14:00:20.680976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.681600Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436049813691192:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.681699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436046892451315:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47110
2025-11-19T14:00:20.682138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436049813691218:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7664
2025-11-19T14:00:20.682478Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436049813691204:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:20.683914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436048611440887:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24122
2025-11-19T14:00:20.684439Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436046892451305:2122] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:20.684446Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436048611440887:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:20.684479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436048611440828:2087] [node 6] ICP18 handshake (actor [1:7574436048611440887:2120]) is too old
2025-11-19T14:00:20.684553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436046892451243:2070] [node 6] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.684564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:20.684570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436046892451243:2070] [node 6] ICP22 created new session: [4:7574436046892451316:2070]
2025-11-19T14:00:20.684575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436046892451316:2070] [node 6] ICS09 handshake done sender: [4:7574436046892451305:2122] self: [4:7574436046892451312:0] peer: [6:7574436049813691191:0] socket: 332 qp: -1
2025-11-19T14:00:20.684580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436046892451316:2070] [node 6] ICS10 traffic start
2025-11-19T14:00:20.684593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:20.683740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436049813691123:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:20.684143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574436049813691187:2075] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:20.684152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:20.684157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436049813691187:2075] [node 4] ICP22 created new session: [6:7574436049813691219:2075]
2025-11-19T14:00:20.684162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436049813691219:2075] [node 4] ICS09 handshake done sender: [6:7574436049813691192:2111] self: [6:7574436049813691191:0] peer: [4:7574436046892451312:0] socket: 329 qp: -1
2025-11-19T14:00:20.684167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436049813691219:2075] [node 4] ICS10 traffic start
2025-11-19T14:00:20.684173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:20.684749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:20.684756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436049813691123:2048] [node 1] ICP22 created new session: [6:7574436049813691221:2048]
2025-11-19T14:00:20.684761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436049813691221:2048] [node 1] ICS09 handshake done sender: [6:7574436049813691204:2117] self: [6:7574436049813691209:0] peer: [1:7574436048611440862:0] socket: 310 qp: -1
2025-11-19T14:00:20.684766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436049813691221:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:20.684771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:20.693514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:20.693617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436050399572159:2048] [node 1] ICS01 socket: 291 reason# EndOfStream
2025-11-19T14:00:20.693626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436050399572059:2048] [node 1] ICP30 unregister session Session# [2:7574436050399572159:2048] VirtualId# [2:7574436050399572146:0]
2025-11-19T14:00:20.693634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436050399572159:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.693667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:20.693687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:20.694125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:20.694193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436049963810012:2048] [node 1] ICS01 socket: 289 reason# EndOfStream
2025-11-19T14:00:20.694198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436049963809922:2048] [node 1] ICP30 unregister session Session# [3:7574436049963810012:2048] VirtualId# [3:7574436049963810004:0]
2025-11-19T14:00:20.694202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436049963810012:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.694227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:20.694242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:20.703336Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:20.703445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436049911506258:2048] [node 1] ICS01 socket: 334 reason# EndOfStream
2025-11-19T14:00:20.703454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436049911506159:2048] [node 1] ICP30 unregister session Session# [5:7574436049911506258:2048] VirtualId# [5:7574436049911506256:0]
2025-11-19T14:00:20.703463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436049911506258:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.703515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:20.703560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:20.704108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:20.704185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436049813691221:2048] [node 1] ICS01 socket: 310 reason# EndOfStream
2025-11-19T14:00:20.704191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436049813691123:2048] [node 1] ICP30 unregister session Session# [6:7574436049813691221:2048] VirtualId# [6:7574436049813691209:0]
2025-11-19T14:00:20.704197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436049813691221:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.704247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:20.704271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:20.704413Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:20.704477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436046892451309:2048] [node 1] ICS01 socket: 304 reason# EndOfStream
2025-11-19T14:00:20.704481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436046892451213:2048] [node 1] ICP30 unregister session Session# [4:7574436046892451309:2048] VirtualId# [4:7574436046892451298:0]
2025-11-19T14:00:20.704486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436046892451309:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.704510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:20.704530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:20.712418Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:20.712525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436049963810024:2049] [node 2] ICS01 socket: 292 reason# EndOfStream
2025-11-19T14:00:20.712535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436049963809923:2049] [node 2] ICP30 unregister session Session# [3:7574436049963810024:2049] VirtualId# [3:7574436049963810022:0]
2025-11-19T14:00:20.712543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436049963810024:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.712588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:20.712610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:20.723991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:20.724115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436046892451302:2049] [node 2] ICS01 socket: 294 reason# EndOfStream
2025-11-19T14:00:20.724127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436046892451214:2049] [node 2] ICP30 unregister session Session# [4:7574436046892451302:2049] VirtualId# [4:7574436046892451297:0]
2025-11-19T14:00:20.724136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436046892451302:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.724181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:20.724207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:20.725988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:20.726060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436049911506250:2049] [node 2] ICS01 socket: 306 reason# EndOfStream
2025-11-19T14:00:20.726066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436049911506160:2049] [node 2] ICP30 unregister session Session# [5:7574436049911506250:2049] VirtualId# [5:7574436049911506248:0]
2025-11-19T14:00:20.726070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436049911506250:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.726103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:20.726131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:20.725730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:20.725815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436049813691213:2049] [node 2] ICS01 socket: 288 reason# EndOfStream
2025-11-19T14:00:20.725822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436049813691124:2049] [node 2] ICP30 unregister session Session# [6:7574436049813691213:2049] VirtualId# [6:7574436049813691206:0]
2025-11-19T14:00:20.725828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436049813691213:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.725881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:20.725911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:20.752789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:20.752904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436049813691215:2050] [node 3] ICS07 socket disconnect 284 reason# EndOfStream
2025-11-19T14:00:20.752913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436049813691215:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.752973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:20.752993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436049813691215:2050] [node 3] ICS15 start handshake
2025-11-19T14:00:20.753207Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:20.753275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436049911506244:2050] [node 3] ICS01 socket: 296 reason# EndOfStream
2025-11-19T14:00:20.753281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436049911506161:2050] [node 3] ICP30 unregister session Session# [5:7574436049911506244:2050] VirtualId# [5:7574436049911506242:0]
2025-11-19T14:00:20.753287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436049911506244:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.753313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:20.753337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:20.753815Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436049813691125:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26384) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:20.753827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436049813691215:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:20.753831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436049813691125:2050] [node 3] ICP30 unregister session Session# [6:7574436049813691215:2050] VirtualId# [6:7574436049813691203:0]
2025-11-19T14:00:20.753874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:20.753880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436049813691125:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26384) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:20.753883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26384) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:20.763933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436049813691125:2050] [node 3] ICP33 wake up from error state
2025-11-19T14:00:20.769043Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T14:00:20.769156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436049911506263:2071] [node 4] ICS01 socket: 333 reason# EndOfStream
2025-11-19T14:00:20.769167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436049911506187:2071] [node 4] ICP30 unregister session Session# [5:7574436049911506263:2071] VirtualId# [5:7574436049911506255:0]
2025-11-19T14:00:20.769175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436049911506263:2071] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.769220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:20.769243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:20.774955Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:20.775078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436049813691219:2075] [node 4] ICS01 socket: 329 reason# EndOfStream
2025-11-19T14:00:20.775089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436049813691187:2075] [node 4] ICP30 unregister session Session# [6:7574436049813691219:2075] VirtualId# [6:7574436049813691191:0]
2025-11-19T14:00:20.775098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436049813691219:2075] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:20.775150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:20.775189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:20.959091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436047995405869:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:20.959110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436047995405870:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:21.012883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436048380272184:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:21.012901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436048380272185:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:21.112689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436052301032828:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:21.112714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436052301032829:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:21.163911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436051477174209:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:21.163934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436051477174210:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:21.163940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436051477174211:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:21.239179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436051546888815:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:21.239200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436051546888816:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:21.239205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436051546888817:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:21.305995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436051701796091:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:21.309890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436051701796092:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:21.309936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436051701796093:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:21.340257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436051546888855:2083] [node 4] ICP01 ready to work
2025-11-19T14:00:21.340568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436052290373227:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:21.340662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436052290373228:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:21.340429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436051477174266:2103] [node 5] ICP01 ready to work
2025-11-19T14:00:21.340719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436051701796110:2065] [node 4] ICP01 ready to work
2025-11-19T14:00:21.347949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436051546888860:2088] [node 6] ICP01 ready to work
2025-11-19T14:00:21.356365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436052675239547:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:21.363959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436052675239552:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:21.364197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436052301032891:2106] [node 6] ICP01 ready to work
2025-11-19T14:00:21.364308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436052301032892:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.365465Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436052290373227:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(24440919ns) {Configure:104(24438953ns) {ProcessConfigured:135(23076214ns) {StartInitialHandshake:157(23075868ns) {DropHandshakes:469(226ns) {DropIncomingHandshake:441(22ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(143ns) PrepareNewSessionHandshake:476(1135ns) {SwitchToState:178(745ns)}}}}}
2025-11-19T14:00:21.365692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436052290373234:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:21.366601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436052290373235:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34079
2025-11-19T14:00:21.366615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436052290373236:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34079
2025-11-19T14:00:21.367005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436052290373228:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25761049ns) {Configure:104(25759779ns) {ProcessConfigured:135(24596539ns) {StartInitialHandshake:157(24596019ns) {DropHandshakes:469(370ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(101ns)} GenerateSessionVirtualId:988(287ns) PrepareNewSessionHandshake:476(644ns) {SwitchToState:178(479ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.367271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436048380272184:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25057794ns) {Configure:104(25057377ns) {ProcessConfigured:135(24769211ns) {StartInitialHandshake:157(24768960ns) {DropHandshakes:469(400ns) {DropIncomingHandshake:441(88ns) DropOutgoingHandshake:456(46ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:476(740ns) {SwitchToState:178(93ns)}}}}}
2025-11-19T14:00:21.368515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436052675239557:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8983
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.368625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436051546888817:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(28466561ns) {Configure:104(28465298ns) {ProcessConfigured:135(26049969ns) {StartInitialHandshake:157(26049404ns) {DropHandshakes:469(258ns) {DropIncomingHandshake:441(16ns) DropOutgoingHandshake:456(31ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:476(659ns) {SwitchToState:178(477ns)}}}}}
2025-11-19T14:00:21.369521Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436051546888855:2083] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(29197160ns) {Configure:104(29196670ns) {ProcessConfigured:135(26872767ns) {StartInitialHandshake:157(26872452ns) {DropHandshakes:469(599ns) {DropIncomingHandshake:441(18ns) DropOutgoingHandshake:456(98ns)} GenerateSessionVirtualId:988(42ns) PrepareNewSessionHandshake:476(612ns) {SwitchToState:178(358ns)}}}}}
2025-11-19T14:00:21.369620Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436051701796110:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(26369987ns) {Configure:104(26368353ns) {ProcessConfigured:135(26080056ns) {StartInitialHandshake:157(26079850ns) {DropHandshakes:469(506ns) {DropIncomingHandshake:441(88ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:476(667ns) {SwitchToState:178(230ns)}}}}}
2025-11-19T14:00:21.369744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436051701796143:2073] [node 5] ICP01 ready to work
2025-11-19T14:00:21.370282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436051701796144:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11306
2025-11-19T14:00:21.370326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436051701796145:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11306
2025-11-19T14:00:21.370483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436051701796092:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(26890246ns) {Configure:104(26889897ns) {ProcessConfigured:135(26671017ns) {StartInitialHandshake:157(26670879ns) {DropHandshakes:469(190ns) {DropIncomingHandshake:441(13ns) DropOutgoingHandshake:456(40ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:476(515ns) {SwitchToState:178(35ns)}}}}}
2025-11-19T14:00:21.380952Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436051701796144:2094] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.381003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436051701796092:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:21.381173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436052301032899:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.383035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436051546888892:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19447
2025-11-19T14:00:21.383092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436051546888893:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19447
2025-11-19T14:00:21.383120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436051546888894:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19447
2025-11-19T14:00:21.383130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436051546888895:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19447
2025-11-19T14:00:21.383141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436051546888896:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19447
2025-11-19T14:00:21.383787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436052675239546:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.383794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436051701796093:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(42405861ns) {Configure:104(42405356ns) {ProcessConfigured:135(42184765ns) {StartInitialHandshake:157(42184467ns) {DropHandshakes:469(226ns) {DropIncomingHandshake:441(43ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:476(1230ns) {SwitchToState:178(49ns)}}}}}
2025-11-19T14:00:21.382061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436052301032891:2106] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(17356488ns) {Configure:104(17355994ns) {ProcessConfigured:135(17157436ns) {StartInitialHandshake:157(17156681ns) {DropHandshakes:469(566ns) {DropIncomingHandshake:441(16ns) DropOutgoingHandshake:456(33ns)} GenerateSessionVirtualId:988(38ns) PrepareNewSessionHandshake:476(848ns) {SwitchToState:178(388ns)}}}}}
2025-11-19T14:00:21.384750Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436052675239554:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.384765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436052675239554:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:21.386605Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436051546888892:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.386642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436051546888855:2083] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:21.384415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436052290373245:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34079
2025-11-19T14:00:21.399943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436047995405870:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(59203529ns) {Configure:104(59201924ns) {ProcessConfigured:135(57711110ns) {StartInitialHandshake:157(57710278ns) {DropHandshakes:469(487ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(94ns) PrepareNewSessionHandshake:476(2086ns) {SwitchToState:178(1127ns)}}}}}
2025-11-19T14:00:21.400033Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436052290373236:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.403240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436051546888816:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(30246770ns) {Configure:104(30245862ns) {ProcessConfigured:135(29931216ns) {StartInitialHandshake:157(29930333ns) {DropHandshakes:469(880ns) {DropIncomingHandshake:441(174ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(187ns) PrepareNewSessionHandshake:476(2139ns) {SwitchToState:178(366ns)}}}}}
2025-11-19T14:00:21.409108Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436051701796145:2095] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.409167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436051701796143:2073] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:21.409064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436051477174284:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:21.409563Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436051477174268:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.409576Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436051477174268:2104] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:21.409013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436052290373246:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34079
2025-11-19T14:00:21.409302Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436052290373245:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.417343Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436052675239546:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(33418983ns) {Configure:104(33417946ns) {ProcessConfigured:135(33080647ns) {StartInitialHandshake:157(33079689ns) {DropHandshakes:469(1035ns) {DropIncomingHandshake:441(309ns) DropOutgoingHandshake:456(95ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:476(1456ns) {SwitchToState:178(755ns)}}}}}
2025-11-19T14:00:21.417825Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436051546888893:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.417867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436051546888816:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:21.417105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436052301032907:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55371
2025-11-19T14:00:21.417159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436052301032908:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55371
2025-11-19T14:00:21.417193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436052301032909:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55371
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.420062Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436052675239549:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.420076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436052675239549:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:21.420465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436052675239560:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8983
2025-11-19T14:00:21.421285Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436052675239549:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.421563Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436052675239554:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.421775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436052675239547:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.421794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:21.421801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436052675239547:2109] [node 5] ICP22 created new session: [2:7574436052675239563:2109]
2025-11-19T14:00:21.421808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436052675239563:2109] [node 5] ICS09 handshake done sender: [2:7574436052675239549:2110] self: [2:7574436052675239548:0] peer: [5:7574436051546888899:0] socket: 292 qp: -1
2025-11-19T14:00:21.421815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436052675239563:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:21.421826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:21.422111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436052675239552:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.422117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:21.422123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436052675239552:2111] [node 6] ICP22 created new session: [2:7574436052675239565:2111]
2025-11-19T14:00:21.422127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436052675239565:2111] [node 6] ICS09 handshake done sender: [2:7574436052675239554:2112] self: [2:7574436052675239553:0] peer: [6:7574436051701796168:0] socket: 296 qp: -1
2025-11-19T14:00:21.422132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436052675239565:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:21.422136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:21.422567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436052290373250:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34079
2025-11-19T14:00:21.424337Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436052301032909:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.424432Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436052301032907:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.424474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436052301032829:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:21.421417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436051546888900:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19447
2025-11-19T14:00:21.421710Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436051546888893:2115] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:21.422755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436051546888816:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.422769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:21.422774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436051546888816:2049] [node 2] ICP22 created new session: [5:7574436051546888901:2049]
2025-11-19T14:00:21.422779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436051546888901:2049] [node 2] ICS09 handshake done sender: [5:7574436051546888893:2115] self: [5:7574436051546888899:0] peer: [2:7574436052675239548:0] socket: 315 qp: -1
2025-11-19T14:00:21.422784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436051546888901:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:21.422790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:21.424110Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436051546888876:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.424127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436051546888876:2103] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:21.425507Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436051546888896:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.426507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436051546888817:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:21.427833Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436052301032898:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.427845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436052301032898:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.432148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436051546888905:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19447
2025-11-19T14:00:21.432363Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436051546888894:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.432396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436051546888815:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:21.432535Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436052301032898:2110] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:21.432750Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436051546888896:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:21.432896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436052301032892:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.432916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:21.432922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436052301032892:2107] [node 5] ICP22 created new session: [3:7574436052301032913:2107]
2025-11-19T14:00:21.432928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436052301032913:2107] [node 5] ICS09 handshake done sender: [3:7574436052301032898:2110] self: [3:7574436052301032897:0] peer: [5:7574436051546888904:0] socket: 314 qp: -1
2025-11-19T14:00:21.432934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436052301032913:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:21.432944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:21.433062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436051546888817:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.433075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:21.433078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436051546888817:2050] [node 3] ICP22 created new session: [5:7574436051546888907:2050]
2025-11-19T14:00:21.433083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436051546888907:2050] [node 3] ICS09 handshake done sender: [5:7574436051546888896:2118] self: [5:7574436051546888904:0] peer: [3:7574436052301032897:0] socket: 318 qp: -1
2025-11-19T14:00:21.433087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436051546888907:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:21.433100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:21.433046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436051477174284:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23824246ns) {Configure:104(23823052ns) {ProcessConfigured:135(23457129ns) {StartInitialHandshake:157(23456178ns) {DropHandshakes:469(948ns) {DropIncomingHandshake:441(99ns) DropOutgoingHandshake:456(183ns)} GenerateSessionVirtualId:988(311ns) PrepareNewSessionHandshake:476(1264ns) {SwitchToState:178(122ns)}}}}}
2025-11-19T14:00:21.433972Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436052675239541:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.433995Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436052675239541:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:21.434398Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436051477174268:2104] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:21.434657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436051477174266:2103] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.434670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:21.434678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436051477174266:2103] [node 5] ICP22 created new session: [4:7574436051477174288:2103]
2025-11-19T14:00:21.434684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436051477174288:2103] [node 5] ICS09 handshake done sender: [4:7574436051477174268:2104] self: [4:7574436051477174267:0] peer: [5:7574436051546888897:0] socket: 289 qp: -1
2025-11-19T14:00:21.434690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436051477174288:2103] [node 5] ICS10 traffic start
2025-11-19T14:00:21.434697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:21.436034Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436052290373235:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.436065Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436052290373246:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.439871Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436052290373233:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.439893Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436052290373233:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:21.441019Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436051546888895:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.441588Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436051546888876:2103] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:21.441671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436051546888860:2088] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.441690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:21.441698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436051546888860:2088] [node 6] ICP22 created new session: [5:7574436051546888911:2088]
2025-11-19T14:00:21.441705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436051546888911:2088] [node 6] ICS09 handshake done sender: [5:7574436051546888876:2103] self: [5:7574436051546888875:0] peer: [6:7574436051701796172:0] socket: 304 qp: -1
2025-11-19T14:00:21.441712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436051546888911:2088] [node 6] ICS10 traffic start
2025-11-19T14:00:21.441721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.443980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436051701796173:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11306
2025-11-19T14:00:21.444361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436051701796174:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11306
2025-11-19T14:00:21.444397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436051701796175:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11306
2025-11-19T14:00:21.445042Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436052675239557:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.445079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436048380272184:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:21.445136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436051701796143:2073] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63307) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436051701796145:2095] held: no
2025-11-19T14:00:21.445281Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436052290373250:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.445317Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436052290373238:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.445320Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436052290373238:2113] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.446048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436052675239570:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8983
2025-11-19T14:00:21.446880Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436052290373238:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:21.447066Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436052675239557:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:21.447136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436048380272184:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.447155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:21.447162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436048380272184:2048] [node 1] ICP22 created new session: [2:7574436052675239571:2048]
2025-11-19T14:00:21.447168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436052675239571:2048] [node 1] ICS09 handshake done sender: [2:7574436052675239557:2114] self: [2:7574436052675239569:0] peer: [1:7574436052290373237:0] socket: 302 qp: -1
2025-11-19T14:00:21.447176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436052675239571:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:21.447184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:21.443380Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436052301032908:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.449041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436051477174290:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12333
2025-11-19T14:00:21.449092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436051477174291:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12333
2025-11-19T14:00:21.449124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436051477174292:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12333
2025-11-19T14:00:21.449152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436051477174293:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12333
2025-11-19T14:00:21.450103Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436051477174292:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.450132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436051477174210:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:21.450252Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436052675239541:2107] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:21.449781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436051546888913:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19447
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.451281Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436051477174266:2103] [node 5] ICP35 event processing took too much time PendingConnection:278(16526760ns) {HandleHandshakeStatus:329(16525675ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(287ns) SwitchToState:178(73ns) ProcessPendingSessionEvents:495(130160ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 106147ns max# 27280ns}}}
2025-11-19T14:00:21.451544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436052301032917:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55371
2025-11-19T14:00:21.451707Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436052301032907:2115] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:21.451883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436052301032829:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.451899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:21.451906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436052301032829:2049] [node 2] ICP22 created new session: [3:7574436052301032918:2049]
2025-11-19T14:00:21.451914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436052301032918:2049] [node 2] ICS09 handshake done sender: [3:7574436052301032907:2115] self: [3:7574436052301032911:0] peer: [2:7574436052675239540:0] socket: 290 qp: -1
2025-11-19T14:00:21.451921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436052301032918:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:21.451928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:21.453240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436047995405869:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.453264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:21.453271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436047995405869:2048] [node 2] ICP22 created new session: [1:7574436052290373255:2048]
2025-11-19T14:00:21.453277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436052290373255:2048] [node 2] ICS09 handshake done sender: [1:7574436052290373238:2113] self: [1:7574436052290373237:0] peer: [2:7574436052675239569:0] socket: 300 qp: -1
2025-11-19T14:00:21.453283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436052290373255:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:21.453290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:21.453989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436052301032892:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(20997102ns) {HandleHandshakeStatus:329(20996249ns) {DropIncomingHandshake:441(4057ns) DropOutgoingHandshake:456(340ns) SwitchToState:178(68ns) ProcessPendingSessionEvents:495(13910265ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13892048ns max# 13819523ns}}}
2025-11-19T14:00:21.454612Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436051546888892:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:21.455775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436051546888855:2083] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.455792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:21.455798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436051546888855:2083] [node 4] ICP22 created new session: [5:7574436051546888914:2083]
2025-11-19T14:00:21.455804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436051546888914:2083] [node 4] ICS09 handshake done sender: [5:7574436051546888892:2114] self: [5:7574436051546888897:0] peer: [4:7574436051477174267:0] socket: 291 qp: -1
2025-11-19T14:00:21.455809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436051546888914:2083] [node 4] ICS10 traffic start
2025-11-19T14:00:21.455816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:21.457200Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436052675239559:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.457213Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436052675239559:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:21.457705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436048380272185:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.457714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:21.457719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436048380272185:2049] [node 3] ICP22 created new session: [2:7574436052675239575:2049]
2025-11-19T14:00:21.457724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436052675239575:2049] [node 3] ICS09 handshake done sender: [2:7574436052675239541:2107] self: [2:7574436052675239540:0] peer: [3:7574436052301032911:0] socket: 288 qp: -1
2025-11-19T14:00:21.457731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436052675239575:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:21.457737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:21.458073Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436052675239560:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.458501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436051546888916:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19447
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.458669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436048380272185:2049] [node 3] ICP18 handshake (actor [2:7574436052675239560:2116]) is too old
2025-11-19T14:00:21.458698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436052675239574:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8983
2025-11-19T14:00:21.458939Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436052675239559:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:21.459018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436052675239546:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.459030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:21.459036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436052675239546:2108] [node 4] ICP22 created new session: [2:7574436052675239578:2108]
2025-11-19T14:00:21.459041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436052675239578:2108] [node 4] ICS09 handshake done sender: [2:7574436052675239559:2115] self: [2:7574436052675239558:0] peer: [4:7574436051477174295:0] socket: 327 qp: -1
2025-11-19T14:00:21.459045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436052675239578:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:21.459051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:21.460072Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436051477174293:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.460109Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436051477174290:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.460760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574436051477174266:2103] [node 5] ICP18 handshake (actor [4:7574436051477174293:2118]) is too old
No RDMA context found for GID: 2025-11-19T14:00:21.460783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436051477174209:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-19T14:00:21.460809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436051477174296:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12333
2025-11-19T14:00:21.460796Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436052290373233:2110] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:21.460846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436051477174297:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12333
2025-11-19T14:00:21.461049Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436051546888894:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:21.461410Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436051701796145:2095] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:21.461110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436051546888815:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.461124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:21.461129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436051546888815:2048] [node 1] ICP22 created new session: [5:7574436051546888917:2048]
2025-11-19T14:00:21.461134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436051546888917:2048] [node 1] ICS09 handshake done sender: [5:7574436051546888894:2116] self: [5:7574436051546888906:0] peer: [1:7574436052290373231:0] socket: 316 qp: -1
2025-11-19T14:00:21.461140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436051546888917:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:21.461147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.461656Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436051701796144:2094] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:21.462182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436051701796176:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11306
2025-11-19T14:00:21.462780Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436052675239574:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.462885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436052675239546:2108] [node 4] ICP18 handshake (actor [2:7574436052675239574:2118]) is too old
2025-11-19T14:00:21.462900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436052675239580:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8983
2025-11-19T14:00:21.463727Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436051477174291:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.463824Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436051477174296:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.463072Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436051701796175:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.463084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436051701796143:2073] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.463093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:21.463098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436051701796143:2073] [node 5] ICP22 created new session: [6:7574436051701796178:2073]
2025-11-19T14:00:21.463103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436051701796178:2073] [node 5] ICS09 handshake done sender: [6:7574436051701796145:2095] self: [6:7574436051701796172:0] peer: [5:7574436051546888875:0] socket: 307 qp: -1
2025-11-19T14:00:21.463108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436051701796178:2073] [node 5] ICS10 traffic start
2025-11-19T14:00:21.463124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:21.464786Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436052290373240:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.464804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436052290373240:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:21.465290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436052290373227:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.465303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:21.465310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436052290373227:2107] [node 5] ICP22 created new session: [1:7574436052290373259:2107]
2025-11-19T14:00:21.465314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436052290373259:2107] [node 5] ICS09 handshake done sender: [1:7574436052290373233:2110] self: [1:7574436052290373231:0] peer: [5:7574436051546888906:0] socket: 299 qp: -1
2025-11-19T14:00:21.465319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436052290373259:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:21.465326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.466725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436052301032921:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55371
2025-11-19T14:00:21.466847Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436052290373240:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:21.466483Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436051701796176:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.466496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436051701796092:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.466506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:21.466511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436051701796092:2049] [node 2] ICP22 created new session: [6:7574436051701796180:2049]
2025-11-19T14:00:21.466517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436051701796180:2049] [node 2] ICS09 handshake done sender: [6:7574436051701796144:2094] self: [6:7574436051701796168:0] peer: [2:7574436052675239553:0] socket: 306 qp: -1
2025-11-19T14:00:21.466522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436051701796180:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:21.466527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:21.466813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436051701796093:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:21.466932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436051701796091:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:21.467158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436051477174210:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5923) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436051477174292:2117] held: no
2025-11-19T14:00:21.467308Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436051477174292:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:21.467824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436051477174211:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:21.468210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436051477174210:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.468226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:21.468232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436051477174210:2049] [node 2] ICP22 created new session: [4:7574436051477174301:2049]
2025-11-19T14:00:21.468237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436051477174301:2049] [node 2] ICS09 handshake done sender: [4:7574436051477174292:2117] self: [4:7574436051477174295:0] peer: [2:7574436052675239558:0] socket: 338 qp: -1
2025-11-19T14:00:21.468243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436051477174301:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:21.468250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.468985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436051701796184:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11306
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.470922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436051477174304:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12333
2025-11-19T14:00:21.471124Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436051477174290:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:21.471931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436052301032902:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.471943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436052301032902:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:21.471212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436051477174209:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.471227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:21.471231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436051477174209:2048] [node 1] ICP22 created new session: [4:7574436051477174305:2048]
2025-11-19T14:00:21.471237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436051477174305:2048] [node 1] ICS09 handshake done sender: [4:7574436051477174290:2115] self: [4:7574436051477174298:0] peer: [1:7574436052290373239:0] socket: 320 qp: -1
2025-11-19T14:00:21.471243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436051477174305:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:21.471248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:21.472298Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436051701796184:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.472413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436051701796110:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.472436Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436052301032904:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.472447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436052301032904:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:21.473138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436052301032922:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55371
2025-11-19T14:00:21.473580Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436052301032904:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:21.473638Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436051477174286:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.473650Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436051477174286:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.474186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436051477174307:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12333
2025-11-19T14:00:21.474256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436051701796187:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11306
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.474355Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436052301032902:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:21.474358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436051701796188:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11306
2025-11-19T14:00:21.474432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436052301032891:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.474443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:21.474450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436052301032891:2106] [node 6] ICP22 created new session: [3:7574436052301032925:2106]
2025-11-19T14:00:21.474455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436052301032925:2106] [node 6] ICS09 handshake done sender: [3:7574436052301032902:2113] self: [3:7574436052301032896:0] peer: [6:7574436051701796182:0] socket: 303 qp: -1
2025-11-19T14:00:21.474460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436052301032925:2106] [node 6] ICS10 traffic start
2025-11-19T14:00:21.474467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:21.474532Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436052290373244:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.474544Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436052290373244:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:21.474855Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436051477174286:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:21.474885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436052301032899:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.475192Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436051477174296:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:21.474899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:21.474903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436052301032899:2108] [node 4] ICP22 created new session: [3:7574436052301032927:2108]
2025-11-19T14:00:21.474908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436052301032927:2108] [node 4] ICS09 handshake done sender: [3:7574436052301032904:2114] self: [3:7574436052301032903:0] peer: [4:7574436051477174300:0] socket: 313 qp: -1
2025-11-19T14:00:21.474912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436052301032927:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:21.474918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:21.475303Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436052301032921:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.475330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436052301032828:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:21.475498Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436052290373244:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:21.479866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436051477174284:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.479896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:21.479905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436051477174284:2108] [node 6] ICP22 created new session: [4:7574436051477174309:2108]
2025-11-19T14:00:21.479912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436051477174309:2108] [node 6] ICS09 handshake done sender: [4:7574436051477174286:2114] self: [4:7574436051477174285:0] peer: [6:7574436051701796186:0] socket: 340 qp: -1
2025-11-19T14:00:21.479919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436051477174309:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:21.479928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:21.479904Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436052290373247:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:21.479918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436052290373247:2118] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:21.480151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436052290373234:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.480164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:21.480168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436052290373234:2109] [node 4] ICP22 created new session: [1:7574436052290373264:2109]
2025-11-19T14:00:21.480173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436052290373264:2109] [node 4] ICS09 handshake done sender: [1:7574436052290373240:2114] self: [1:7574436052290373239:0] peer: [4:7574436051477174298:0] socket: 301 qp: -1
2025-11-19T14:00:21.480178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436052290373264:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:21.480195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:21.484266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436052290373228:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.484290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:21.484299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436052290373228:2108] [node 6] ICP22 created new session: [1:7574436052290373266:2108]
2025-11-19T14:00:21.484306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436052290373266:2108] [node 6] ICS09 handshake done sender: [1:7574436052290373244:2115] self: [1:7574436052290373232:0] peer: [6:7574436051701796183:0] socket: 344 qp: -1
2025-11-19T14:00:21.484312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436052290373266:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:21.484320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:21.485440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436052675239552:2111] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:21.485036Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436051477174211:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19416) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436051477174296:2119] held: no
2025-11-19T14:00:21.485122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436051477174211:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.485131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:21.485138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436051477174211:2050] [node 3] ICP22 created new session: [4:7574436051477174311:2050]
2025-11-19T14:00:21.485143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436051477174311:2050] [node 3] ICS09 handshake done sender: [4:7574436051477174296:2119] self: [4:7574436051477174300:0] peer: [3:7574436052301032903:0] socket: 319 qp: -1
2025-11-19T14:00:21.485148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436051477174311:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:21.485154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:21.486000Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436051701796175:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:21.486894Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436051701796184:2121] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:21.486970Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436051701796110:2065] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11568) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436051701796184:2121] held: no
2025-11-19T14:00:21.487061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436051701796093:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.487070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:21.487075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436051701796093:2050] [node 3] ICP22 created new session: [6:7574436051701796190:2050]
2025-11-19T14:00:21.487083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436051701796190:2050] [node 3] ICS09 handshake done sender: [6:7574436051701796175:2119] self: [6:7574436051701796182:0] peer: [3:7574436052301032896:0] socket: 322 qp: -1
2025-11-19T14:00:21.487088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436051701796190:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:21.487094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:21.487551Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436052301032922:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.487957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436052301032930:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55371
2025-11-19T14:00:21.487984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574436052301032891:2106] [node 6] ICP18 handshake (actor [3:7574436052301032922:2120]) is too old
2025-11-19T14:00:21.488812Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436052301032921:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:21.488885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436052301032828:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.488898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:21.488903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436052301032828:2048] [node 1] ICP22 created new session: [3:7574436052301032931:2048]
2025-11-19T14:00:21.488907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436052301032931:2048] [node 1] ICS09 handshake done sender: [3:7574436052301032921:2119] self: [3:7574436052301032929:0] peer: [1:7574436052290373230:0] socket: 329 qp: -1
2025-11-19T14:00:21.488912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436052301032931:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:21.488918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:21.488192Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436052290373247:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:21.488356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436047995405870:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:21.488371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:21.488376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436047995405870:2049] [node 3] ICP22 created new session: [1:7574436052290373269:2049]
2025-11-19T14:00:21.488381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436052290373269:2049] [node 3] ICS09 handshake done sender: [1:7574436052290373247:2118] self: [1:7574436052290373230:0] peer: [3:7574436052301032929:0] socket: 311 qp: -1
2025-11-19T14:00:21.488386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436052290373269:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:21.488393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:21.493429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436051701796189:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11306
2025-11-19T14:00:21.496232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436051701796110:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.496255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:21.496264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436051701796110:2065] [node 4] ICP22 created new session: [6:7574436051701796192:2065]
2025-11-19T14:00:21.496272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436051701796192:2065] [node 4] ICS09 handshake done sender: [6:7574436051701796184:2121] self: [6:7574436051701796186:0] peer: [4:7574436051477174285:0] socket: 341 qp: -1
2025-11-19T14:00:21.496282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436051701796192:2065] [node 4] ICS10 traffic start
2025-11-19T14:00:21.496290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:21.497390Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436051701796091:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8069) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436051701796176:2120] held: no
2025-11-19T14:00:21.497680Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436051701796176:2120] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:21.498043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436051701796091:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:21.498058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:21.498064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436051701796091:2048] [node 1] ICP22 created new session: [6:7574436051701796194:2048]
2025-11-19T14:00:21.498070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436051701796194:2048] [node 1] ICS09 handshake done sender: [6:7574436051701796176:2120] self: [6:7574436051701796183:0] peer: [1:7574436052290373232:0] socket: 328 qp: -1
2025-11-19T14:00:21.498077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436051701796194:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:21.498083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:21.505767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:21.505865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436052675239571:2048] [node 1] ICS01 socket: 302 reason# EndOfStream
2025-11-19T14:00:21.505875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436048380272184:2048] [node 1] ICP30 unregister session Session# [2:7574436052675239571:2048] VirtualId# [2:7574436052675239569:0]
2025-11-19T14:00:21.505883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436052675239571:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:21.505918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:21.505941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:21.506430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:21.506501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436052301032931:2048] [node 1] ICS01 socket: 329 reason# EndOfStream
2025-11-19T14:00:21.506506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436052301032828:2048] [node 1] ICP30 unregister session Session# [3:7574436052301032931:2048] VirtualId# [3:7574436052301032929:0]
2025-11-19T14:00:21.506511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436052301032931:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:21.506541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:21.506560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:21.517574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:21.517680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436051546888917:2048] [node 1] ICS01 socket: 316 reason# EndOfStream
2025-11-19T14:00:21.517691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436051546888815:2048] [node 1] ICP30 unregister session Session# [5:7574436051546888917:2048] VirtualId# [5:7574436051546888906:0]
2025-11-19T14:00:21.517699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436051546888917:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:21.517744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:21.517769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:21.517805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:21.517862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436051701796194:2048] [node 1] ICS01 socket: 328 reason# EndOfStream
2025-11-19T14:00:21.517866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436051701796091:2048] [node 1] ICP30 unregister session Session# [6:7574436051701796194:2048] VirtualId# [6:7574436051701796183:0]
2025-11-19T14:00:21.517870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436051701796194:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:21.517897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:21.517922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:21.517950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:21.518008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436051477174305:2048] [node 1] ICS01 socket: 320 reason# EndOfStream
2025-11-19T14:00:21.518013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436051477174209:2048] [node 1] ICP30 unregister session Session# [4:7574436051477174305:2048] VirtualId# [4:7574436051477174298:0]
2025-11-19T14:00:21.518017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436051477174305:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:21.518040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:21.518059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:21.531919Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:21.532040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436052301032918:2049] [node 2] ICS01 socket: 290 reason# EndOfStream
2025-11-19T14:00:21.532051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436052301032829:2049] [node 2] ICP30 unregister session Session# [3:7574436052301032918:2049] VirtualId# [3:7574436052301032911:0]
2025-11-19T14:00:21.532061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436052301032918:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:21.532113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:21.532137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:21.553396Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:21.553511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436051477174301:2049] [node 2] ICS01 socket: 338 reason# EndOfStream
2025-11-19T14:00:21.553522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436051477174210:2049] [node 2] ICP30 unregister session Session# [4:7574436051477174301:2049] VirtualId# [4:7574436051477174295:0]
2025-11-19T14:00:21.553532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436051477174301:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:21.553580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:21.553605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:21.553642Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:21.553700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436051546888901:2049] [node 2] ICS01 socket: 315 reason# EndOfStream
2025-11-19T14:00:21.553705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436051546888816:2049] [node 2] ICP30 unregister session Session# [5:7574436051546888901:2049] VirtualId# [5:7574436051546888899:0]
2025-11-19T14:00:21.553710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436051546888901:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:21.553735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:21.553753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:21.553782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:21.553840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436051701796180:2049] [node 2] ICS01 socket: 306 reason# EndOfStream
2025-11-19T14:00:21.553846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436051701796092:2049] [node 2] ICP30 unregister session Session# [6:7574436051701796180:2049] VirtualId# [6:7574436051701796168:0]
2025-11-19T14:00:21.553850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436051701796180:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:21.553872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:21.553889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:21.573042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:21.573102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436051546888907:2050] [node 3] ICS01 socket: 318 reason# EndOfStream
2025-11-19T14:00:21.573107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436051546888817:2050] [node 3] ICP30 unregister session Session# [5:7574436051546888907:2050] VirtualId# [5:7574436051546888904:0]
2025-11-19T14:00:21.573111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436051546888907:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:21.573136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:21.573154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:21.572787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:21.572899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436051701796190:2050] [node 3] ICS01 socket: 322 reason# EndOfStream
2025-11-19T14:00:21.572910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436051701796093:2050] [node 3] ICP30 unregister session Session# [6:7574436051701796190:2050] VirtualId# [6:7574436051701796182:0]
2025-11-19T14:00:21.572919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436051701796190:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:21.572974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:21.573004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:21.614503Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:21.614611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436051701796192:2065] [node 4] ICS01 socket: 341 reason# EndOfStream
2025-11-19T14:00:21.614622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436051701796110:2065] [node 4] ICP30 unregister session Session# [6:7574436051701796192:2065] VirtualId# [6:7574436051701796186:0]
2025-11-19T14:00:21.614630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436051701796192:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:21.614682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:21.614706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:21.724027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436050954485904:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:21.724046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436050954485905:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:21.752045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436051656852873:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:21.752063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436051656852874:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:21.788010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436050744792297:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:21.788023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436050744792298:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:21.863855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436054720678511:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:21.863872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436054720678512:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:21.863876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436054720678513:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:21.913915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436051045122518:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:21.913933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436051045122519:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:21.913939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436051045122520:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:21.971861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436051428471891:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:21.971877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436051428471892:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:21.971881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436051428471893:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:22.032402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436055723439222:2081] [node 5] ICP01 ready to work
2025-11-19T14:00:22.032270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436055340089861:2093] [node 4] ICP01 ready to work
2025-11-19T14:00:22.033077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436055723439225:2077] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.032452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436059015645856:2089] [node 6] ICP01 ready to work
2025-11-19T14:00:22.032570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436059015645857:2093] [node 5] ICP01 ready to work
2025-11-19T14:00:22.033618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059015645864:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56116
2025-11-19T14:00:22.036287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436055249453266:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:22.040684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436055249453271:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.045101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436055249453265:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:22.046490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436055249453276:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38976
2025-11-19T14:00:22.046505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436055249453277:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38976
2025-11-19T14:00:22.048543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436055340089882:2104] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.049756Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436055723439222:2081] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17301015ns) {Configure:104(17300564ns) {ProcessConfigured:135(10635535ns) {StartInitialHandshake:157(10635142ns) {DropHandshakes:469(635ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(238ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:476(652ns) {SwitchToState:178(341ns)}}}}}
2025-11-19T14:00:22.050022Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436051428471892:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17938602ns) {Configure:104(17937939ns) {ProcessConfigured:135(10830414ns) {StartInitialHandshake:157(10829646ns) {DropHandshakes:469(485ns) {DropIncomingHandshake:441(16ns) DropOutgoingHandshake:456(18ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:476(819ns) {SwitchToState:178(157ns)}}}}}
2025-11-19T14:00:22.046684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436055951820232:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:22.049492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436055951820239:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34105
2025-11-19T14:00:22.049673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436055249453281:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38976
2025-11-19T14:00:22.049538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436055951820240:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34105
2025-11-19T14:00:22.051195Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436055951820239:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.051922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436051656852873:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:22.053455Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436059015645864:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.057015Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436055249453276:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.057071Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436055249453261:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.057080Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436055249453261:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:22.057121Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436055249453277:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.056483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436055039759662:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56926
2025-11-19T14:00:22.059686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436055039759665:2112] [node 4] ICP01 ready to work
2025-11-19T14:00:22.064363Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436054720678512:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(19687447ns) {Configure:104(19686644ns) {ProcessConfigured:135(19471484ns) {StartInitialHandshake:157(19470467ns) {DropHandshakes:469(180ns) {DropIncomingHandshake:441(29ns) DropOutgoingHandshake:456(29ns)} GenerateSessionVirtualId:988(149ns) PrepareNewSessionHandshake:476(1278ns) {SwitchToState:178(581ns)}}}}}
2025-11-19T14:00:22.064431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436055951820243:2113] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.065300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059015645887:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56116
2025-11-19T14:00:22.068058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436055039759657:2105] [node 5] ICP01 ready to work
2025-11-19T14:00:22.068398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436055951820231:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.076454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436055723439263:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45579
2025-11-19T14:00:22.076522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436055723439264:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45579
2025-11-19T14:00:22.076557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436055723439265:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45579
2025-11-19T14:00:22.076649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436051428471893:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(44542411ns) {Configure:104(44540491ns) {ProcessConfigured:135(37479011ns) {StartInitialHandshake:157(37477826ns) {DropHandshakes:469(529ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(22ns)} GenerateSessionVirtualId:988(154ns) PrepareNewSessionHandshake:476(837ns) {SwitchToState:178(363ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.077001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436055249453284:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38976
2025-11-19T14:00:22.077023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436055249453285:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38976
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.077243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436055951820249:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34105
2025-11-19T14:00:22.077434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436055340089891:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10574
2025-11-19T14:00:22.077492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436055340089892:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10574
2025-11-19T14:00:22.077503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436055340089893:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10574
2025-11-19T14:00:22.079217Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436055249453261:2107] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:22.078616Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436055340089882:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(29994580ns) {Configure:104(29993602ns) {ProcessConfigured:135(29791468ns) {StartInitialHandshake:157(29790878ns) {DropHandshakes:469(318ns) {DropIncomingHandshake:441(100ns) DropOutgoingHandshake:456(32ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(1209ns) {SwitchToState:178(238ns)}}}}}
2025-11-19T14:00:22.079484Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436055340089891:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.079528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436055340089861:2093] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:22.079704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436050954485904:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.079727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:22.079734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436050954485904:2048] [node 2] ICP22 created new session: [1:7574436055249453287:2048]
2025-11-19T14:00:22.079740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436055249453287:2048] [node 2] ICS09 handshake done sender: [1:7574436055249453261:2107] self: [1:7574436055249453260:0] peer: [2:7574436055951820241:0] socket: 294 qp: -1
2025-11-19T14:00:22.079766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436055249453287:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:22.079791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:22.080837Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436055723439264:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.080874Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436055723439263:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.080903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436051428471891:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:22.083776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436055039759660:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:22.091557Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436055039759657:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23353435ns) {Configure:104(23352827ns) {ProcessConfigured:135(23028910ns) {StartInitialHandshake:157(23028464ns) {DropHandshakes:469(1106ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(235ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(752ns) {SwitchToState:178(217ns)}}}}}
2025-11-19T14:00:22.091613Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436055039759662:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.091250Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436055723439225:2077] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(58094077ns) {Configure:104(58093137ns) {ProcessConfigured:135(51756387ns) {StartInitialHandshake:157(51754807ns) {DropHandshakes:469(242ns) {DropIncomingHandshake:441(17ns) DropOutgoingHandshake:456(27ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:476(1577ns) {SwitchToState:178(785ns)}}}}}
2025-11-19T14:00:22.091368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436055723439225:2077] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:22.091805Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436055340089892:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.091865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436051045122518:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.092196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436055340089897:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10574
2025-11-19T14:00:22.092053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436055951820231:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23527749ns) {Configure:104(23526197ns) {ProcessConfigured:135(23258523ns) {StartInitialHandshake:157(23257927ns) {DropHandshakes:469(367ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(270ns) PrepareNewSessionHandshake:476(1358ns) {SwitchToState:178(763ns)}}}}}
2025-11-19T14:00:22.092798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436051656852874:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21871856ns) {Configure:104(21869319ns) {ProcessConfigured:135(21564120ns) {StartInitialHandshake:157(21563704ns) {DropHandshakes:469(890ns) {DropIncomingHandshake:441(196ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:476(722ns) {SwitchToState:178(53ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.093374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436055723439270:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45579
2025-11-19T14:00:22.093944Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436055723439263:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:22.094041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436055723439225:2077] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.094064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:22.094071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436055723439225:2077] [node 4] ICP22 created new session: [6:7574436055723439271:2077]
2025-11-19T14:00:22.094087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436055723439271:2077] [node 4] ICS09 handshake done sender: [6:7574436055723439263:2113] self: [6:7574436055723439269:0] peer: [4:7574436059015645858:0] socket: 313 qp: -1
2025-11-19T14:00:22.094097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436055723439271:2077] [node 4] ICS10 traffic start
2025-11-19T14:00:22.094119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:22.092161Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436059015645859:2096] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.092176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436059015645859:2096] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:22.093555Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436059015645859:2096] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:22.094609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436059015645856:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.094617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:22.094623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436059015645856:2089] [node 6] ICP22 created new session: [4:7574436059015645889:2089]
2025-11-19T14:00:22.094628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436059015645889:2089] [node 6] ICS09 handshake done sender: [4:7574436059015645859:2096] self: [4:7574436059015645858:0] peer: [6:7574436055723439269:0] socket: 298 qp: -1
2025-11-19T14:00:22.094632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436059015645889:2089] [node 6] ICS10 traffic start
2025-11-19T14:00:22.094648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:22.103991Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436059015645863:2098] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.104040Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436059015645863:2098] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:22.104488Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436050954485904:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(24656914ns) {HandleHandshakeStatus:329(24655392ns) {DropIncomingHandshake:441(39ns) DropOutgoingHandshake:456(382ns) SwitchToState:178(48ns) ProcessPendingSessionEvents:495(24502632ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24479119ns max# 24379163ns}}}
2025-11-19T14:00:22.104872Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436055249453273:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.104884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436055249453273:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:22.104957Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436055249453270:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.104964Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436055249453270:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.108544Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436055039759665:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(48775134ns) {Configure:104(48773921ns) {ProcessConfigured:135(48445921ns) {StartInitialHandshake:157(48444815ns) {DropHandshakes:469(845ns) {DropIncomingHandshake:441(59ns) DropOutgoingHandshake:456(147ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:476(1941ns) {SwitchToState:178(1174ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.108833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436055951820252:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34105
2025-11-19T14:00:22.109024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059015645892:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56116
2025-11-19T14:00:22.109676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436055039759660:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25707263ns) {Configure:104(25706205ns) {ProcessConfigured:135(25359887ns) {StartInitialHandshake:157(25358954ns) {DropHandshakes:469(811ns) {DropIncomingHandshake:441(93ns) DropOutgoingHandshake:456(214ns)} GenerateSessionVirtualId:988(359ns) PrepareNewSessionHandshake:476(759ns) {SwitchToState:178(308ns)}}}}}
2025-11-19T14:00:22.109986Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436055951820239:2111] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:22.110148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436051656852873:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.110172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:22.110181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436051656852873:2048] [node 1] ICP22 created new session: [2:7574436055951820253:2048]
2025-11-19T14:00:22.110187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436055951820253:2048] [node 1] ICS09 handshake done sender: [2:7574436055951820239:2111] self: [2:7574436055951820241:0] peer: [1:7574436055249453260:0] socket: 301 qp: -1
2025-11-19T14:00:22.110195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436055951820253:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:22.110217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.111222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436055039759674:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56926
2025-11-19T14:00:22.116910Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436059015645892:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.116981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436054720678511:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:22.122379Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436059015645856:2089] [node 6] ICP35 event processing took too much time PendingConnection:278(27690003ns) {HandleHandshakeStatus:329(27689070ns) {DropIncomingHandshake:441(24ns) DropOutgoingHandshake:456(159ns) SwitchToState:178(69ns) ProcessPendingSessionEvents:495(17800633ns) {unordered ForwardSessionEventToSession:612 num# 60 sum# 17782936ns max# 17726856ns}}}
2025-11-19T14:00:22.128931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436055249453279:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.128959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436055249453279:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:22.130781Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436059015645863:2098] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:22.132287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436059015645857:2093] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.132320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:22.132340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436059015645857:2093] [node 5] ICP22 created new session: [4:7574436059015645895:2093]
2025-11-19T14:00:22.132347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436059015645895:2093] [node 5] ICS09 handshake done sender: [4:7574436059015645863:2098] self: [4:7574436059015645862:0] peer: [5:7574436055340089895:0] socket: 300 qp: -1
2025-11-19T14:00:22.132356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436059015645895:2093] [node 5] ICS10 traffic start
2025-11-19T14:00:22.132368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.132836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436055340089899:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10574
2025-11-19T14:00:22.133071Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436055340089891:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:22.133145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436055340089861:2093] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.132837Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436059015645887:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.132992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436054720678512:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:22.132120Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436055249453279:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:22.133717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436055249453265:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.133731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:22.133737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436055249453265:2108] [node 4] ICP22 created new session: [1:7574436055249453291:2108]
2025-11-19T14:00:22.133742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436055249453291:2108] [node 4] ICS09 handshake done sender: [1:7574436055249453279:2115] self: [1:7574436055249453278:0] peer: [4:7574436059015645893:0] socket: 330 qp: -1
2025-11-19T14:00:22.133748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436055249453291:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:22.133756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:22.133157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:22.133170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436055340089861:2093] [node 4] ICP22 created new session: [5:7574436055340089900:2093]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.141225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436055039759674:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.141411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436055039759676:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56926
2025-11-19T14:00:22.133176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436055340089900:2093] [node 4] ICS09 handshake done sender: [5:7574436055340089891:2114] self: [5:7574436055340089895:0] peer: [4:7574436059015645862:0] socket: 302 qp: -1
2025-11-19T14:00:22.133184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436055340089900:2093] [node 4] ICS10 traffic start
2025-11-19T14:00:22.133191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:22.133734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436055340089898:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10574
2025-11-19T14:00:22.141429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436055340089903:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10574
2025-11-19T14:00:22.142830Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436054720678511:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16536) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436059015645892:2117] held: no
2025-11-19T14:00:22.142353Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436055249453273:2112] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:22.145187Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436055039759676:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.145232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436050744792297:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.146583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436055039759679:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56926
2025-11-19T14:00:22.146935Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436055039759676:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:22.144224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436055249453271:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.144256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:22.144264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436055249453271:2110] [node 5] ICP22 created new session: [1:7574436055249453296:2110]
2025-11-19T14:00:22.144272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436055249453296:2110] [node 5] ICS09 handshake done sender: [1:7574436055249453273:2112] self: [1:7574436055249453272:0] peer: [5:7574436055340089896:0] socket: 306 qp: -1
2025-11-19T14:00:22.144280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436055249453296:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:22.144291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:22.144492Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436055249453270:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:22.145487Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436055249453280:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.145491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436055249453280:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:22.145922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436055249453266:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.145933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:22.145939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436055249453266:2109] [node 6] ICP22 created new session: [1:7574436055249453298:2109]
2025-11-19T14:00:22.145944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436055249453298:2109] [node 6] ICS09 handshake done sender: [1:7574436055249453270:2111] self: [1:7574436055249453269:0] peer: [6:7574436055723439267:0] socket: 304 qp: -1
2025-11-19T14:00:22.145952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436055249453298:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:22.145961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:22.146730Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436055249453280:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:22.146969Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436055249453285:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.147071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436050744792297:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.147090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:22.147096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436050744792297:2048] [node 1] ICP22 created new session: [3:7574436055039759680:2048]
2025-11-19T14:00:22.147102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436055039759680:2048] [node 1] ICS09 handshake done sender: [3:7574436055039759676:2118] self: [3:7574436055039759678:0] peer: [1:7574436055249453267:0] socket: 348 qp: -1
2025-11-19T14:00:22.147108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436055039759680:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:22.147125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:22.148390Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436055340089892:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:22.148483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436051045122518:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.148499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:22.148507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436051045122518:2048] [node 1] ICP22 created new session: [5:7574436055340089904:2048]
2025-11-19T14:00:22.148515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436055340089904:2048] [node 1] ICS09 handshake done sender: [5:7574436055340089892:2115] self: [5:7574436055340089896:0] peer: [1:7574436055249453272:0] socket: 326 qp: -1
2025-11-19T14:00:22.148522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436055340089904:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:22.148542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.152234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436055723439274:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45579
2025-11-19T14:00:22.153587Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436055723439265:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.153622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436051428471892:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:22.153772Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436055951820249:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.153802Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436055951820236:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.153806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436055951820236:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.154449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436055723439276:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45579
2025-11-19T14:00:22.154677Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436055723439264:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:22.154787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436051428471891:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.154809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:22.154815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436051428471891:2048] [node 1] ICP22 created new session: [6:7574436055723439277:2048]
2025-11-19T14:00:22.154821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436055723439277:2048] [node 1] ICS09 handshake done sender: [6:7574436055723439264:2114] self: [6:7574436055723439267:0] peer: [1:7574436055249453269:0] socket: 323 qp: -1
2025-11-19T14:00:22.154830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436055723439277:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:22.154841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:22.156166Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436055951820236:2110] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:22.156360Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436055723439265:2115] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:22.156780Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436055951820245:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.156791Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436055951820245:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:22.156595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436051428471892:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.156614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:22.156621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436051428471892:2049] [node 2] ICP22 created new session: [6:7574436055723439279:2049]
2025-11-19T14:00:22.156626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436055723439279:2049] [node 2] ICS09 handshake done sender: [6:7574436055723439265:2115] self: [6:7574436055723439275:0] peer: [2:7574436055951820235:0] socket: 324 qp: -1
2025-11-19T14:00:22.156632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436055723439279:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:22.156648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:22.156669Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436055249453284:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.156817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436055249453265:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(22991364ns) {HandleHandshakeStatus:329(22990633ns) {DropIncomingHandshake:441(3644ns) DropOutgoingHandshake:456(117ns) SwitchToState:178(46ns) ProcessPendingSessionEvents:495(22918393ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22879345ns max# 22799753ns}}}
2025-11-19T14:00:22.156912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436055249453266:2109] [node 6] ICP18 handshake (actor [1:7574436055249453284:2118]) is too old
2025-11-19T14:00:22.159798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436055039759675:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56926
2025-11-19T14:00:22.160084Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436055039759675:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.160105Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574436051045122518:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:22.160558Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436055951820240:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.160573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436055951820232:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.160588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:22.160595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436055951820232:2108] [node 6] ICP22 created new session: [2:7574436055951820260:2108]
2025-11-19T14:00:22.160600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436055951820260:2108] [node 6] ICS09 handshake done sender: [2:7574436055951820236:2110] self: [2:7574436055951820235:0] peer: [6:7574436055723439275:0] socket: 309 qp: -1
2025-11-19T14:00:22.160604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436055951820260:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:22.160611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:22.161535Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436055340089897:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.161571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436051045122520:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:22.161695Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436055039759669:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.161703Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436055039759669:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:22.162448Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436055039759669:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:22.162528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436055039759657:2105] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.162545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:22.162550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436055039759657:2105] [node 5] ICP22 created new session: [3:7574436055039759685:2105]
2025-11-19T14:00:22.162555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436055039759685:2105] [node 5] ICS09 handshake done sender: [3:7574436055039759669:2113] self: [3:7574436055039759667:0] peer: [5:7574436055340089906:0] socket: 332 qp: -1
2025-11-19T14:00:22.162560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436055039759685:2105] [node 5] ICS10 traffic start
2025-11-19T14:00:22.162566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.162872Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436055340089893:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.162919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436055340089907:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10574
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.163149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436055723439281:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45579
2025-11-19T14:00:22.164518Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436055340089897:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:22.164648Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436055723439281:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.164672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436051428471893:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:22.165175Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436055039759673:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.165183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436055039759673:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:22.164778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436051045122520:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.164795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:22.164802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436051045122520:2050] [node 3] ICP22 created new session: [5:7574436055340089909:2050]
2025-11-19T14:00:22.164807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436055340089909:2050] [node 3] ICS09 handshake done sender: [5:7574436055340089897:2118] self: [5:7574436055340089906:0] peer: [3:7574436055039759667:0] socket: 333 qp: -1
2025-11-19T14:00:22.164813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436055340089909:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:22.164820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.166018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436055723439283:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45579
2025-11-19T14:00:22.166103Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436055039759673:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:22.166241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436055039759660:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.166257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:22.166262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436055039759660:2108] [node 6] ICP22 created new session: [3:7574436055039759689:2108]
2025-11-19T14:00:22.166277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436055039759689:2108] [node 6] ICS09 handshake done sender: [3:7574436055039759673:2115] self: [3:7574436055039759668:0] peer: [6:7574436055723439282:0] socket: 354 qp: -1
2025-11-19T14:00:22.166282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436055039759689:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:22.166287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.168549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436050954485905:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.168589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:22.168597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436050954485905:2049] [node 3] ICP22 created new session: [1:7574436055249453302:2049]
2025-11-19T14:00:22.168603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436055249453302:2049] [node 3] ICS09 handshake done sender: [1:7574436055249453280:2116] self: [1:7574436055249453267:0] peer: [3:7574436055039759678:0] socket: 307 qp: -1
2025-11-19T14:00:22.168609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436055249453302:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:22.168623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.172786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059015645900:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56116
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.173056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059015645899:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56116
2025-11-19T14:00:22.173174Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436059015645900:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.173216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436054720678513:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:22.173334Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436055039759672:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.173346Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436055039759672:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.175254Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436059015645892:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:22.175426Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436055039759672:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:22.175088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436055951820259:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34105
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.175774Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436055723439281:2121] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:22.176001Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436055340089898:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.176034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436051045122519:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:22.175697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436054720678511:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.175717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:22.175730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436054720678511:2048] [node 1] ICP22 created new session: [4:7574436059015645904:2048]
2025-11-19T14:00:22.175738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436059015645904:2048] [node 1] ICS09 handshake done sender: [4:7574436059015645892:2117] self: [4:7574436059015645893:0] peer: [1:7574436055249453278:0] socket: 331 qp: -1
2025-11-19T14:00:22.175763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436059015645904:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:22.175773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:22.176522Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436055951820245:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.176590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436055951820258:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34105
2025-11-19T14:00:22.176651Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436055951820259:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.177036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574436051428471892:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(20354020ns) {HandleHandshakeStatus:329(20353422ns) {DropIncomingHandshake:441(285ns) DropOutgoingHandshake:456(3590ns) SwitchToState:178(94ns) ProcessPendingSessionEvents:495(5151318ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5131195ns max# 5051254ns}}}
2025-11-19T14:00:22.177052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436051428471893:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.177065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:22.177069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436051428471893:2050] [node 3] ICP22 created new session: [6:7574436055723439284:2050]
2025-11-19T14:00:22.177073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436055723439284:2050] [node 3] ICS09 handshake done sender: [6:7574436055723439281:2121] self: [6:7574436055723439282:0] peer: [3:7574436055039759668:0] socket: 355 qp: -1
2025-11-19T14:00:22.177078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436055723439284:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:22.177085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:22.178226Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436055951820250:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.178236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436055951820250:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:22.178325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436055951820258:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.178335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436055951820243:2113] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.178344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:22.178348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436055951820243:2113] [node 4] ICP22 created new session: [2:7574436055951820265:2113]
2025-11-19T14:00:22.178352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436055951820265:2113] [node 4] ICS09 handshake done sender: [2:7574436055951820245:2114] self: [2:7574436055951820244:0] peer: [4:7574436059015645898:0] socket: 321 qp: -1
2025-11-19T14:00:22.178357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436055951820265:2113] [node 4] ICS10 traffic start
2025-11-19T14:00:22.178363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:22.177891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059015645902:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56116
2025-11-19T14:00:22.178134Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436059015645900:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:22.178150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059015645903:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56116
2025-11-19T14:00:22.179778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436055039759665:2112] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.179807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:22.179815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436055039759665:2112] [node 4] ICP22 created new session: [3:7574436055039759693:2112]
2025-11-19T14:00:22.179823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436055039759693:2112] [node 4] ICS09 handshake done sender: [3:7574436055039759672:2114] self: [3:7574436055039759666:0] peer: [4:7574436059015645901:0] socket: 350 qp: -1
2025-11-19T14:00:22.179829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436055039759693:2112] [node 4] ICS10 traffic start
2025-11-19T14:00:22.179836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:22.179856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436055039759692:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56926
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.185224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436055723439286:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45579
2025-11-19T14:00:22.185821Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436055723439286:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.185860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436055723439222:2081] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:22.185984Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436055340089894:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.185995Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436055340089894:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.187016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436055723439288:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45579
2025-11-19T14:00:22.187187Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436055340089894:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:22.187348Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436055723439286:2123] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:22.187489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436055340089882:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.187504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:22.187509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436055340089882:2104] [node 6] ICP22 created new session: [5:7574436055340089914:2104]
2025-11-19T14:00:22.187515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436055340089914:2104] [node 6] ICS09 handshake done sender: [5:7574436055340089894:2117] self: [5:7574436055340089883:0] peer: [6:7574436055723439287:0] socket: 296 qp: -1
2025-11-19T14:00:22.187519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436055340089914:2104] [node 6] ICS10 traffic start
2025-11-19T14:00:22.187527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:22.187619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436055723439222:2081] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.187629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:22.187635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436055723439222:2081] [node 5] ICP22 created new session: [6:7574436055723439289:2081]
2025-11-19T14:00:22.187639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436055723439289:2081] [node 5] ICS09 handshake done sender: [6:7574436055723439286:2123] self: [6:7574436055723439287:0] peer: [5:7574436055340089883:0] socket: 315 qp: -1
2025-11-19T14:00:22.187647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436055723439289:2081] [node 5] ICS10 traffic start
2025-11-19T14:00:22.187654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:22.184588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436050954485905:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(35272372ns) {HandleHandshakeStatus:329(35271668ns) {DropIncomingHandshake:441(11967ns) DropOutgoingHandshake:456(307ns) SwitchToState:178(214ns) ProcessPendingSessionEvents:495(8504748ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8484220ns max# 8401510ns}}}
2025-11-19T14:00:22.187097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436054720678513:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.187124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:22.187138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436054720678513:2050] [node 3] ICP22 created new session: [4:7574436059015645906:2050]
2025-11-19T14:00:22.187145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436059015645906:2050] [node 3] ICS09 handshake done sender: [4:7574436059015645900:2119] self: [4:7574436059015645901:0] peer: [3:7574436055039759666:0] socket: 358 qp: -1
2025-11-19T14:00:22.187152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436059015645906:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:22.187161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:22.189030Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436059015645887:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:22.191255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436054720678512:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.191269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:22.191275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436054720678512:2049] [node 2] ICP22 created new session: [4:7574436059015645908:2049]
2025-11-19T14:00:22.191281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436059015645908:2049] [node 2] ICS09 handshake done sender: [4:7574436059015645887:2116] self: [4:7574436059015645898:0] peer: [2:7574436055951820244:0] socket: 322 qp: -1
2025-11-19T14:00:22.191287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436059015645908:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:22.191294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:22.193724Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436055951820250:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:22.196466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436055951820231:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.196488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:22.196496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436055951820231:2107] [node 5] ICP22 created new session: [2:7574436055951820269:2107]
2025-11-19T14:00:22.196502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436055951820269:2107] [node 5] ICS09 handshake done sender: [2:7574436055951820250:2116] self: [2:7574436055951820247:0] peer: [5:7574436055340089911:0] socket: 338 qp: -1
2025-11-19T14:00:22.196507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436055951820269:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:22.196515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:22.197020Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436055039759692:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.197067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436050744792298:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:22.199935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436051045122519:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14725) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436055340089898:2119] held: no
2025-11-19T14:00:22.197861Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436055951820251:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:22.197874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436055951820251:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:22.200106Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436055951820251:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:22.200203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436051656852874:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:22.200220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:22.200228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436051656852874:2049] [node 3] ICP22 created new session: [2:7574436055951820272:2049]
2025-11-19T14:00:22.200247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436055951820272:2049] [node 3] ICS09 handshake done sender: [2:7574436055951820251:2117] self: [2:7574436055951820248:0] peer: [3:7574436055039759695:0] socket: 360 qp: -1
2025-11-19T14:00:22.200253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436055951820272:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:22.200261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:22.201032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436055951820243:2113] [node 4] ICP35 event processing took too much time PendingConnection:278(22595278ns) {HandleHandshakeStatus:329(22594650ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(200ns) SwitchToState:178(240ns) ProcessPendingSessionEvents:495(22532413ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22511517ns max# 22436016ns}}}
2025-11-19T14:00:22.201183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436055951820243:2113] [node 4] ICP18 handshake (actor [2:7574436055951820258:2119]) is too old
2025-11-19T14:00:22.208341Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436050744792298:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14725) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436055039759692:2120] held: no
2025-11-19T14:00:22.208449Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436055039759665:2112] [node 4] ICP35 event processing took too much time PendingConnection:278(32731040ns) {HandleHandshakeStatus:329(32729837ns) {DropIncomingHandshake:441(7194ns) DropOutgoingHandshake:456(288ns) SwitchToState:178(246ns) ProcessPendingSessionEvents:495(15013825ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14990715ns max# 14902594ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.209540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436055340089916:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10574
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:22.210192Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436055340089898:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:22.209721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436055039759696:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56926
2025-11-19T14:00:22.210314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436051045122519:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.210333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:22.210341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436051045122519:2049] [node 2] ICP22 created new session: [5:7574436055340089917:2049]
2025-11-19T14:00:22.210348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436055340089917:2049] [node 2] ICS09 handshake done sender: [5:7574436055340089898:2119] self: [5:7574436055340089911:0] peer: [2:7574436055951820247:0] socket: 339 qp: -1
2025-11-19T14:00:22.210355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436055340089917:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:22.210366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:22.210888Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436055039759692:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:22.210974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436050744792298:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:22.210989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:22.210995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436050744792298:2049] [node 2] ICP22 created new session: [3:7574436055039759697:2049]
2025-11-19T14:00:22.211001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436055039759697:2049] [node 2] ICS09 handshake done sender: [3:7574436055039759692:2120] self: [3:7574436055039759695:0] peer: [2:7574436055951820248:0] socket: 362 qp: -1
2025-11-19T14:00:22.211006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436055039759697:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:22.211013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:22.223112Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:22.223214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436055951820253:2048] [node 1] ICS01 socket: 301 reason# EndOfStream
2025-11-19T14:00:22.223224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436051656852873:2048] [node 1] ICP30 unregister session Session# [2:7574436055951820253:2048] VirtualId# [2:7574436055951820241:0]
2025-11-19T14:00:22.223232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436055951820253:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:22.223267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:22.223288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:22.223784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:22.223847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436055039759680:2048] [node 1] ICS01 socket: 348 reason# EndOfStream
2025-11-19T14:00:22.223853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436050744792297:2048] [node 1] ICP30 unregister session Session# [3:7574436055039759680:2048] VirtualId# [3:7574436055039759678:0]
2025-11-19T14:00:22.223859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436055039759680:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:22.223885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:22.223901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:22.245630Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:22.245738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436059015645904:2048] [node 1] ICS01 socket: 331 reason# EndOfStream
2025-11-19T14:00:22.245748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436054720678511:2048] [node 1] ICP30 unregister session Session# [4:7574436059015645904:2048] VirtualId# [4:7574436059015645893:0]
2025-11-19T14:00:22.245756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436059015645904:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:22.245811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:22.245835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:22.246020Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:22.246080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436055340089904:2048] [node 1] ICS01 socket: 326 reason# EndOfStream
2025-11-19T14:00:22.246083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436051045122518:2048] [node 1] ICP30 unregister session Session# [5:7574436055340089904:2048] VirtualId# [5:7574436055340089896:0]
2025-11-19T14:00:22.246087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436055340089904:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:22.246114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:22.246136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:22.245873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:22.245935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436055723439277:2048] [node 1] ICS01 socket: 323 reason# EndOfStream
2025-11-19T14:00:22.245939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436051428471891:2048] [node 1] ICP30 unregister session Session# [6:7574436055723439277:2048] VirtualId# [6:7574436055723439267:0]
2025-11-19T14:00:22.245943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436055723439277:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:22.245971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:22.245993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:22.250523Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:22.250617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436055039759697:2049] [node 2] ICS01 socket: 362 reason# EndOfStream
2025-11-19T14:00:22.250627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436050744792298:2049] [node 2] ICP30 unregister session Session# [3:7574436055039759697:2049] VirtualId# [3:7574436055039759695:0]
2025-11-19T14:00:22.250635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436055039759697:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:22.250663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:22.250682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:22.262491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:22.262604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436055340089917:2049] [node 2] ICS01 socket: 339 reason# EndOfStream
2025-11-19T14:00:22.262614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436051045122519:2049] [node 2] ICP30 unregister session Session# [5:7574436055340089917:2049] VirtualId# [5:7574436055340089911:0]
2025-11-19T14:00:22.262622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436055340089917:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:22.262673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:22.262698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:22.262737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:22.262800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436055723439279:2049] [node 2] ICS07 socket disconnect 324 reason# EndOfStream
2025-11-19T14:00:22.262805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436055723439279:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:22.262832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:22.262851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436055723439279:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:22.263073Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:22.263137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436059015645908:2049] [node 2] ICS01 socket: 322 reason# EndOfStream
2025-11-19T14:00:22.263143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436054720678512:2049] [node 2] ICP30 unregister session Session# [4:7574436059015645908:2049] VirtualId# [4:7574436059015645898:0]
2025-11-19T14:00:22.263147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436059015645908:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:22.263182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:22.263200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:22.267250Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436051428471892:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14725) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:22.267269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436055723439279:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:22.267275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436051428471892:2049] [node 2] ICP30 unregister session Session# [6:7574436055723439279:2049] VirtualId# [6:7574436055723439275:0]
2025-11-19T14:00:22.267323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:22.267330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436051428471892:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14725) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:22.267333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14725) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:22.276356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436051428471892:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:22.286937Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:22.287048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436055340089909:2050] [node 3] ICS01 socket: 333 reason# EndOfStream
2025-11-19T14:00:22.287057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436051045122520:2050] [node 3] ICP30 unregister session Session# [5:7574436055340089909:2050] VirtualId# [5:7574436055340089906:0]
2025-11-19T14:00:22.287065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436055340089909:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:22.287116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:22.287143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:22.287183Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:22.287239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436055723439284:2050] [node 3] ICS01 socket: 355 reason# EndOfStream
2025-11-19T14:00:22.287246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436051428471893:2050] [node 3] ICP30 unregister session Session# [6:7574436055723439284:2050] VirtualId# [6:7574436055723439282:0]
2025-11-19T14:00:22.287249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436055723439284:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:22.287272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:22.287290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:22.309849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:22.309963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436055723439271:2077] [node 4] ICS01 socket: 313 reason# EndOfStream
2025-11-19T14:00:22.309975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436055723439225:2077] [node 4] ICP30 unregister session Session# [6:7574436055723439271:2077] VirtualId# [6:7574436055723439269:0]
2025-11-19T14:00:22.309982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436055723439271:2077] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:22.310042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:22.310070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:22.640131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436058129869592:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:22.640145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436058129869593:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:22.728035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436055340261555:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:22.728053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436055340261556:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:22.811881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436057765897990:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:22.811894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436057765897991:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:22.899716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436055158038208:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:22.899737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436055158038209:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:22.899743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436055158038210:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:22.963312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436058669258882:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:22.963332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436058669258883:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:22.963352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436058669258884:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:23.029845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436059413531810:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:23.029862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436059413531811:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:23.029867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436059413531812:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.072290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436059635228911:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38164
2025-11-19T14:00:23.072635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436059635228914:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:23.074578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436059453005529:2069] [node 6] ICP01 ready to work
2025-11-19T14:00:23.076064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436062964226207:2071] [node 6] ICP01 ready to work
2025-11-19T14:00:23.076236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436062060865348:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:23.076203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436062424836955:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.078469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436062424836960:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:23.079193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436059413531826:2063] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58412
2025-11-19T14:00:23.081009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436059413531841:2072] [node 4] ICP01 ready to work
2025-11-19T14:00:23.085922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436059453005549:2082] [node 5] ICP01 ready to work
2025-11-19T14:00:23.091945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436062424836954:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:23.092809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436059635228917:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:23.092897Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436059635228911:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.096222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436058669258882:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24245338ns) {Configure:104(24244607ns) {ProcessConfigured:135(23844387ns) {StartInitialHandshake:157(23843674ns) {DropHandshakes:469(576ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:476(1571ns) {SwitchToState:178(922ns)}}}}}
2025-11-19T14:00:23.097080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436062424836966:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13935
2025-11-19T14:00:23.097100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436062424836967:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13935
2025-11-19T14:00:23.097474Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436059413531811:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17003039ns) {Configure:104(17002555ns) {ProcessConfigured:135(16710969ns) {StartInitialHandshake:157(16710580ns) {DropHandshakes:469(556ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(174ns) PrepareNewSessionHandshake:476(1080ns) {SwitchToState:178(812ns)}}}}}
2025-11-19T14:00:23.099928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436055340261555:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(16091611ns) {Configure:104(16090852ns) {ProcessConfigured:135(15768298ns) {StartInitialHandshake:157(15767738ns) {DropHandshakes:469(818ns) {DropIncomingHandshake:441(91ns) DropOutgoingHandshake:456(85ns)} GenerateSessionVirtualId:988(271ns) PrepareNewSessionHandshake:476(1345ns) {SwitchToState:178(602ns)}}}}}
2025-11-19T14:00:23.099190Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436059413531841:2072] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18117542ns) {Configure:104(18116946ns) {ProcessConfigured:135(17901970ns) {StartInitialHandshake:157(17901609ns) {DropHandshakes:469(416ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:476(787ns) {SwitchToState:178(439ns)}}}}}
2025-11-19T14:00:23.100036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436055340261555:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.101400Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436059413531826:2063] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.101531Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062424836949:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.101547Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436062424836949:2107] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.102569Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436062424836949:2107] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.103086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436062964226243:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1067
2025-11-19T14:00:23.102706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436058129869592:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.102729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:23.102737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436058129869592:2048] [node 2] ICP22 created new session: [1:7574436062424836969:2048]
2025-11-19T14:00:23.102743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436062424836969:2048] [node 2] ICS09 handshake done sender: [1:7574436062424836949:2107] self: [1:7574436062424836948:0] peer: [2:7574436059635228925:0] socket: 300 qp: -1
2025-11-19T14:00:23.102753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436062424836969:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:23.102764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:23.108165Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436062964226243:2104] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.108235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436058669258882:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.103873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436059413531810:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(19966944ns) {Configure:104(19966005ns) {ProcessConfigured:135(19702800ns) {StartInitialHandshake:157(19702074ns) {DropHandshakes:469(487ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(1664ns) {SwitchToState:178(726ns)}}}}}
2025-11-19T14:00:23.103960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436059413531810:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.109211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436062964226207:2071] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(33083366ns) {Configure:104(33082245ns) {ProcessConfigured:135(32792265ns) {StartInitialHandshake:157(32791497ns) {DropHandshakes:469(607ns) {DropIncomingHandshake:441(78ns) DropOutgoingHandshake:456(184ns)} GenerateSessionVirtualId:988(152ns) PrepareNewSessionHandshake:476(779ns) {SwitchToState:178(363ns)}}}}}
2025-11-19T14:00:23.111938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062424836964:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.111958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436062424836964:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.114707Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062424836967:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.114773Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436062424836964:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:23.114798Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062424836962:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.114806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436062424836962:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.116063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436062060865352:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:23.115929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436062964226213:2069] [node 4] ICP01 ready to work
2025-11-19T14:00:23.115951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436059635228919:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:23.116423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436059413531856:2079] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.118306Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062424836966:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.118566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436062424836955:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.118578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:23.118595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436062424836955:2109] [node 5] ICP22 created new session: [1:7574436062424836974:2109]
2025-11-19T14:00:23.118601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436062424836974:2109] [node 5] ICS09 handshake done sender: [1:7574436062424836964:2113] self: [1:7574436062424836956:0] peer: [5:7574436062964226244:0] socket: 321 qp: -1
2025-11-19T14:00:23.118608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436062424836974:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:23.118619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:23.118848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436062424836955:2109] [node 5] ICP18 handshake (actor [1:7574436062424836967:2115]) is too old
2025-11-19T14:00:23.118883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436062424836973:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13935
2025-11-19T14:00:23.120031Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436055158038209:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(30882268ns) {Configure:104(30881403ns) {ProcessConfigured:135(30595054ns) {StartInitialHandshake:157(30594410ns) {DropHandshakes:469(455ns) {DropIncomingHandshake:441(105ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(9614ns) PrepareNewSessionHandshake:476(1240ns) {SwitchToState:178(740ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.120674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436059635228914:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(47984328ns) {Configure:104(47983921ns) {ProcessConfigured:135(47608410ns) {StartInitialHandshake:157(47607725ns) {DropHandshakes:469(497ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(66ns)} GenerateSessionVirtualId:988(116ns) PrepareNewSessionHandshake:476(1162ns) {SwitchToState:178(852ns)}}}}}
2025-11-19T14:00:23.121063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436059413531889:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58412
2025-11-19T14:00:23.121116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436059413531890:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58412
2025-11-19T14:00:23.125052Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436062424836954:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(32949067ns) {Configure:104(32947683ns) {ProcessConfigured:135(32595530ns) {StartInitialHandshake:157(32594773ns) {DropHandshakes:469(404ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:476(1321ns) {SwitchToState:178(854ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.126880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059453005582:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44614
2025-11-19T14:00:23.126927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059453005583:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44614
2025-11-19T14:00:23.125334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436058669258884:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(51872134ns) {Configure:104(51871591ns) {ProcessConfigured:135(51613105ns) {StartInitialHandshake:157(51612722ns) {DropHandshakes:469(380ns) {DropIncomingHandshake:441(101ns) DropOutgoingHandshake:456(109ns)} GenerateSessionVirtualId:988(149ns) PrepareNewSessionHandshake:476(592ns) {SwitchToState:178(147ns)}}}}}
2025-11-19T14:00:23.125360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436058669258882:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28470) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436062964226243:2104] held: no
2025-11-19T14:00:23.127897Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436059453005582:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.126476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436062060865355:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27659
2025-11-19T14:00:23.126544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436062060865356:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27659
2025-11-19T14:00:23.126586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436062060865357:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27659
2025-11-19T14:00:23.126622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436062060865358:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27659
2025-11-19T14:00:23.127410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436062060865353:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:23.127493Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436062060865356:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.125764Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436059635228917:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(32793502ns) {Configure:104(32793040ns) {ProcessConfigured:135(32486669ns) {StartInitialHandshake:157(32486467ns) {DropHandshakes:469(347ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(31ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(663ns) {SwitchToState:178(316ns)}}}}}
2025-11-19T14:00:23.126849Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436059413531890:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.126884Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436059413531889:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.126967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436059413531841:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.135062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436059635228933:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38164
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.135320Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436062060865357:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.135374Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436062060865358:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.135399Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436062060865355:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.136047Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436062424836962:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:23.136180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436062424836960:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.136189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:23.136199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436062424836960:2111] [node 6] ICP22 created new session: [1:7574436062424836980:2111]
2025-11-19T14:00:23.136205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436062424836980:2111] [node 6] ICS09 handshake done sender: [1:7574436062424836962:2112] self: [1:7574436062424836961:0] peer: [6:7574436059413531882:0] socket: 304 qp: -1
2025-11-19T14:00:23.136211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436062424836980:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:23.136255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.136859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436062424836982:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13935
2025-11-19T14:00:23.140521Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436062964226213:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24448496ns) {Configure:104(24446870ns) {ProcessConfigured:135(24079618ns) {StartInitialHandshake:157(24079229ns) {DropHandshakes:469(813ns) {DropIncomingHandshake:441(125ns) DropOutgoingHandshake:456(106ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:476(1158ns) {SwitchToState:178(441ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.135823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436059635228932:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38164
2025-11-19T14:00:23.135874Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436059635228911:2107] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:23.136009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436055340261555:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.136033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:23.136041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436055340261555:2048] [node 1] ICP22 created new session: [2:7574436059635228934:2048]
2025-11-19T14:00:23.136050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436059635228934:2048] [node 1] ICS09 handshake done sender: [2:7574436059635228911:2107] self: [2:7574436059635228925:0] peer: [1:7574436062424836948:0] socket: 301 qp: -1
2025-11-19T14:00:23.136058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436059635228934:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:23.136068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:23.140329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436059635228919:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(24232231ns) {Configure:104(24231676ns) {ProcessConfigured:135(23889195ns) {StartInitialHandshake:157(23888373ns) {DropHandshakes:469(1457ns) {DropIncomingHandshake:441(170ns) DropOutgoingHandshake:456(174ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:476(1291ns) {SwitchToState:178(508ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.141147Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436059413531856:2079] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(24620428ns) {Configure:104(24620242ns) {ProcessConfigured:135(24340487ns) {StartInitialHandshake:157(24340045ns) {DropHandshakes:469(222ns) {DropIncomingHandshake:441(16ns) DropOutgoingHandshake:456(26ns)} GenerateSessionVirtualId:988(142ns) PrepareNewSessionHandshake:476(993ns) {SwitchToState:178(63ns)}}}}}
2025-11-19T14:00:23.141301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436059413531856:2079] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.141386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436059413531893:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58412
2025-11-19T14:00:23.141569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436059413531894:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58412
2025-11-19T14:00:23.136380Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436057765897990:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(52396322ns) {Configure:104(52395089ns) {ProcessConfigured:135(52092168ns) {StartInitialHandshake:157(52091194ns) {DropHandshakes:469(370ns) {DropIncomingHandshake:441(70ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:476(1004ns) {SwitchToState:178(440ns)}}}}}
2025-11-19T14:00:23.136458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436057765897990:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.139843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436062060865348:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(63536566ns) {Configure:104(63535834ns) {ProcessConfigured:135(63146267ns) {StartInitialHandshake:157(63145544ns) {DropHandshakes:469(435ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(138ns) PrepareNewSessionHandshake:476(1264ns) {SwitchToState:178(524ns)}}}}}
2025-11-19T14:00:23.140912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436057765897991:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(24781981ns) {Configure:104(24781328ns) {ProcessConfigured:135(24446244ns) {StartInitialHandshake:157(24445689ns) {DropHandshakes:469(536ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:476(837ns) {SwitchToState:178(352ns)}}}}}
2025-11-19T14:00:23.140997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436057765897991:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.143983Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436059635228921:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.143995Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436059635228921:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.144501Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436059635228932:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.144967Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436059635228921:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:23.144954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059453005585:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44614
2025-11-19T14:00:23.145081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436055340261556:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.145109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:23.145116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436055340261556:2049] [node 3] ICP22 created new session: [2:7574436059635228940:2049]
2025-11-19T14:00:23.145123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436059635228940:2049] [node 3] ICS09 handshake done sender: [2:7574436059635228921:2111] self: [2:7574436059635228916:0] peer: [3:7574436062060865370:0] socket: 310 qp: -1
2025-11-19T14:00:23.145130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436059635228940:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:23.145150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:23.148500Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436059413531826:2063] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:23.148690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436059413531810:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.148739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:23.148745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436059413531810:2048] [node 1] ICP22 created new session: [6:7574436059413531896:2048]
2025-11-19T14:00:23.148752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436059413531896:2048] [node 1] ICS09 handshake done sender: [6:7574436059413531826:2063] self: [6:7574436059413531882:0] peer: [1:7574436062424836961:0] socket: 305 qp: -1
2025-11-19T14:00:23.148759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436059413531896:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:23.148786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:23.148811Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062424836957:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.148841Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436062424836957:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.149152Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436059453005548:2087] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.149180Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436059453005548:2087] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.149584Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436062964226245:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.149607Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436062964226245:2105] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.149224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436059635228931:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38164
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.152170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436062060865353:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24534832ns) {Configure:104(24533817ns) {ProcessConfigured:135(24219413ns) {StartInitialHandshake:157(24218044ns) {DropHandshakes:469(790ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(180ns) PrepareNewSessionHandshake:476(1181ns) {SwitchToState:178(810ns)}}}}}
2025-11-19T14:00:23.152993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436059413531898:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58412
2025-11-19T14:00:23.153433Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436059635228931:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.153491Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436059453005583:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.153531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436055158038208:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.153956Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436059453005585:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.153988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436055158038209:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.153687Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062424836976:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.153697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436062424836976:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.154413Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436062424836976:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:23.154134Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436059635228930:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.154140Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436059635228930:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.154543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436062424836954:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.154562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:23.154568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436062424836954:2108] [node 4] ICP22 created new session: [1:7574436062424836984:2108]
2025-11-19T14:00:23.154575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436062424836984:2108] [node 4] ICS09 handshake done sender: [1:7574436062424836976:2117] self: [1:7574436062424836965:0] peer: [4:7574436059453005586:0] socket: 316 qp: -1
2025-11-19T14:00:23.154580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436062424836984:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:23.154588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.155968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436062964226259:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1067
2025-11-19T14:00:23.156921Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436062424836957:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.156191Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436059635228930:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:23.157059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436062964226260:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1067
2025-11-19T14:00:23.157133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436059413531900:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58412
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.157349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436059635228917:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.157363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:23.157370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436059635228917:2108] [node 4] ICP22 created new session: [2:7574436059635228945:2108]
2025-11-19T14:00:23.157375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436059635228945:2108] [node 4] ICS09 handshake done sender: [2:7574436059635228930:2114] self: [2:7574436059635228923:0] peer: [4:7574436059453005587:0] socket: 344 qp: -1
2025-11-19T14:00:23.157380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436059635228945:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:23.157390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:23.157214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436062964226261:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1067
2025-11-19T14:00:23.157235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436062964226262:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1067
2025-11-19T14:00:23.157781Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436062964226260:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.157807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436058669258883:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.158010Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436062964226243:2104] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:23.158135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436058669258882:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.158146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:23.158153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436058669258882:2048] [node 1] ICP22 created new session: [5:7574436062964226264:2048]
2025-11-19T14:00:23.158158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436062964226264:2048] [node 1] ICS09 handshake done sender: [5:7574436062964226243:2104] self: [5:7574436062964226244:0] peer: [1:7574436062424836956:0] socket: 322 qp: -1
2025-11-19T14:00:23.158165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436062964226264:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:23.158192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:23.158582Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436059635228928:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.158591Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436059635228928:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.159483Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436059635228928:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:23.159647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436059635228914:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.159668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:23.159673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436059635228914:2109] [node 5] ICP22 created new session: [2:7574436059635228948:2109]
2025-11-19T14:00:23.159679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436059635228948:2109] [node 5] ICS09 handshake done sender: [2:7574436059635228928:2113] self: [2:7574436059635228915:0] peer: [5:7574436062964226263:0] socket: 328 qp: -1
2025-11-19T14:00:23.159684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436059635228948:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:23.159691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:23.159689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436062424836960:2111] [node 6] ICP35 event processing took too much time PendingConnection:278(23396336ns) {HandleHandshakeStatus:329(23395968ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(192ns) SwitchToState:178(230ns) ProcessPendingSessionEvents:495(10056821ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10038698ns max# 9949069ns}}}
2025-11-19T14:00:23.159813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436058129869593:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.159824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:23.159832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436058129869593:2049] [node 3] ICP22 created new session: [1:7574436062424836987:2049]
2025-11-19T14:00:23.159832Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574436058129869592:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:23.159839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436062424836987:2049] [node 3] ICS09 handshake done sender: [1:7574436062424836957:2110] self: [1:7574436062424836953:0] peer: [3:7574436062060865364:0] socket: 302 qp: -1
2025-11-19T14:00:23.159844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436062424836987:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:23.159878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.160357Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436059453005548:2087] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:23.160509Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436059413531889:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.160534Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436058669258883:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5269) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436062964226260:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.160953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436062060865373:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27659
2025-11-19T14:00:23.161290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436062060865374:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27659
2025-11-19T14:00:23.161351Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436062060865357:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:23.160581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436059413531841:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.160596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:23.160600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436059413531841:2072] [node 4] ICP22 created new session: [6:7574436059413531901:2072]
2025-11-19T14:00:23.160606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436059413531901:2072] [node 4] ICS09 handshake done sender: [6:7574436059413531889:2115] self: [6:7574436059413531891:0] peer: [4:7574436059453005531:0] socket: 314 qp: -1
2025-11-19T14:00:23.160610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436059413531901:2072] [node 4] ICS10 traffic start
2025-11-19T14:00:23.160624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:23.161752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436059453005529:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.161775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:23.161782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436059453005529:2069] [node 6] ICP22 created new session: [4:7574436059453005591:2069]
2025-11-19T14:00:23.161788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436059453005591:2069] [node 6] ICS09 handshake done sender: [4:7574436059453005548:2087] self: [4:7574436059453005531:0] peer: [6:7574436059413531891:0] socket: 311 qp: -1
2025-11-19T14:00:23.161793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436059453005591:2069] [node 6] ICS10 traffic start
2025-11-19T14:00:23.161809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:23.163578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436057765897991:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.163595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:23.163602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436057765897991:2049] [node 2] ICP22 created new session: [3:7574436062060865376:2049]
2025-11-19T14:00:23.163608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436062060865376:2049] [node 2] ICS09 handshake done sender: [3:7574436062060865357:2112] self: [3:7574436062060865370:0] peer: [2:7574436059635228916:0] socket: 331 qp: -1
2025-11-19T14:00:23.163614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436062060865376:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:23.163631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.163355Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436059413531898:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.163399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436059413531811:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.163998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436062060865375:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27659
2025-11-19T14:00:23.163926Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436062964226245:2105] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:23.164115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436062964226266:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1067
2025-11-19T14:00:23.164136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436062964226207:2071] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.164145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:23.164150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436062964226207:2071] [node 6] ICP22 created new session: [5:7574436062964226268:2071]
2025-11-19T14:00:23.164156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436062964226268:2071] [node 6] ICS09 handshake done sender: [5:7574436062964226245:2105] self: [5:7574436062964226208:0] peer: [6:7574436059413531895:0] socket: 323 qp: -1
2025-11-19T14:00:23.164162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436062964226268:2071] [node 6] ICS10 traffic start
2025-11-19T14:00:23.164177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:23.164246Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436062964226260:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.170109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059453005590:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44614
2025-11-19T14:00:23.172262Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436062060865374:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.172306Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436062060865355:2110] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:23.174540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436057765897990:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.174564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:23.174574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436057765897990:2048] [node 1] ICP22 created new session: [3:7574436062060865380:2048]
2025-11-19T14:00:23.174582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436062060865380:2048] [node 1] ICS09 handshake done sender: [3:7574436062060865355:2110] self: [3:7574436062060865364:0] peer: [1:7574436062424836953:0] socket: 308 qp: -1
2025-11-19T14:00:23.174589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436062060865380:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:23.174614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:23.175828Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436059453005585:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.176138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436059413531904:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58412
2025-11-19T14:00:23.175964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436055158038209:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.175979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:23.175984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436055158038209:2049] [node 2] ICP22 created new session: [4:7574436059453005595:2049]
2025-11-19T14:00:23.175990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436059453005595:2049] [node 2] ICS09 handshake done sender: [4:7574436059453005585:2117] self: [4:7574436059453005587:0] peer: [2:7574436059635228923:0] socket: 345 qp: -1
2025-11-19T14:00:23.175995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436059453005595:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:23.176002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:23.176275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059453005589:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44614
2025-11-19T14:00:23.176594Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436059413531890:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:23.176819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436059413531856:2079] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.176833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:23.176839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436059413531856:2079] [node 5] ICP22 created new session: [6:7574436059413531905:2079]
2025-11-19T14:00:23.176844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436059413531905:2079] [node 5] ICS09 handshake done sender: [6:7574436059413531890:2116] self: [6:7574436059413531895:0] peer: [5:7574436062964226208:0] socket: 329 qp: -1
2025-11-19T14:00:23.176850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436059413531905:2079] [node 5] ICS10 traffic start
2025-11-19T14:00:23.176857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:23.176301Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436059635228936:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.176317Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436059635228936:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.179783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436055158038208:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28470) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436059453005583:2116] held: no
2025-11-19T14:00:23.179985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436055158038208:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(17743863ns) {HandleHandshakeStatus:395(17743346ns) {LogHandshakeFail:472(2162ns) {UpdateErrorStateLog:544(1046ns)} DropOutgoingHandshake:456(606ns) ProcessPendingSessionEvents:495(88701ns) {unordered DropSessionEvent:512 num# 100 sum# 82661ns max# 4064ns}}}
2025-11-19T14:00:23.180296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436057765897991:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(16630971ns) {HandleHandshakeStatus:329(16630477ns) {DropIncomingHandshake:441(156ns) DropOutgoingHandshake:456(2064ns) SwitchToState:178(86ns) ProcessPendingSessionEvents:495(111984ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 91075ns max# 23042ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.184016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436058669258883:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.184043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:23.184054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436058669258883:2049] [node 2] ICP22 created new session: [5:7574436062964226271:2049]
2025-11-19T14:00:23.184061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436062964226271:2049] [node 2] ICS09 handshake done sender: [5:7574436062964226260:2117] self: [5:7574436062964226263:0] peer: [2:7574436059635228915:0] socket: 354 qp: -1
2025-11-19T14:00:23.184068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436062964226271:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:23.184089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:23.184332Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436062964226262:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.184536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436062964226273:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1067
2025-11-19T14:00:23.188309Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436059453005583:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.199988Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436059635228936:2118] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:23.200331Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436059413531894:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.200391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436059413531812:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.200542Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436062060865366:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.200561Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436062060865366:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.190022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059453005598:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44614
2025-11-19T14:00:23.200189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059453005594:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44614
2025-11-19T14:00:23.201576Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436062964226207:2071] [node 6] ICP35 event processing took too much time PendingConnection:278(37356451ns) {HandleHandshakeStatus:329(37355926ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(141ns) SwitchToState:178(71ns) ProcessPendingSessionEvents:495(22635614ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22620942ns max# 22544495ns}}}
2025-11-19T14:00:23.201724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574436062964226207:2071] [node 6] ICP18 handshake (actor [5:7574436062964226262:2119]) is too old
2025-11-19T14:00:23.200541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574436059413531856:2079] [node 5] ICP35 event processing took too much time PendingConnection:278(23623851ns) {HandleHandshakeStatus:329(23622870ns) {DropIncomingHandshake:441(131ns) DropOutgoingHandshake:456(1156ns) SwitchToState:178(520ns) ProcessPendingSessionEvents:495(23540527ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23513304ns max# 23426689ns}}}
2025-11-19T14:00:23.201066Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436055158038209:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(25015435ns) {HandleHandshakeStatus:329(25014642ns) {DropIncomingHandshake:441(148ns) DropOutgoingHandshake:456(961ns) SwitchToState:178(257ns) ProcessPendingSessionEvents:495(24023951ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24003313ns max# 23918233ns}}}
2025-11-19T14:00:23.201111Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436059453005598:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.201906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436055158038208:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.201921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:23.201929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436055158038208:2048] [node 1] ICP22 created new session: [4:7574436059453005600:2048]
2025-11-19T14:00:23.201936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436059453005600:2048] [node 1] ICS09 handshake done sender: [4:7574436059453005583:2116] self: [4:7574436059453005586:0] peer: [1:7574436062424836965:0] socket: 334 qp: -1
2025-11-19T14:00:23.201945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436059453005600:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:23.201982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.202724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436059413531908:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58412
2025-11-19T14:00:23.202650Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436062964226273:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.202676Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436062964226261:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.202702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436062964226213:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.202711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436058669258884:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.202875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436059635228919:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.202909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:23.202917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436059635228919:2110] [node 6] ICP22 created new session: [2:7574436059635228951:2110]
2025-11-19T14:00:23.202924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436059635228951:2110] [node 6] ICS09 handshake done sender: [2:7574436059635228936:2118] self: [2:7574436059635228927:0] peer: [6:7574436059413531903:0] socket: 347 qp: -1
2025-11-19T14:00:23.202931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436059635228951:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:23.202944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:23.203249Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436059413531898:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:23.203320Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436062060865360:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.203331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436062060865360:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.203314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436059413531811:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.203325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:23.203330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436059413531811:2049] [node 2] ICP22 created new session: [6:7574436059413531909:2049]
2025-11-19T14:00:23.203336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436059413531909:2049] [node 2] ICS09 handshake done sender: [6:7574436059413531898:2120] self: [6:7574436059413531903:0] peer: [2:7574436059635228927:0] socket: 315 qp: -1
2025-11-19T14:00:23.203341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436059413531909:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:23.203346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:23.203419Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436062060865366:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:23.203524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436062060865348:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.203538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:23.203543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436062060865348:2108] [node 6] ICP22 created new session: [3:7574436062060865383:2108]
2025-11-19T14:00:23.203549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436062060865383:2108] [node 6] ICS09 handshake done sender: [3:7574436062060865366:2116] self: [3:7574436062060865349:0] peer: [6:7574436059413531907:0] socket: 340 qp: -1
2025-11-19T14:00:23.203553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436062060865383:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:23.203573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:23.203617Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436059453005553:2089] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.203626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436059453005553:2089] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.204499Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436059453005553:2089] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:23.204571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436059453005549:2082] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.204589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:23.204594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436059453005549:2082] [node 5] ICP22 created new session: [4:7574436059453005603:2082]
2025-11-19T14:00:23.204600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436059453005603:2082] [node 5] ICS09 handshake done sender: [4:7574436059453005553:2089] self: [4:7574436059453005552:0] peer: [5:7574436062964226274:0] socket: 318 qp: -1
2025-11-19T14:00:23.204604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436059453005603:2082] [node 5] ICS10 traffic start
2025-11-19T14:00:23.204609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.209380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436059413531911:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58412
2025-11-19T14:00:23.209704Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436059413531894:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:23.210094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436059413531812:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.210114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:23.210120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436059413531812:2050] [node 3] ICP22 created new session: [6:7574436059413531912:2050]
2025-11-19T14:00:23.210127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436059413531912:2050] [node 3] ICS09 handshake done sender: [6:7574436059413531894:2119] self: [6:7574436059413531907:0] peer: [3:7574436062060865349:0] socket: 342 qp: -1
2025-11-19T14:00:23.210135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436059413531912:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:23.210158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:23.209119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436062964226276:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1067
2025-11-19T14:00:23.209983Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436062964226273:2121] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:23.211650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436062964226213:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.211669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:23.211676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436062964226213:2069] [node 4] ICP22 created new session: [5:7574436062964226277:2069]
2025-11-19T14:00:23.211683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436062964226277:2069] [node 4] ICS09 handshake done sender: [5:7574436062964226273:2121] self: [5:7574436062964226274:0] peer: [4:7574436059453005552:0] socket: 338 qp: -1
2025-11-19T14:00:23.211690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436062964226277:2069] [node 4] ICS10 traffic start
2025-11-19T14:00:23.211698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:23.214086Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436062060865360:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.216296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436062964226279:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1067
2025-11-19T14:00:23.216827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436062060865352:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.216854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:23.216861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436062060865352:2107] [node 5] ICP22 created new session: [3:7574436062060865386:2107]
2025-11-19T14:00:23.216868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436062060865386:2107] [node 5] ICS09 handshake done sender: [3:7574436062060865360:2114] self: [3:7574436062060865359:0] peer: [5:7574436062964226275:0] socket: 333 qp: -1
2025-11-19T14:00:23.216874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436062060865386:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:23.216882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:23.217474Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436059453005594:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.217700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436055158038210:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.218194Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436062964226261:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:23.218340Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436062060865371:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.218348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436062060865371:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.219343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436059453005606:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44614
2025-11-19T14:00:23.219598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436058669258884:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.219620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:23.219627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436058669258884:2050] [node 3] ICP22 created new session: [5:7574436062964226280:2050]
2025-11-19T14:00:23.219634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436062964226280:2050] [node 3] ICS09 handshake done sender: [5:7574436062964226261:2118] self: [5:7574436062964226275:0] peer: [3:7574436062060865359:0] socket: 355 qp: -1
2025-11-19T14:00:23.219640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436062964226280:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:23.219649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:23.220974Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436062060865371:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:23.221153Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436059453005594:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:23.221266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436055158038210:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.221283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:23.221288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436055158038210:2050] [node 3] ICP22 created new session: [4:7574436059453005607:2050]
2025-11-19T14:00:23.221294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436059453005607:2050] [node 3] ICS09 handshake done sender: [4:7574436059453005594:2120] self: [4:7574436059453005605:0] peer: [3:7574436062060865362:0] socket: 303 qp: -1
2025-11-19T14:00:23.221299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436059453005607:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:23.221319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:23.222700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436062060865353:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.222716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:23.222721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436062060865353:2109] [node 4] ICP22 created new session: [3:7574436062060865389:2109]
2025-11-19T14:00:23.222726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436062060865389:2109] [node 4] ICS09 handshake done sender: [3:7574436062060865371:2118] self: [3:7574436062060865362:0] peer: [4:7574436059453005605:0] socket: 319 qp: -1
2025-11-19T14:00:23.222730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436062060865389:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:23.222735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:23.249221Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:23.249342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436059635228934:2048] [node 1] ICS07 socket disconnect 301 reason# EndOfStream
2025-11-19T14:00:23.249349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436059635228934:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.249388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:23.249404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436059635228934:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:23.250157Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:23.250224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436062060865380:2048] [node 1] ICS01 socket: 308 reason# EndOfStream
2025-11-19T14:00:23.250232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436057765897990:2048] [node 1] ICP30 unregister session Session# [3:7574436062060865380:2048] VirtualId# [3:7574436062060865364:0]
2025-11-19T14:00:23.250238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436062060865380:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.250262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:23.250282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:23.256405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436055340261555:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28470) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T14:00:23.256422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436059635228934:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:23.256840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436055340261555:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28470) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:23.256851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436059635228934:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:23.256856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436055340261555:2048] [node 1] ICP30 unregister session Session# [2:7574436059635228934:2048] VirtualId# [2:7574436059635228925:0]
2025-11-19T14:00:23.256912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:23.256919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574436055340261555:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28470) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:23.256922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28470) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:23.259790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:23.259739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:23.259843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436059453005600:2048] [node 1] ICS01 socket: 334 reason# EndOfStream
2025-11-19T14:00:23.259851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436055158038208:2048] [node 1] ICP30 unregister session Session# [4:7574436059453005600:2048] VirtualId# [4:7574436059453005586:0]
2025-11-19T14:00:23.259859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436059453005600:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.259917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:23.259944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:23.259877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436062964226264:2048] [node 1] ICS01 socket: 322 reason# EndOfStream
2025-11-19T14:00:23.259884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436058669258882:2048] [node 1] ICP30 unregister session Session# [5:7574436062964226264:2048] VirtualId# [5:7574436062964226244:0]
2025-11-19T14:00:23.259889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436062964226264:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.259940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:23.259965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:23.260065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:23.260128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436059413531896:2048] [node 1] ICS07 socket disconnect 305 reason# EndOfStream
2025-11-19T14:00:23.260133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436059413531896:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.260162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:23.260180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436059413531896:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:23.260601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436059413531810:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28470) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:23.260612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436059413531896:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:23.260616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436059413531810:2048] [node 1] ICP30 unregister session Session# [6:7574436059413531896:2048] VirtualId# [6:7574436059413531882:0]
2025-11-19T14:00:23.260659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:23.260664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436059413531810:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28470) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:23.260667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28470) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:23.270438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436059413531810:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:23.270727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:23.270824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436062060865376:2049] [node 2] ICS01 socket: 331 reason# EndOfStream
2025-11-19T14:00:23.270833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436057765897991:2049] [node 2] ICP30 unregister session Session# [3:7574436062060865376:2049] VirtualId# [3:7574436062060865370:0]
2025-11-19T14:00:23.270841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436062060865376:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.270879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:23.270900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:23.307907Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:23.308019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436062964226271:2049] [node 2] ICS01 socket: 354 reason# EndOfStream
2025-11-19T14:00:23.308039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436058669258883:2049] [node 2] ICP30 unregister session Session# [5:7574436062964226271:2049] VirtualId# [5:7574436062964226263:0]
2025-11-19T14:00:23.308048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436062964226271:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.308105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:23.308131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:23.308276Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:23.308338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436059453005595:2049] [node 2] ICS07 socket disconnect 345 reason# EndOfStream
2025-11-19T14:00:23.308342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436059453005595:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.308372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:23.308390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436059453005595:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:23.311351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436055158038209:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5269) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:23.311362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436059453005595:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:23.311367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436055158038209:2049] [node 2] ICP30 unregister session Session# [4:7574436059453005595:2049] VirtualId# [4:7574436059453005587:0]
2025-11-19T14:00:23.311423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:23.311430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436055158038209:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5269) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:23.311433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5269) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:23.311822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:23.311919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436059413531909:2049] [node 2] ICS01 socket: 315 reason# EndOfStream
2025-11-19T14:00:23.311927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436059413531811:2049] [node 2] ICP30 unregister session Session# [6:7574436059413531909:2049] VirtualId# [6:7574436059413531903:0]
2025-11-19T14:00:23.311933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436059413531909:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.311997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:23.312022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:23.323609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574436055158038209:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:23.328384Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:23.328510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436062964226280:2050] [node 3] ICS01 socket: 355 reason# EndOfStream
2025-11-19T14:00:23.328520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436058669258884:2050] [node 3] ICP30 unregister session Session# [5:7574436062964226280:2050] VirtualId# [5:7574436062964226275:0]
2025-11-19T14:00:23.328531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436062964226280:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.328594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:23.328623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:23.328668Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:23.328734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436059413531912:2050] [node 3] ICS01 socket: 342 reason# EndOfStream
2025-11-19T14:00:23.328740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436059413531812:2050] [node 3] ICP30 unregister session Session# [6:7574436059413531912:2050] VirtualId# [6:7574436059413531907:0]
2025-11-19T14:00:23.328745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436059413531912:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.328776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:23.328797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:23.343840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:23.343967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436059413531901:2072] [node 4] ICS01 socket: 314 reason# EndOfStream
2025-11-19T14:00:23.343981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436059413531841:2072] [node 4] ICP30 unregister session Session# [6:7574436059413531901:2072] VirtualId# [6:7574436059413531891:0]
2025-11-19T14:00:23.343990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436059413531901:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.344056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:23.344085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:23.579886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436062448672534:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:23.579914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436062448672535:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:23.612255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436061677077715:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:23.612276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436061677077716:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:23.630159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436063369499162:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:23.630174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436063369499163:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:23.642420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436060517455473:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:23.642443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436060517455474:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:23.642448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436060517455475:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:23.699463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436060986715224:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:23.699489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436060986715225:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:23.699495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436060986715226:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:23.724209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436060986715248:2065] [node 6] ICP01 ready to work
2025-11-19T14:00:23.724309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436060986715250:2068] [node 4] ICP01 ready to work
2025-11-19T14:00:23.725167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436062448672594:2106] [node 5] ICP01 ready to work
2025-11-19T14:00:23.727911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436061677077769:2098] [node 6] ICP01 ready to work
2025-11-19T14:00:23.730988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436060835919357:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:23.731048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436060835919358:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:23.731077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436060835919359:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:23.731427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436060835919395:2061] [node 4] ICP01 ready to work
2025-11-19T14:00:23.731523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436060835919396:2076] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.743514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436060986715250:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(16377267ns) {Configure:104(16376216ns) {ProcessConfigured:135(16130996ns) {StartInitialHandshake:157(16130209ns) {DropHandshakes:469(178ns) {DropIncomingHandshake:441(29ns) DropOutgoingHandshake:456(18ns)} GenerateSessionVirtualId:988(150ns) PrepareNewSessionHandshake:476(1394ns) {SwitchToState:178(823ns)}}}}}
2025-11-19T14:00:23.743888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436060986715286:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64373
2025-11-19T14:00:23.744175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436060517455475:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17250403ns) {Configure:104(17248790ns) {ProcessConfigured:135(16810426ns) {StartInitialHandshake:157(16809671ns) {DropHandshakes:469(585ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(99ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(848ns) {SwitchToState:178(755ns)}}}}}
2025-11-19T14:00:23.744364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436060517455515:2070] [node 6] ICP01 ready to work
2025-11-19T14:00:23.744539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436060517455516:2079] [node 5] ICP01 ready to work
2025-11-19T14:00:23.752194Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436061677077769:2098] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23520660ns) {Configure:104(23519771ns) {ProcessConfigured:135(23333889ns) {StartInitialHandshake:157(23333520ns) {DropHandshakes:469(212ns) {DropIncomingHandshake:441(23ns) DropOutgoingHandshake:456(14ns)} GenerateSessionVirtualId:988(183ns) PrepareNewSessionHandshake:476(1742ns) {SwitchToState:178(406ns)}}}}}
2025-11-19T14:00:23.752436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436061677077785:2103] [node 4] ICP01 ready to work
2025-11-19T14:00:23.752587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436061677077786:2104] [node 5] ICP01 ready to work
2025-11-19T14:00:23.752303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436063369499216:2090] [node 4] ICP01 ready to work
2025-11-19T14:00:23.752529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436063369499217:2094] [node 5] ICP01 ready to work
2025-11-19T14:00:23.752638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436063369499219:2096] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.774719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436063369499231:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20245
2025-11-19T14:00:23.774744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436063369499232:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20245
2025-11-19T14:00:23.776112Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436060835919358:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(44502245ns) {Configure:104(44501700ns) {ProcessConfigured:135(44081601ns) {StartInitialHandshake:157(44081112ns) {DropHandshakes:469(1664ns) {DropIncomingHandshake:441(153ns) DropOutgoingHandshake:456(442ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:476(1424ns) {SwitchToState:178(448ns)}}}}}
2025-11-19T14:00:23.775669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436061677077787:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61453
2025-11-19T14:00:23.775690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436061677077788:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61453
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.789785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436060835919424:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36404
2025-11-19T14:00:23.789814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436060835919425:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36404
2025-11-19T14:00:23.789828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436060835919426:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36404
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.793426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436063369499243:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20245
2025-11-19T14:00:23.793452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436063369499244:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20245
2025-11-19T14:00:23.808377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436062448672606:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.808552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436062448672607:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:23.811515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436062448672601:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25158
2025-11-19T14:00:23.811591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436062448672602:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25158
2025-11-19T14:00:23.811605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436062448672603:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25158
2025-11-19T14:00:23.811619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436062448672604:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25158
2025-11-19T14:00:23.811633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436062448672605:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25158
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.812068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436063369499245:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20245
2025-11-19T14:00:23.812272Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062448672601:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.812955Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436060835919425:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.813061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436060835919359:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.820012Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436063369499234:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.820029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436063369499234:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.821679Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436061677077788:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.821838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436061677077796:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61453
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.822168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436060835919439:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36404
2025-11-19T14:00:23.827939Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062448672605:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.830967Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436063369499234:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:23.831267Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436060835919425:2112] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:23.831354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436060835919359:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.831378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:23.831387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436060835919359:2050] [node 3] ICP22 created new session: [6:7574436060835919440:2050]
2025-11-19T14:00:23.831394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436060835919440:2050] [node 3] ICS09 handshake done sender: [6:7574436060835919425:2112] self: [6:7574436060835919438:0] peer: [3:7574436063369499233:0] socket: 326 qp: -1
2025-11-19T14:00:23.831404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436060835919440:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:23.831416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:23.832531Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436063369499243:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.832560Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436063369499244:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.832613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436063369499219:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.832624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:23.832631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436063369499219:2096] [node 6] ICP22 created new session: [3:7574436063369499247:2096]
2025-11-19T14:00:23.832637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436063369499247:2096] [node 6] ICS09 handshake done sender: [3:7574436063369499234:2113] self: [3:7574436063369499233:0] peer: [6:7574436060835919438:0] socket: 318 qp: -1
2025-11-19T14:00:23.832643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436063369499247:2096] [node 6] ICS10 traffic start
2025-11-19T14:00:23.832652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:23.841249Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436060986715286:2102] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.841344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436060986715224:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.841692Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436061677077796:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.844035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574436063369499219:2096] [node 6] ICP18 handshake (actor [3:7574436063369499244:2118]) is too old
2025-11-19T14:00:23.844235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436063369499163:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.844308Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436063369499245:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.844522Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436061677077787:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.844077Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062448672602:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.844130Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062448672596:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.844133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436062448672596:2107] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.844684Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062448672604:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.844758Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436060835919426:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.844856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436060835919358:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.845313Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436062448672596:2107] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:23.845018Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436061677077784:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.845027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436061677077784:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.847093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436060835919444:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36404
2025-11-19T14:00:23.847206Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436061677077784:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:23.847323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436061677077769:2098] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.847344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:23.847352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436061677077769:2098] [node 6] ICP22 created new session: [2:7574436061677077801:2098]
2025-11-19T14:00:23.847359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436061677077801:2098] [node 6] ICS09 handshake done sender: [2:7574436061677077784:2112] self: [2:7574436061677077774:0] peer: [6:7574436060835919443:0] socket: 324 qp: -1
2025-11-19T14:00:23.847366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436061677077801:2098] [node 6] ICS10 traffic start
2025-11-19T14:00:23.847376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:23.853744Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436060835919358:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3568) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436060835919426:2113] held: no
2025-11-19T14:00:23.853853Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436060835919426:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:23.854009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436060835919358:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.854029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:23.854035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436060835919358:2049] [node 2] ICP22 created new session: [6:7574436060835919445:2049]
2025-11-19T14:00:23.854041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436060835919445:2049] [node 2] ICS09 handshake done sender: [6:7574436060835919426:2113] self: [6:7574436060835919443:0] peer: [2:7574436061677077774:0] socket: 327 qp: -1
2025-11-19T14:00:23.854049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436060835919445:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:23.854056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:23.861698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436062448672594:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.861730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:23.861740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436062448672594:2106] [node 5] ICP22 created new session: [1:7574436062448672623:2106]
2025-11-19T14:00:23.861748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436062448672623:2106] [node 5] ICS09 handshake done sender: [1:7574436062448672596:2107] self: [1:7574436062448672595:0] peer: [5:7574436060986715303:0] socket: 302 qp: -1
2025-11-19T14:00:23.861755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436062448672623:2106] [node 5] ICS10 traffic start
2025-11-19T14:00:23.861765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:23.862401Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436063369499231:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.862457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436063369499162:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.862621Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436061677077773:2102] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.862633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436061677077773:2102] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.861458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436060986715304:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64373
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.864427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436063369499253:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20245
2025-11-19T14:00:23.864566Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436061677077773:2102] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:23.864716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436061677077716:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.864740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:23.864747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436061677077716:2049] [node 3] ICP22 created new session: [2:7574436061677077805:2049]
2025-11-19T14:00:23.864753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436061677077805:2049] [node 3] ICS09 handshake done sender: [2:7574436061677077773:2102] self: [2:7574436061677077772:0] peer: [3:7574436063369499250:0] socket: 323 qp: -1
2025-11-19T14:00:23.864759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436061677077805:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:23.864767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:23.865614Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436060986715286:2102] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.866950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436060986715224:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.866971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:23.866978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436060986715224:2048] [node 1] ICP22 created new session: [5:7574436060986715308:2048]
2025-11-19T14:00:23.866985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436060986715308:2048] [node 1] ICS09 handshake done sender: [5:7574436060986715286:2102] self: [5:7574436060986715303:0] peer: [1:7574436062448672595:0] socket: 308 qp: -1
2025-11-19T14:00:23.866990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436060986715308:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:23.866998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:23.867363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436060986715305:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64373
2025-11-19T14:00:23.867408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436060986715306:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64373
2025-11-19T14:00:23.867439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436060986715307:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64373
2025-11-19T14:00:23.867846Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062448672609:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.867862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436062448672609:2116] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.868730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436060835919448:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36404
2025-11-19T14:00:23.871813Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436062448672609:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:23.872046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436062448672535:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.872081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:23.872087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436062448672535:2049] [node 3] ICP22 created new session: [1:7574436062448672627:2049]
2025-11-19T14:00:23.872094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436062448672627:2049] [node 3] ICS09 handshake done sender: [1:7574436062448672609:2116] self: [1:7574436062448672608:0] peer: [3:7574436063369499251:0] socket: 337 qp: -1
2025-11-19T14:00:23.872101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436062448672627:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:23.872109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.881925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436060517455551:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40701
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.882008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436060517455552:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40701
2025-11-19T14:00:23.882039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436060517455553:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40701
2025-11-19T14:00:23.882196Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436060835919424:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.882271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436060835919395:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.882709Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436060986715305:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.882735Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436060986715307:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.882930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436060986715225:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.883176Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436061677077792:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.883187Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436061677077792:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.884270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436060835919450:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36404
2025-11-19T14:00:23.884483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436063369499163:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3568) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436063369499243:2117] held: no
2025-11-19T14:00:23.884646Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436063369499232:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.884660Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436063369499243:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:23.884779Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436060517455552:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.884816Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436060517455526:2096] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.884818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436060517455526:2096] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.885457Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436060517455553:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.885559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436060517455474:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.885785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436060835919451:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36404
2025-11-19T14:00:23.885882Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436061677077792:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:23.886009Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436061677077790:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.886016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436061677077790:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.886730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436061677077786:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.886747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:23.886754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436061677077786:2104] [node 5] ICP22 created new session: [2:7574436061677077809:2104]
2025-11-19T14:00:23.886762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436061677077809:2104] [node 5] ICS09 handshake done sender: [2:7574436061677077792:2116] self: [2:7574436061677077791:0] peer: [5:7574436060986715312:0] socket: 320 qp: -1
2025-11-19T14:00:23.886770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436061677077809:2104] [node 5] ICS10 traffic start
2025-11-19T14:00:23.886782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:23.887426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436063369499163:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.887444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:23.887448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436063369499163:2049] [node 2] ICP22 created new session: [3:7574436063369499255:2049]
2025-11-19T14:00:23.887453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436063369499255:2049] [node 2] ICS09 handshake done sender: [3:7574436063369499243:2117] self: [3:7574436063369499250:0] peer: [2:7574436061677077772:0] socket: 325 qp: -1
2025-11-19T14:00:23.887458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436063369499255:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:23.887462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.887942Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436060986715306:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.889300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436060986715313:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64373
2025-11-19T14:00:23.894952Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436060517455526:2096] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:23.895305Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436060835919424:2111] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:23.895417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436060835919395:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.895441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:23.895450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436060835919395:2061] [node 4] ICP22 created new session: [6:7574436060835919452:2061]
2025-11-19T14:00:23.895457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436060835919452:2061] [node 4] ICS09 handshake done sender: [6:7574436060835919424:2111] self: [6:7574436060835919449:0] peer: [4:7574436060517455517:0] socket: 322 qp: -1
2025-11-19T14:00:23.895466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436060835919452:2061] [node 4] ICS10 traffic start
2025-11-19T14:00:23.895481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:23.896887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436060517455515:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.896914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:23.896921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436060517455515:2070] [node 6] ICP22 created new session: [4:7574436060517455557:2070]
2025-11-19T14:00:23.896928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436060517455557:2070] [node 6] ICS09 handshake done sender: [4:7574436060517455526:2096] self: [4:7574436060517455517:0] peer: [6:7574436060835919449:0] socket: 315 qp: -1
2025-11-19T14:00:23.896936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436060517455557:2070] [node 6] ICS10 traffic start
2025-11-19T14:00:23.896944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.898399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436061677077812:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61453
2025-11-19T14:00:23.899893Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436061677077790:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:23.900046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436061677077785:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.900060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:23.900065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436061677077785:2103] [node 4] ICP22 created new session: [2:7574436061677077813:2103]
2025-11-19T14:00:23.900071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436061677077813:2103] [node 4] ICS09 handshake done sender: [2:7574436061677077790:2115] self: [2:7574436061677077789:0] peer: [4:7574436060517455556:0] socket: 340 qp: -1
2025-11-19T14:00:23.900077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436061677077813:2103] [node 4] ICS10 traffic start
2025-11-19T14:00:23.900085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:23.900477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436060986715226:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.901187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436060986715225:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3568) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436060986715307:2119] held: no
2025-11-19T14:00:23.901321Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436060986715307:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:23.901473Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436061677077812:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.901501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436061677077715:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.902387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436063369499254:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20245
2025-11-19T14:00:23.902567Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436060835919448:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.902595Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436060835919450:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.902618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436060835919357:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.902629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436060835919396:2076] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.901592Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062448672615:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.901605Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436062448672615:2119] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.904084Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436060517455551:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.904687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436060517455473:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.904709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436060517455559:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40701
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.904832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436061677077816:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61453
2025-11-19T14:00:23.904849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436061677077817:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61453
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.905000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436060517455560:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40701
2025-11-19T14:00:23.905073Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436060517455553:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:23.906272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436060517455474:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.906295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:23.906302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436060517455474:2049] [node 2] ICP22 created new session: [4:7574436060517455563:2049]
2025-11-19T14:00:23.906309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436060517455563:2049] [node 2] ICS09 handshake done sender: [4:7574436060517455553:2117] self: [4:7574436060517455556:0] peer: [2:7574436061677077789:0] socket: 356 qp: -1
2025-11-19T14:00:23.906316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436060517455563:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:23.906323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:23.907627Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436060986715262:2079] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.907639Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436060986715262:2079] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.908360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436060986715225:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.908373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:23.908378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436060986715225:2049] [node 2] ICP22 created new session: [5:7574436060986715315:2049]
2025-11-19T14:00:23.908386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436060986715315:2049] [node 2] ICS09 handshake done sender: [5:7574436060986715307:2119] self: [5:7574436060986715312:0] peer: [2:7574436061677077791:0] socket: 352 qp: -1
2025-11-19T14:00:23.908392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436060986715315:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:23.908400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.908998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436060835919456:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36404
2025-11-19T14:00:23.909292Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436063369499238:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.909304Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436063369499238:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.909995Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436063369499162:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18018) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436063369499245:2119] held: no
2025-11-19T14:00:23.910104Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436063369499245:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:23.910257Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436062448672615:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:23.910454Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436061677077812:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:23.910530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436061677077715:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.910544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:23.910549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436061677077715:2048] [node 1] ICP22 created new session: [2:7574436061677077818:2048]
2025-11-19T14:00:23.910554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436061677077818:2048] [node 1] ICS09 handshake done sender: [2:7574436061677077812:2118] self: [2:7574436061677077815:0] peer: [1:7574436062448672614:0] socket: 344 qp: -1
2025-11-19T14:00:23.910560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436061677077818:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:23.910568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:23.911590Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062448672613:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.911599Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436062448672613:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.912314Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062448672603:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.912342Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436062448672611:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.912345Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436062448672611:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.912899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436062448672534:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.912913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:23.912919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436062448672534:2048] [node 2] ICP22 created new session: [1:7574436062448672631:2048]
2025-11-19T14:00:23.912924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436062448672631:2048] [node 2] ICS09 handshake done sender: [1:7574436062448672615:2119] self: [1:7574436062448672614:0] peer: [2:7574436061677077815:0] socket: 332 qp: -1
2025-11-19T14:00:23.912932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436062448672631:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:23.912940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:23.913748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436063369499162:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.913765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:23.913772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436063369499162:2048] [node 1] ICP22 created new session: [3:7574436063369499260:2048]
2025-11-19T14:00:23.913777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436063369499260:2048] [node 1] ICS09 handshake done sender: [3:7574436063369499245:2119] self: [3:7574436063369499251:0] peer: [1:7574436062448672608:0] socket: 338 qp: -1
2025-11-19T14:00:23.913783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436063369499260:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:23.913790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:23.915911Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436060986715262:2079] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:23.916173Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436060835919450:2121] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:23.916262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436060835919396:2076] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.916279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:23.916299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436060835919396:2076] [node 5] ICP22 created new session: [6:7574436060835919457:2076]
2025-11-19T14:00:23.916305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436060835919457:2076] [node 5] ICS09 handshake done sender: [6:7574436060835919450:2121] self: [6:7574436060835919455:0] peer: [5:7574436060986715260:0] socket: 357 qp: -1
2025-11-19T14:00:23.916312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436060835919457:2076] [node 5] ICS10 traffic start
2025-11-19T14:00:23.916320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:23.917536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436060986715248:2065] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.917557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:23.917563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436060986715248:2065] [node 6] ICP22 created new session: [5:7574436060986715320:2065]
2025-11-19T14:00:23.917570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436060986715320:2065] [node 6] ICS09 handshake done sender: [5:7574436060986715262:2079] self: [5:7574436060986715260:0] peer: [6:7574436060835919455:0] socket: 309 qp: -1
2025-11-19T14:00:23.917575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436060986715320:2065] [node 6] ICS10 traffic start
2025-11-19T14:00:23.917584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.920150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436060517455565:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40701
2025-11-19T14:00:23.920484Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436061677077785:2103] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.922631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436060835919459:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36404
2025-11-19T14:00:23.922776Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436063369499238:2115] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:23.928077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436063369499217:2094] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.928111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:23.928119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436063369499217:2094] [node 5] ICP22 created new session: [3:7574436063369499262:2094]
2025-11-19T14:00:23.928126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436063369499262:2094] [node 5] ICS09 handshake done sender: [3:7574436063369499238:2115] self: [3:7574436063369499237:0] peer: [5:7574436060986715314:0] socket: 331 qp: -1
2025-11-19T14:00:23.928134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436063369499262:2094] [node 5] ICS10 traffic start
2025-11-19T14:00:23.928144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:23.928687Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436062448672613:2118] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:23.928903Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436060835919448:2120] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:23.928979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436060835919357:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.928994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:23.928999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436060835919357:2048] [node 1] ICP22 created new session: [6:7574436060835919460:2048]
2025-11-19T14:00:23.929004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436060835919460:2048] [node 1] ICS09 handshake done sender: [6:7574436060835919448:2120] self: [6:7574436060835919454:0] peer: [1:7574436062448672612:0] socket: 354 qp: -1
2025-11-19T14:00:23.929010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436060835919460:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:23.929018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:23.930076Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436062448672611:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.930540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436060986715319:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64373
2025-11-19T14:00:23.930709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436062448672607:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.930732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:23.930747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436062448672607:2110] [node 6] ICP22 created new session: [1:7574436062448672636:2110]
2025-11-19T14:00:23.930753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436062448672636:2110] [node 6] ICS09 handshake done sender: [1:7574436062448672613:2118] self: [1:7574436062448672612:0] peer: [6:7574436060835919454:0] socket: 333 qp: -1
2025-11-19T14:00:23.930759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436062448672636:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:23.930766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:23.931129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436062448672606:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.931140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:23.931144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436062448672606:2109] [node 4] ICP22 created new session: [1:7574436062448672638:2109]
2025-11-19T14:00:23.931149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436062448672638:2109] [node 4] ICS09 handshake done sender: [1:7574436062448672611:2117] self: [1:7574436062448672610:0] peer: [4:7574436060517455562:0] socket: 339 qp: -1
2025-11-19T14:00:23.931155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436062448672638:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:23.931160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.932151Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436060517455560:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.932188Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436060517455565:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.932247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436060517455567:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40701
2025-11-19T14:00:23.936066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436060517455475:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:23.936169Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436060517455551:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:23.936308Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436063369499240:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.936320Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436063369499240:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.938249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436060517455473:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.938270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:23.938276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436060517455473:2048] [node 1] ICP22 created new session: [4:7574436060517455571:2048]
2025-11-19T14:00:23.938284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436060517455571:2048] [node 1] ICS09 handshake done sender: [4:7574436060517455551:2115] self: [4:7574436060517455562:0] peer: [1:7574436062448672610:0] socket: 341 qp: -1
2025-11-19T14:00:23.938291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436060517455571:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:23.938302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:23.939960Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436063369499240:2116] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:23.940533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436060986715226:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5455) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436060986715306:2118] held: no
2025-11-19T14:00:23.940654Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436060986715306:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:23.940796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436063369499216:2090] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.940820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:23.940828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436063369499216:2090] [node 4] ICP22 created new session: [3:7574436063369499265:2090]
2025-11-19T14:00:23.940835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436063369499265:2090] [node 4] ICS09 handshake done sender: [3:7574436063369499240:2116] self: [3:7574436063369499239:0] peer: [4:7574436060517455568:0] socket: 355 qp: -1
2025-11-19T14:00:23.940842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436063369499265:2090] [node 4] ICS10 traffic start
2025-11-19T14:00:23.940852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.948114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436060986715322:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64373
2025-11-19T14:00:23.948172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436060986715226:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.948198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:23.948205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436060986715226:2050] [node 3] ICP22 created new session: [5:7574436060986715323:2050]
2025-11-19T14:00:23.948212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436060986715323:2050] [node 3] ICS09 handshake done sender: [5:7574436060986715306:2118] self: [5:7574436060986715314:0] peer: [3:7574436063369499237:0] socket: 351 qp: -1
2025-11-19T14:00:23.948219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436060986715323:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:23.948229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:23.949979Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436060986715322:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.950016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436060986715250:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.950256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436060517455570:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40701
2025-11-19T14:00:23.950408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436060517455475:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5455) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436060517455560:2119] held: no
2025-11-19T14:00:23.950522Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436060517455532:2099] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:23.950525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436060517455532:2099] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:23.951204Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436060517455560:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:23.951871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436060517455475:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.951888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:23.951895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436060517455475:2050] [node 3] ICP22 created new session: [4:7574436060517455574:2050]
2025-11-19T14:00:23.951901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436060517455574:2050] [node 3] ICS09 handshake done sender: [4:7574436060517455560:2119] self: [4:7574436060517455568:0] peer: [3:7574436063369499239:0] socket: 364 qp: -1
2025-11-19T14:00:23.951910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436060517455574:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:23.951918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:23.952565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436060986715326:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64373
2025-11-19T14:00:23.954214Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436060517455532:2099] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:23.954431Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436060986715322:2122] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:23.954504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436060986715250:2068] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:23.954519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:23.954525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436060986715250:2068] [node 4] ICP22 created new session: [5:7574436060986715327:2068]
2025-11-19T14:00:23.954530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436060986715327:2068] [node 4] ICS09 handshake done sender: [5:7574436060986715322:2122] self: [5:7574436060986715325:0] peer: [4:7574436060517455531:0] socket: 329 qp: -1
2025-11-19T14:00:23.954536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436060986715327:2068] [node 4] ICS10 traffic start
2025-11-19T14:00:23.954545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:23.955928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436060517455516:2079] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:23.955949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:23.955955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436060517455516:2079] [node 5] ICP22 created new session: [4:7574436060517455576:2079]
2025-11-19T14:00:23.955961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436060517455576:2079] [node 5] ICS09 handshake done sender: [4:7574436060517455532:2099] self: [4:7574436060517455531:0] peer: [5:7574436060986715325:0] socket: 365 qp: -1
2025-11-19T14:00:23.955966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436060517455576:2079] [node 5] ICS10 traffic start
2025-11-19T14:00:23.955974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:23.971840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:23.971953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436061677077818:2048] [node 1] ICS01 socket: 344 reason# EndOfStream
2025-11-19T14:00:23.971964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436061677077715:2048] [node 1] ICP30 unregister session Session# [2:7574436061677077818:2048] VirtualId# [2:7574436061677077815:0]
2025-11-19T14:00:23.971973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436061677077818:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.972016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:23.972039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:23.972518Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:23.972584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436063369499260:2048] [node 1] ICS01 socket: 338 reason# EndOfStream
2025-11-19T14:00:23.972589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436063369499162:2048] [node 1] ICP30 unregister session Session# [3:7574436063369499260:2048] VirtualId# [3:7574436063369499251:0]
2025-11-19T14:00:23.972594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436063369499260:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.972632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:23.972652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:23.975916Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:23.976003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436060517455571:2048] [node 1] ICS01 socket: 341 reason# EndOfStream
2025-11-19T14:00:23.976012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436060517455473:2048] [node 1] ICP30 unregister session Session# [4:7574436060517455571:2048] VirtualId# [4:7574436060517455562:0]
2025-11-19T14:00:23.976019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436060517455571:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.976074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:23.976101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:23.976139Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:23.976200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436060835919460:2048] [node 1] ICS01 socket: 354 reason# EndOfStream
2025-11-19T14:00:23.976205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436060835919357:2048] [node 1] ICP30 unregister session Session# [6:7574436060835919460:2048] VirtualId# [6:7574436060835919454:0]
2025-11-19T14:00:23.976210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436060835919460:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.976239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:23.976259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:23.976352Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:23.976439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436060986715308:2048] [node 1] ICS01 socket: 308 reason# EndOfStream
2025-11-19T14:00:23.976447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436060986715224:2048] [node 1] ICP30 unregister session Session# [5:7574436060986715308:2048] VirtualId# [5:7574436060986715303:0]
2025-11-19T14:00:23.976454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436060986715308:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.976493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:23.976517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:23.984382Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:23.984494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436063369499255:2049] [node 2] ICS01 socket: 325 reason# EndOfStream
2025-11-19T14:00:23.984505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436063369499163:2049] [node 2] ICP30 unregister session Session# [3:7574436063369499255:2049] VirtualId# [3:7574436063369499250:0]
2025-11-19T14:00:23.984512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436063369499255:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.984562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:23.984589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:23.995157Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:23.995276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436060986715315:2049] [node 2] ICS01 socket: 352 reason# EndOfStream
2025-11-19T14:00:23.995287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436060986715225:2049] [node 2] ICP30 unregister session Session# [5:7574436060986715315:2049] VirtualId# [5:7574436060986715312:0]
2025-11-19T14:00:23.995296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436060986715315:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.995343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:23.995366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:23.997950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:23.998048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436060517455563:2049] [node 2] ICS01 socket: 356 reason# EndOfStream
2025-11-19T14:00:23.998057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436060517455474:2049] [node 2] ICP30 unregister session Session# [4:7574436060517455563:2049] VirtualId# [4:7574436060517455556:0]
2025-11-19T14:00:23.998063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436060517455563:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.998116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:23.998142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:23.998246Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:23.998309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436060835919445:2049] [node 2] ICS01 socket: 327 reason# EndOfStream
2025-11-19T14:00:23.998315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436060835919358:2049] [node 2] ICP30 unregister session Session# [6:7574436060835919445:2049] VirtualId# [6:7574436060835919443:0]
2025-11-19T14:00:23.998319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436060835919445:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:23.998352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:23.998375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:24.046487Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:24.046669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436060986715323:2050] [node 3] ICS01 socket: 351 reason# EndOfStream
2025-11-19T14:00:24.046680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436060986715226:2050] [node 3] ICP30 unregister session Session# [5:7574436060986715323:2050] VirtualId# [5:7574436060986715314:0]
2025-11-19T14:00:24.046690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436060986715323:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:24.046756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:24.046786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:24.046837Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:24.046914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436060835919440:2050] [node 3] ICS01 socket: 326 reason# EndOfStream
2025-11-19T14:00:24.046920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436060835919359:2050] [node 3] ICP30 unregister session Session# [6:7574436060835919440:2050] VirtualId# [6:7574436060835919438:0]
2025-11-19T14:00:24.046925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436060835919440:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:24.046964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:24.046988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:24.071642Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:24.071784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436060835919452:2061] [node 4] ICS01 socket: 322 reason# EndOfStream
2025-11-19T14:00:24.071798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436060835919395:2061] [node 4] ICP30 unregister session Session# [6:7574436060835919452:2061] VirtualId# [6:7574436060835919449:0]
2025-11-19T14:00:24.071807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436060835919452:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:24.071879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:24.071909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:24.680078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436067522206455:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:24.680095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436067522206456:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:24.744108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436065372999872:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:24.744130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436065372999873:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:24.840291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436067662166537:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:24.840310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436067662166538:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:24.900120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436064293076661:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:24.900146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436064293076662:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:24.900152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436064293076663:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:24.911860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436066778810776:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:24.911889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436066778810777:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:24.915651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436066778810775:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:24.958351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436067156975310:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:24.958368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436067156975311:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:24.958374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436067156975312:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:25.017831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436071817173813:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:25.017955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436071817173814:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.020169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436071073778132:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21078
2025-11-19T14:00:25.020076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436068588044018:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61805
2025-11-19T14:00:25.019658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436071817173822:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:25.024097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436071451942658:2097] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.026325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436071957133893:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40972
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.028033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436071817173830:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53766
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.029200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436071073778133:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21078
2025-11-19T14:00:25.029331Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436068588044018:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.029754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436064293076661:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.030314Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436071073778132:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.030813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436066778810775:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.031125Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436071957133893:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.031652Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436071073778133:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.031852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436071073778136:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:25.032128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436071073778136:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.032103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436071451942670:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:25.032522Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436071451942663:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.032527Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574436071451942663:2102] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.031513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436067662166537:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.035949Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071817173816:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.035968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436071817173816:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.036212Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071817173818:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.036218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436071817173818:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.036651Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071817173824:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.036654Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436071817173824:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.036138Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436071451942663:2102] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:25.039901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574436071451942658:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.039926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:25.039938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436071451942658:2097] [node 5] ICP22 created new session: [6:7574436071451942675:2097]
2025-11-19T14:00:25.039945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436071451942675:2097] [node 5] ICS09 handshake done sender: [6:7574436071451942663:2102] self: [6:7574436071451942662:0] peer: [5:7574436071073778137:0] socket: 317 qp: -1
2025-11-19T14:00:25.039952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436071451942675:2097] [node 5] ICS10 traffic start
2025-11-19T14:00:25.039978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:25.037627Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436071817173818:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.041699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436068588044020:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61805
2025-11-19T14:00:25.043376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436069667967235:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:25.043513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436069667967236:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:25.041540Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436071817173824:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.044956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436071957133900:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:25.045073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436071957133901:2110] [node 5] ICP01 ready to work
2025-11-19T14:00:25.045966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436071957133898:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40972
2025-11-19T14:00:25.045594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436068588044023:2110] [node 5] ICP01 ready to work
2025-11-19T14:00:25.045711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436068588044024:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:25.046376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436071817173814:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.046397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:25.046405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436071817173814:2108] [node 5] ICP22 created new session: [1:7574436071817173833:2108]
2025-11-19T14:00:25.046411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436071817173833:2108] [node 5] ICS09 handshake done sender: [1:7574436071817173818:2110] self: [1:7574436071817173817:0] peer: [5:7574436071073778134:0] socket: 313 qp: -1
2025-11-19T14:00:25.046419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436071817173833:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:25.046428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:25.046708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436069667967233:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:25.047412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436071073778140:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.052978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436071073778139:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21078
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.062064Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436068588044020:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.063642Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436071073778133:2109] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:25.061037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436071451942680:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2888
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.063784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436071073778136:2110] [node 6] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.063806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:25.063812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436071073778136:2110] [node 6] ICP22 created new session: [5:7574436071073778148:2110]
2025-11-19T14:00:25.063820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436071073778148:2110] [node 6] ICS09 handshake done sender: [5:7574436071073778133:2109] self: [5:7574436071073778137:0] peer: [6:7574436071451942662:0] socket: 321 qp: -1
2025-11-19T14:00:25.063829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436071073778148:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:25.063855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:25.064125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436071073778142:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21078
2025-11-19T14:00:25.063916Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436069667967233:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17071098ns) {Configure:104(17070541ns) {ProcessConfigured:135(16834417ns) {StartInitialHandshake:157(16834064ns) {DropHandshakes:469(392ns) {DropIncomingHandshake:441(29ns) DropOutgoingHandshake:456(38ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:476(1178ns) {SwitchToState:178(377ns)}}}}}
2025-11-19T14:00:25.064885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574436071451942658:2097] [node 5] ICP35 event processing took too much time PendingConnection:278(24907721ns) {HandleHandshakeStatus:329(24906541ns) {DropIncomingHandshake:441(215ns) DropOutgoingHandshake:456(752ns) SwitchToState:178(479ns) ProcessPendingSessionEvents:495(16930207ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 16916383ns max# 16845129ns}}}
2025-11-19T14:00:25.064942Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436071451942680:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.069675Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436064293076663:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(23111092ns) {Configure:104(23110131ns) {ProcessConfigured:135(21659458ns) {StartInitialHandshake:157(21658485ns) {DropHandshakes:469(382ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(149ns) PrepareNewSessionHandshake:476(1119ns) {SwitchToState:178(22ns)}}}}}
2025-11-19T14:00:25.069485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436067156975311:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.072534Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436069667967238:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.072550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436069667967238:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.073309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436066778810776:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(25760720ns) {Configure:104(25760090ns) {ProcessConfigured:135(25228257ns) {StartInitialHandshake:157(25227645ns) {DropHandshakes:469(152ns) {DropIncomingHandshake:441(21ns) DropOutgoingHandshake:456(15ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(1777ns) {SwitchToState:178(349ns)}}}}}
2025-11-19T14:00:25.073432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436071073778147:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21078
2025-11-19T14:00:25.073681Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436071073778132:2108] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:25.073870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436067522206456:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.073897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:25.074023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436071957133908:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:25.073907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436067522206456:2049] [node 3] ICP22 created new session: [1:7574436071817173835:2049]
2025-11-19T14:00:25.073915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436071817173835:2049] [node 3] ICS09 handshake done sender: [1:7574436071817173824:2112] self: [1:7574436071817173823:0] peer: [3:7574436071957133894:0] socket: 318 qp: -1
2025-11-19T14:00:25.073926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436071817173835:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:25.073935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:25.074120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436069667967247:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28261
2025-11-19T14:00:25.074333Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436069667967238:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:25.074408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436069667967235:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.074419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:25.074425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436069667967235:2109] [node 6] ICP22 created new session: [2:7574436069667967249:2109]
2025-11-19T14:00:25.074430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436069667967249:2109] [node 6] ICS09 handshake done sender: [2:7574436069667967238:2111] self: [2:7574436069667967237:0] peer: [6:7574436071451942689:0] socket: 329 qp: -1
2025-11-19T14:00:25.074434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436069667967249:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:25.074450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.080690Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436071957133893:2107] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:25.080924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436071817173814:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(34463634ns) {HandleHandshakeStatus:329(34462886ns) {DropIncomingHandshake:441(34ns) DropOutgoingHandshake:456(187ns) SwitchToState:178(220ns) ProcessPendingSessionEvents:495(27060945ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 27045819ns max# 26979617ns}}}
2025-11-19T14:00:25.081446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436071817173838:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53766
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.081991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436071957133899:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40972
2025-11-19T14:00:25.085877Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436071817173816:2109] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:25.085882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436066778810775:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.085909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:25.085917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436066778810775:2048] [node 1] ICP22 created new session: [5:7574436071073778153:2048]
2025-11-19T14:00:25.085924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436071073778153:2048] [node 1] ICS09 handshake done sender: [5:7574436071073778132:2108] self: [5:7574436071073778134:0] peer: [1:7574436071817173817:0] socket: 315 qp: -1
2025-11-19T14:00:25.085931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436071073778153:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:25.085939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.086233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436071451942690:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2888
2025-11-19T14:00:25.086254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436071451942691:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2888
2025-11-19T14:00:25.086846Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436069667967247:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.087164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436071957133909:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40972
2025-11-19T14:00:25.087321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436067662166537:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.087329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:25.087334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436067662166537:2048] [node 1] ICP22 created new session: [3:7574436071957133915:2048]
2025-11-19T14:00:25.087342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436071957133915:2048] [node 1] ICS09 handshake done sender: [3:7574436071957133893:2107] self: [3:7574436071957133894:0] peer: [1:7574436071817173823:0] socket: 319 qp: -1
2025-11-19T14:00:25.087346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436071957133915:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:25.087379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:25.086094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436071817173813:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.086106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:25.086111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436071817173813:2107] [node 4] ICP22 created new session: [1:7574436071817173841:2107]
2025-11-19T14:00:25.086115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436071817173841:2107] [node 4] ICS09 handshake done sender: [1:7574436071817173816:2109] self: [1:7574436071817173815:0] peer: [4:7574436068588044019:0] socket: 312 qp: -1
2025-11-19T14:00:25.086120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436071817173841:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:25.086125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:25.092035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436071451942692:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2888
2025-11-19T14:00:25.094279Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436069667967235:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(19770379ns) {HandleHandshakeStatus:329(19769104ns) {DropIncomingHandshake:441(29ns) DropOutgoingHandshake:456(131ns) SwitchToState:178(46ns) ProcessPendingSessionEvents:495(12034154ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 12018815ns max# 11951542ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.094876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436067522206456:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(20920152ns) {HandleHandshakeStatus:329(20919392ns) {DropIncomingHandshake:441(327ns) DropOutgoingHandshake:456(500ns) SwitchToState:178(134ns) ProcessPendingSessionEvents:495(12571210ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12555125ns max# 12482673ns}}}
2025-11-19T14:00:25.094975Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071817173838:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.094992Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071817173830:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.096205Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436071957133899:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.096262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436071957133917:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40972
2025-11-19T14:00:25.096277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436067662166538:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.097899Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436069667967240:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.097910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436069667967240:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.103631Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436069667967240:2112] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:25.099996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436068588044037:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61805
2025-11-19T14:00:25.105689Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436071451942690:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.105710Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436071451942680:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.105921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436065372999873:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.105944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:25.105951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436065372999873:2049] [node 3] ICP22 created new session: [2:7574436069667967257:2049]
2025-11-19T14:00:25.105957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436069667967257:2049] [node 3] ICS09 handshake done sender: [2:7574436069667967240:2112] self: [2:7574436069667967239:0] peer: [3:7574436071957133918:0] socket: 330 qp: -1
2025-11-19T14:00:25.105964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436069667967257:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:25.105974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:25.106344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436069667967254:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28261
2025-11-19T14:00:25.106355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436069667967255:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28261
2025-11-19T14:00:25.106367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436069667967256:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28261
2025-11-19T14:00:25.106692Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436071817173813:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(20505734ns) {HandleHandshakeStatus:329(20505052ns) {DropIncomingHandshake:441(44ns) DropOutgoingHandshake:456(277ns) SwitchToState:178(87ns) ProcessPendingSessionEvents:495(19025197ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19008435ns max# 18932260ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.113612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436068588044038:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61805
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.114033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436066778810775:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(28073185ns) {HandleHandshakeStatus:329(28071905ns) {DropIncomingHandshake:441(491ns) DropOutgoingHandshake:456(381ns) SwitchToState:178(230ns) ProcessPendingSessionEvents:495(27967763ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 27948866ns max# 27893584ns}}}
2025-11-19T14:00:25.116178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436067156975311:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.116210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:25.116221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436067156975311:2049] [node 2] ICP22 created new session: [6:7574436071451942693:2049]
2025-11-19T14:00:25.116229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436071451942693:2049] [node 2] ICS09 handshake done sender: [6:7574436071451942680:2112] self: [6:7574436071451942689:0] peer: [2:7574436069667967237:0] socket: 333 qp: -1
2025-11-19T14:00:25.116239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436071451942693:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:25.116250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:25.116890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436071451942670:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.117077Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436069667967254:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.117107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436065372999872:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.120491Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071817173828:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.120505Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436071817173828:2114] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.128312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436068588044039:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61805
2025-11-19T14:00:25.128417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436071073778156:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21078
2025-11-19T14:00:25.132181Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436071817173828:2114] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:25.132001Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436068588044029:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.132019Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436068588044029:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.132811Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436068588044018:2108] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:25.132648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436067522206455:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.132674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:25.132682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436067522206455:2048] [node 2] ICP22 created new session: [1:7574436071817173846:2048]
2025-11-19T14:00:25.132689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436071817173846:2048] [node 2] ICS09 handshake done sender: [1:7574436071817173828:2114] self: [1:7574436071817173827:0] peer: [2:7574436069667967260:0] socket: 347 qp: -1
2025-11-19T14:00:25.132696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436071817173846:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:25.132705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.133305Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436071451942692:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.133408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436067156975310:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.133430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436071451942696:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2888
2025-11-19T14:00:25.133747Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436071451942690:2116] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:25.133815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436071451942670:2107] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.133828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:25.133833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436071451942670:2107] [node 4] ICP22 created new session: [6:7574436071451942698:2107]
2025-11-19T14:00:25.133839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436071451942698:2107] [node 4] ICS09 handshake done sender: [6:7574436071451942690:2116] self: [6:7574436071451942695:0] peer: [4:7574436068588044027:0] socket: 341 qp: -1
2025-11-19T14:00:25.133846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436071451942698:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:25.133853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:25.133652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436067662166537:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(46226544ns) {HandleHandshakeStatus:329(46226061ns) {DropIncomingHandshake:441(142ns) DropOutgoingHandshake:456(54ns) SwitchToState:178(44ns) ProcessPendingSessionEvents:495(5149477ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5133651ns max# 5064302ns}}}
2025-11-19T14:00:25.133697Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436071957133909:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.132982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436068588044040:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61805
2025-11-19T14:00:25.133050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436064293076661:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.133060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:25.133064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436064293076661:2048] [node 1] ICP22 created new session: [4:7574436068588044042:2048]
2025-11-19T14:00:25.133070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436068588044042:2048] [node 1] ICS09 handshake done sender: [4:7574436068588044018:2108] self: [4:7574436068588044019:0] peer: [1:7574436071817173815:0] socket: 314 qp: -1
2025-11-19T14:00:25.133075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436068588044042:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:25.133082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:25.133417Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436068588044029:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:25.135082Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436068588044040:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.135093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436068588044024:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.135152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:25.135159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436068588044024:2111] [node 6] ICP22 created new session: [4:7574436068588044044:2111]
2025-11-19T14:00:25.135166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436068588044044:2111] [node 6] ICS09 handshake done sender: [4:7574436068588044029:2113] self: [4:7574436068588044027:0] peer: [6:7574436071451942695:0] socket: 337 qp: -1
2025-11-19T14:00:25.135171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436068588044044:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:25.135178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:25.136387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436064293076662:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.136714Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436069667967242:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.136727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436069667967242:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.137821Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436069667967242:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:25.137910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436069667967236:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.137933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:25.137939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436069667967236:2110] [node 4] ICP22 created new session: [2:7574436069667967262:2110]
2025-11-19T14:00:25.137945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436069667967262:2110] [node 4] ICS09 handshake done sender: [2:7574436069667967242:2113] self: [2:7574436069667967241:0] peer: [4:7574436068588044047:0] socket: 359 qp: -1
2025-11-19T14:00:25.137951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436069667967262:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:25.137958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:25.138414Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071817173826:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.138425Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436071817173826:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.139887Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436071817173826:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:25.139976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436071817173822:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.139995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:25.140001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436071817173822:2111] [node 6] ICP22 created new session: [1:7574436071817173849:2111]
2025-11-19T14:00:25.140006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436071817173849:2111] [node 6] ICS09 handshake done sender: [1:7574436071817173826:2113] self: [1:7574436071817173825:0] peer: [6:7574436071451942697:0] socket: 346 qp: -1
2025-11-19T14:00:25.140011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436071817173849:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:25.140019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:25.140432Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436065372999872:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1746) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436069667967254:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.144415Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436068588044039:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.144457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436068588044048:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61805
2025-11-19T14:00:25.144619Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436064293076662:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25966) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436068588044040:2119] held: no
2025-11-19T14:00:25.144813Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436067156975310:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1746) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436071451942692:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.145162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436071957133919:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40972
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.145299Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436071073778147:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.145448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436071073778157:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21078
2025-11-19T14:00:25.145481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436071073778140:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.145694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436071957133920:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40972
2025-11-19T14:00:25.147156Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436071957133917:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.147175Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436071957133899:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:25.147273Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436071073778156:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.147303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436066778810777:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.151913Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436071957133905:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.151923Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436071957133905:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.152629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436067662166538:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.152651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:25.152663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436067662166538:2049] [node 2] ICP22 created new session: [3:7574436071957133922:2049]
2025-11-19T14:00:25.152672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436071957133922:2049] [node 2] ICS09 handshake done sender: [3:7574436071957133899:2112] self: [3:7574436071957133918:0] peer: [2:7574436069667967239:0] socket: 331 qp: -1
2025-11-19T14:00:25.152681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436071957133922:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:25.152693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.160268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436071073778160:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21078
2025-11-19T14:00:25.160388Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436071957133905:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:25.160684Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436071073778156:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:25.160751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436066778810777:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.160775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:25.160782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436066778810777:2050] [node 3] ICP22 created new session: [5:7574436071073778161:2050]
2025-11-19T14:00:25.160789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436071073778161:2050] [node 3] ICS09 handshake done sender: [5:7574436071073778156:2118] self: [5:7574436071073778159:0] peer: [3:7574436071957133904:0] socket: 358 qp: -1
2025-11-19T14:00:25.160796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436071073778161:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:25.160806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:25.162095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436071957133901:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.162115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:25.162122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436071957133901:2110] [node 5] ICP22 created new session: [3:7574436071957133927:2110]
2025-11-19T14:00:25.162128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436071957133927:2110] [node 5] ICS09 handshake done sender: [3:7574436071957133905:2114] self: [3:7574436071957133904:0] peer: [5:7574436071073778159:0] socket: 356 qp: -1
2025-11-19T14:00:25.162133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436071957133927:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:25.162141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.198320Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436068588044040:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.198886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436065372999873:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:25.199038Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436069667967255:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.199105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436069667967264:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28261
2025-11-19T14:00:25.199210Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436071073778157:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.199257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436066778810776:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.198501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436071451942700:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2888
2025-11-19T14:00:25.199571Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436069667967246:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.199580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436069667967246:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.199906Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436069667967254:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:25.200371Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436068588044038:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.200410Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436068588044026:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.200414Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436068588044026:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.200463Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436069667967246:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.202049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436071451942701:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2888
2025-11-19T14:00:25.202248Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436071451942692:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:25.202089Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436071957133920:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.202134Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436071957133903:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.202137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436071957133903:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.201070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436064293076663:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.201097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436064293076662:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.201112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:25.201118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436064293076662:2049] [node 2] ICP22 created new session: [4:7574436068588044051:2049]
2025-11-19T14:00:25.201124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436068588044051:2049] [node 2] ICS09 handshake done sender: [4:7574436068588044040:2119] self: [4:7574436068588044047:0] peer: [2:7574436069667967241:0] socket: 364 qp: -1
2025-11-19T14:00:25.201131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436068588044051:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:25.201139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.203188Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436068588044026:2112] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:25.203232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436068588044054:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61805
2025-11-19T14:00:25.203263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436068588044023:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.203287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:25.203293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436068588044023:2110] [node 5] ICP22 created new session: [4:7574436068588044055:2110]
2025-11-19T14:00:25.203299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436068588044055:2110] [node 5] ICS09 handshake done sender: [4:7574436068588044026:2112] self: [4:7574436068588044025:0] peer: [5:7574436071073778158:0] socket: 336 qp: -1
2025-11-19T14:00:25.203305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436068588044055:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:25.203313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:25.203738Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436071957133903:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:25.203898Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436068588044039:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.204533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436071073778140:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28145) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436071073778147:2116] held: no
2025-11-19T14:00:25.204708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436071073778164:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21078
2025-11-19T14:00:25.204831Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436071073778157:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:25.204877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436066778810776:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.204889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:25.204893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436066778810776:2049] [node 2] ICP22 created new session: [5:7574436071073778165:2049]
2025-11-19T14:00:25.204899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436071073778165:2049] [node 2] ICS09 handshake done sender: [5:7574436071073778157:2119] self: [5:7574436071073778163:0] peer: [2:7574436069667967245:0] socket: 369 qp: -1
2025-11-19T14:00:25.204904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436071073778165:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:25.204909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:25.207835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436065372999872:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.207856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:25.207863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436065372999872:2048] [node 1] ICP22 created new session: [2:7574436069667967267:2048]
2025-11-19T14:00:25.207870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436069667967267:2048] [node 1] ICS09 handshake done sender: [2:7574436069667967254:2116] self: [2:7574436069667967260:0] peer: [1:7574436071817173827:0] socket: 354 qp: -1
2025-11-19T14:00:25.207878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436069667967267:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:25.207884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:25.208108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436069667967233:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.208118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:25.208122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436069667967233:2108] [node 5] ICP22 created new session: [2:7574436069667967269:2108]
2025-11-19T14:00:25.208127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436069667967269:2108] [node 5] ICS09 handshake done sender: [2:7574436069667967246:2114] self: [2:7574436069667967245:0] peer: [5:7574436071073778163:0] socket: 367 qp: -1
2025-11-19T14:00:25.208130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436069667967269:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:25.208137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:25.208914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436071957133900:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.208934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:25.208939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436071957133900:2109] [node 4] ICP22 created new session: [3:7574436071957133931:2109]
2025-11-19T14:00:25.208943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436071957133931:2109] [node 4] ICS09 handshake done sender: [3:7574436071957133903:2113] self: [3:7574436071957133902:0] peer: [4:7574436068588044049:0] socket: 355 qp: -1
2025-11-19T14:00:25.208948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436071957133931:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:25.208955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:25.211808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436067156975310:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.211826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:25.211832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436067156975310:2048] [node 1] ICP22 created new session: [6:7574436071451942702:2048]
2025-11-19T14:00:25.211838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436071451942702:2048] [node 1] ICS09 handshake done sender: [6:7574436071451942692:2118] self: [6:7574436071451942697:0] peer: [1:7574436071817173825:0] socket: 352 qp: -1
2025-11-19T14:00:25.211844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436071451942702:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:25.211851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:25.212094Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436071451942701:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.212181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436067156975312:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.215255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436064293076663:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.215276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:25.215283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436064293076663:2050] [node 3] ICP22 created new session: [4:7574436068588044057:2050]
2025-11-19T14:00:25.215289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436068588044057:2050] [node 3] ICS09 handshake done sender: [4:7574436068588044039:2118] self: [4:7574436068588044049:0] peer: [3:7574436071957133902:0] socket: 363 qp: -1
2025-11-19T14:00:25.215296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436068588044057:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:25.215305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:25.216704Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574436064293076663:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:25.217119Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436071957133911:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.217133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436071957133911:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.218098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436071073778167:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21078
2025-11-19T14:00:25.218271Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436071957133911:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:25.218355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436071957133908:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.218371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:25.218376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436071957133908:2108] [node 6] ICP22 created new session: [3:7574436071957133934:2108]
2025-11-19T14:00:25.218381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436071957133934:2108] [node 6] ICS09 handshake done sender: [3:7574436071957133911:2117] self: [3:7574436071957133910:0] peer: [6:7574436071451942704:0] socket: 335 qp: -1
2025-11-19T14:00:25.218385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436071957133934:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:25.218391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.220789Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436067156975312:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3232) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436071451942701:2121] held: no
2025-11-19T14:00:25.220968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436071451942705:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2888
2025-11-19T14:00:25.221100Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436071073778147:2116] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:25.221244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436071073778140:2111] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.221258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:25.221263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436071073778140:2111] [node 4] ICP22 created new session: [5:7574436071073778168:2111]
2025-11-19T14:00:25.221269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436071073778168:2111] [node 4] ICS09 handshake done sender: [5:7574436071073778147:2116] self: [5:7574436071073778158:0] peer: [4:7574436068588044025:0] socket: 339 qp: -1
2025-11-19T14:00:25.221275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436071073778168:2111] [node 4] ICS10 traffic start
2025-11-19T14:00:25.221282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:25.221687Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436071451942701:2121] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:25.221745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436067156975312:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.221757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:25.221761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436067156975312:2050] [node 3] ICP22 created new session: [6:7574436071451942706:2050]
2025-11-19T14:00:25.221766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436071451942706:2050] [node 3] ICS09 handshake done sender: [6:7574436071451942701:2121] self: [6:7574436071451942704:0] peer: [3:7574436071957133910:0] socket: 360 qp: -1
2025-11-19T14:00:25.221771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436071451942706:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:25.221777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:25.230585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:25.230680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436069667967267:2048] [node 1] ICS01 socket: 354 reason# EndOfStream
2025-11-19T14:00:25.230691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436065372999872:2048] [node 1] ICP30 unregister session Session# [2:7574436069667967267:2048] VirtualId# [2:7574436069667967260:0]
2025-11-19T14:00:25.230699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436069667967267:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.230739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:25.230761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:25.231201Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:25.231392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574436071957133915:2048] [node 1] ICS07 socket disconnect 319 reason# EndOfStream
2025-11-19T14:00:25.231401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436071957133915:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.231440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:25.231457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436071957133915:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:25.234064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436067662166537:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1746) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T14:00:25.234077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436071957133915:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:25.234369Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436067662166537:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1746) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:25.234381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436071957133915:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:25.234386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436067662166537:2048] [node 1] ICP30 unregister session Session# [3:7574436071957133915:2048] VirtualId# [3:7574436071957133894:0]
2025-11-19T14:00:25.234437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:25.234444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436067662166537:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1746) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:25.234447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1746) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:25.244023Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:25.244128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574436071073778153:2048] [node 1] ICS07 socket disconnect 315 reason# EndOfStream
2025-11-19T14:00:25.244136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436071073778153:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.244196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:25.244220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574436071073778153:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:25.244441Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:25.244505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436071451942702:2048] [node 1] ICS01 socket: 352 reason# EndOfStream
2025-11-19T14:00:25.244511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436067156975310:2048] [node 1] ICP30 unregister session Session# [6:7574436071451942702:2048] VirtualId# [6:7574436071451942697:0]
2025-11-19T14:00:25.244517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436071451942702:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.244547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:25.244572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:25.244734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:25.244794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436068588044042:2048] [node 1] ICS01 socket: 314 reason# EndOfStream
2025-11-19T14:00:25.244799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436064293076661:2048] [node 1] ICP30 unregister session Session# [4:7574436068588044042:2048] VirtualId# [4:7574436068588044019:0]
2025-11-19T14:00:25.244803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436068588044042:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.244831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:25.244852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:25.245319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574436067662166537:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:25.245500Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436066778810775:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1746) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:25.245515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436071073778153:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:25.245518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436066778810775:2048] [node 1] ICP30 unregister session Session# [5:7574436071073778153:2048] VirtualId# [5:7574436071073778134:0]
2025-11-19T14:00:25.245567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:25.245574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574436066778810775:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1746) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:25.245577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1746) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:25.246306Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:25.246370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436071957133922:2049] [node 2] ICS01 socket: 331 reason# EndOfStream
2025-11-19T14:00:25.246376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436067662166538:2049] [node 2] ICP30 unregister session Session# [3:7574436071957133922:2049] VirtualId# [3:7574436071957133918:0]
2025-11-19T14:00:25.246380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436071957133922:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.246414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:25.246432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:25.255911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574436066778810775:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:25.258226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:25.258290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436071451942693:2049] [node 2] ICS01 socket: 333 reason# EndOfStream
2025-11-19T14:00:25.258295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436067156975311:2049] [node 2] ICP30 unregister session Session# [6:7574436071451942693:2049] VirtualId# [6:7574436071451942689:0]
2025-11-19T14:00:25.258299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436071451942693:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.258325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:25.258346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:25.258377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:25.258439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436068588044051:2049] [node 2] ICS01 socket: 364 reason# EndOfStream
2025-11-19T14:00:25.258444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436064293076662:2049] [node 2] ICP30 unregister session Session# [4:7574436068588044051:2049] VirtualId# [4:7574436068588044047:0]
2025-11-19T14:00:25.258448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436068588044051:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.258473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:25.258494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:25.257928Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:25.258044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436071073778165:2049] [node 2] ICS01 socket: 369 reason# EndOfStream
2025-11-19T14:00:25.258053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436066778810776:2049] [node 2] ICP30 unregister session Session# [5:7574436071073778165:2049] VirtualId# [5:7574436071073778163:0]
2025-11-19T14:00:25.258061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436071073778165:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.258109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:25.258135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:25.278936Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:25.279046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436071451942706:2050] [node 3] ICS01 socket: 360 reason# EndOfStream
2025-11-19T14:00:25.279057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436067156975312:2050] [node 3] ICP30 unregister session Session# [6:7574436071451942706:2050] VirtualId# [6:7574436071451942704:0]
2025-11-19T14:00:25.279065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436071451942706:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.279107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:25.279130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:25.279172Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T14:00:25.279252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436068588044057:2050] [node 3] ICS01 socket: 363 reason# EndOfStream
2025-11-19T14:00:25.279257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436064293076663:2050] [node 3] ICP30 unregister session Session# [4:7574436068588044057:2050] VirtualId# [4:7574436068588044049:0]
2025-11-19T14:00:25.279261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436068588044057:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.279285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:25.279304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:25.279440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:25.279535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436071073778161:2050] [node 3] ICS01 socket: 358 reason# EndOfStream
2025-11-19T14:00:25.279539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436066778810777:2050] [node 3] ICP30 unregister session Session# [5:7574436071073778161:2050] VirtualId# [5:7574436071073778159:0]
2025-11-19T14:00:25.279542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436071073778161:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.279571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:25.279602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:25.318032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:25.318151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436071451942698:2107] [node 4] ICS01 socket: 341 reason# EndOfStream
2025-11-19T14:00:25.318162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436071451942670:2107] [node 4] ICP30 unregister session Session# [6:7574436071451942698:2107] VirtualId# [6:7574436071451942695:0]
2025-11-19T14:00:25.318171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436071451942698:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.318228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:25.318256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:25.326085Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T14:00:25.326198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436071451942675:2097] [node 5] ICS01 socket: 317 reason# EndOfStream
2025-11-19T14:00:25.326210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436071451942658:2097] [node 5] ICP30 unregister session Session# [6:7574436071451942675:2097] VirtualId# [6:7574436071451942662:0]
2025-11-19T14:00:25.326219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436071451942675:2097] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:25.326261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T14:00:25.326285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:25.584115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436071234763410:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:25.584136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436071234763411:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:25.612444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436070916529934:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:25.612468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436070916529935:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:25.676035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436070049120243:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:25.676056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436070049120244:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:25.735515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436067943112361:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:25.735534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436067943112362:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:25.735540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436067943112363:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:25.780256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436069434734977:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:25.780274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436069434734978:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:25.780279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436069434734979:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:25.804133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436070054529279:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:25.804151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436070054529280:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:25.804156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436070054529281:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:25.839275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436071234763471:2107] [node 6] ICP01 ready to work
2025-11-19T14:00:25.840072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436070049120270:2071] [node 5] ICP01 ready to work
2025-11-19T14:00:25.845016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436069434734999:2066] [node 6] ICP01 ready to work
2025-11-19T14:00:25.845029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436067943112392:2070] [node 6] ICP01 ready to work
2025-11-19T14:00:25.848153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436070054529309:2061] [node 4] ICP01 ready to work
2025-11-19T14:00:25.851429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436070054529329:2087] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.852022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436070916530000:2106] [node 5] ICP01 ready to work
2025-11-19T14:00:25.852137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436070916530001:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:25.852928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436070916530002:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6988
2025-11-19T14:00:25.853619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436070049120299:2077] [node 6] ICP01 ready to work
2025-11-19T14:00:25.853735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436070049120301:2087] [node 4] ICP01 ready to work
2025-11-19T14:00:25.854923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436070916529991:2103] [node 6] ICP01 ready to work
2025-11-19T14:00:25.855443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436071234763475:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:25.855882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436071234763476:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:25.860901Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436070916529935:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(16313211ns) {Configure:104(16312290ns) {ProcessConfigured:135(15944938ns) {StartInitialHandshake:157(15944007ns) {DropHandshakes:469(757ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(241ns) PrepareNewSessionHandshake:476(1013ns) {SwitchToState:178(728ns)}}}}}
2025-11-19T14:00:25.866907Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436069434734978:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(21834871ns) {Configure:104(21834050ns) {ProcessConfigured:135(21424246ns) {StartInitialHandshake:157(21423979ns) {DropHandshakes:469(569ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:476(1099ns) {SwitchToState:178(907ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.865719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436071234763410:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(21074030ns) {Configure:104(21073229ns) {ProcessConfigured:135(20266874ns) {StartInitialHandshake:157(20266114ns) {DropHandshakes:469(467ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(252ns) PrepareNewSessionHandshake:476(1872ns) {SwitchToState:178(1725ns)}}}}}
2025-11-19T14:00:25.867669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436069434735040:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21062
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.876554Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436070916529991:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21498475ns) {Configure:104(21496649ns) {ProcessConfigured:135(21204817ns) {StartInitialHandshake:157(21203969ns) {DropHandshakes:469(427ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(294ns) PrepareNewSessionHandshake:476(2472ns) {SwitchToState:178(1077ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.876745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436070054529353:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48247
2025-11-19T14:00:25.876779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436070054529354:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48247
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.878798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436070049120243:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23728928ns) {Configure:104(23728023ns) {ProcessConfigured:135(23412938ns) {StartInitialHandshake:157(23412140ns) {DropHandshakes:469(578ns) {DropIncomingHandshake:441(96ns) DropOutgoingHandshake:456(98ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:476(1980ns) {SwitchToState:178(1000ns)}}}}}
2025-11-19T14:00:25.879039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436070049120320:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44121
2025-11-19T14:00:25.879581Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436069434734999:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(31736352ns) {Configure:104(24578220ns) {ProcessConfigured:135(24204803ns) {StartInitialHandshake:157(24204338ns) {DropHandshakes:469(480ns) {DropIncomingHandshake:441(91ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(286ns) PrepareNewSessionHandshake:476(1628ns) {SwitchToState:178(1093ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.879625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436071234763411:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(24056095ns) {Configure:104(24055468ns) {ProcessConfigured:135(23701014ns) {StartInitialHandshake:157(23700185ns) {DropHandshakes:469(410ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(45ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:476(1247ns) {SwitchToState:178(936ns)}}}}}
2025-11-19T14:00:25.879934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436071234763489:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35437
2025-11-19T14:00:25.879948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436071234763490:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35437
2025-11-19T14:00:25.879961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436071234763491:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35437
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.879894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436070054529281:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(23928668ns) {Configure:104(23928112ns) {ProcessConfigured:135(23649668ns) {StartInitialHandshake:157(23649081ns) {DropHandshakes:469(458ns) {DropIncomingHandshake:441(86ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(266ns) PrepareNewSessionHandshake:476(1023ns) {SwitchToState:178(635ns)}}}}}
2025-11-19T14:00:25.880147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436070916530014:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6988
2025-11-19T14:00:25.880169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436070916530015:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6988
2025-11-19T14:00:25.880180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436070916530016:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6988
2025-11-19T14:00:25.882290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436067943112426:2067] [node 5] ICP01 ready to work
2025-11-19T14:00:25.886184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436069434735034:2075] [node 4] ICP01 ready to work
2025-11-19T14:00:25.889310Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436070049120270:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(49155422ns) {Configure:104(49154573ns) {ProcessConfigured:135(47768080ns) {StartInitialHandshake:157(47767353ns) {DropHandshakes:469(255ns) {DropIncomingHandshake:441(29ns) DropOutgoingHandshake:456(23ns)} GenerateSessionVirtualId:988(50ns) PrepareNewSessionHandshake:476(1580ns) {SwitchToState:178(909ns)}}}}}
2025-11-19T14:00:25.895115Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436069434735040:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.895177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436069434734978:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.899788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436067943112392:2070] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(54628272ns) {Configure:104(54627391ns) {ProcessConfigured:135(54251944ns) {StartInitialHandshake:157(54250988ns) {DropHandshakes:469(389ns) {DropIncomingHandshake:441(27ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:476(1965ns) {SwitchToState:178(987ns)}}}}}
2025-11-19T14:00:25.900880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436067943112430:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-19T14:00:25.900945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436067943112431:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-19T14:00:25.900956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436067943112432:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-19T14:00:25.900967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436067943112433:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-19T14:00:25.901950Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436067943112430:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.903797Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436070916530004:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.903825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436070916530004:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.904768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436069434735057:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21062
2025-11-19T14:00:25.904789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436069434735058:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21062
2025-11-19T14:00:25.905226Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436070916530004:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:25.905315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436070916530000:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.905333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:25.905340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436070916530000:2106] [node 5] ICP22 created new session: [2:7574436070916530018:2106]
2025-11-19T14:00:25.905348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436070916530018:2106] [node 5] ICS09 handshake done sender: [2:7574436070916530004:2111] self: [2:7574436070916530003:0] peer: [5:7574436069434735056:0] socket: 323 qp: -1
2025-11-19T14:00:25.905357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436070916530018:2106] [node 5] ICS10 traffic start
2025-11-19T14:00:25.905367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.909714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436070049120323:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44121
2025-11-19T14:00:25.908979Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071234763490:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.909139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436071234763492:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35437
2025-11-19T14:00:25.910080Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436069434735040:2106] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:25.910196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436069434734978:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.910219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:25.910225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436069434734978:2049] [node 2] ICP22 created new session: [5:7574436069434735059:2049]
2025-11-19T14:00:25.910233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436069434735059:2049] [node 2] ICS09 handshake done sender: [5:7574436069434735040:2106] self: [5:7574436069434735056:0] peer: [2:7574436070916530003:0] socket: 334 qp: -1
2025-11-19T14:00:25.910242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436069434735059:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:25.910268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:25.914615Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436070916530014:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.914837Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436070049120323:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.914867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436070049120244:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.914998Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436070916530012:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.915004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436070916530012:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.915715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436070049120325:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44121
2025-11-19T14:00:25.919871Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436070916530012:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:25.920287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436070916529935:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.920321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:25.920330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436070916529935:2049] [node 3] ICP22 created new session: [2:7574436070916530023:2049]
2025-11-19T14:00:25.920339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436070916530023:2049] [node 3] ICS09 handshake done sender: [2:7574436070916530012:2114] self: [2:7574436070916529994:0] peer: [3:7574436070049120324:0] socket: 355 qp: -1
2025-11-19T14:00:25.920347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436070916530023:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:25.920359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:25.920337Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436070049120323:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:25.920543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436070049120244:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.920560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:25.920565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436070049120244:2049] [node 2] ICP22 created new session: [3:7574436070049120326:2049]
2025-11-19T14:00:25.920571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436070049120326:2049] [node 2] ICS09 handshake done sender: [3:7574436070049120323:2116] self: [3:7574436070049120324:0] peer: [2:7574436070916529994:0] socket: 357 qp: -1
2025-11-19T14:00:25.920578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436070049120326:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:25.920598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.924302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436067943112362:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(66939822ns) {Configure:104(66938528ns) {ProcessConfigured:135(66620643ns) {StartInitialHandshake:157(66619661ns) {DropHandshakes:469(494ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:476(1142ns) {SwitchToState:178(181ns)}}}}}
2025-11-19T14:00:25.923374Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071234763489:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.923495Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436070054529354:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.923529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436070054529279:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.924181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436070054529360:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48247
2025-11-19T14:00:25.924198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436070054529361:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48247
2025-11-19T14:00:25.928089Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436067943112363:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(45703525ns) {Configure:104(45701740ns) {ProcessConfigured:135(45334091ns) {StartInitialHandshake:157(45333034ns) {DropHandshakes:469(2201ns) {DropIncomingHandshake:441(217ns) DropOutgoingHandshake:456(188ns)} GenerateSessionVirtualId:988(929ns) PrepareNewSessionHandshake:476(1734ns) {SwitchToState:178(891ns)}}}}}
2025-11-19T14:00:25.935266Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436070916530002:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.935299Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436070916530015:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.935542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436070916529934:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.936568Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436070054529353:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.936607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436070054529281:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.936358Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071234763477:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.936380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436071234763477:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.937470Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436070049120303:2101] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.937483Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436070049120303:2101] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.938259Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436067943112433:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.938295Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436067943112431:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.938367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436067943112361:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.938377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436067943112362:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.938557Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436070049120303:2101] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:25.938689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436070049120299:2077] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.938708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:25.938716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436070049120299:2077] [node 6] ICP22 created new session: [3:7574436070049120330:2077]
2025-11-19T14:00:25.938724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436070049120330:2077] [node 6] ICS09 handshake done sender: [3:7574436070049120303:2101] self: [3:7574436070049120302:0] peer: [6:7574436070054529362:0] socket: 326 qp: -1
2025-11-19T14:00:25.938731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436070049120330:2077] [node 6] ICS10 traffic start
2025-11-19T14:00:25.938740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:25.939213Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436067943112432:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.939240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436067943112363:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.939877Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071234763491:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.940330Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436071234763477:2110] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:25.941456Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436070054529360:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.941483Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436070054529361:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.941511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436070054529309:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.941521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436070054529329:2087] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.939360Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436070916530006:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.939370Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436070916530006:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.941663Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436070916529935:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(21276771ns) {HandleHandshakeStatus:329(21275396ns) {DropIncomingHandshake:441(7238ns) DropOutgoingHandshake:456(259ns) SwitchToState:178(341ns) ProcessPendingSessionEvents:495(20562222ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20542617ns max# 20453659ns}}}
2025-11-19T14:00:25.941880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436070916530000:2106] [node 5] ICP35 event processing took too much time PendingConnection:278(36472811ns) {HandleHandshakeStatus:329(36471265ns) {DropIncomingHandshake:441(518ns) DropOutgoingHandshake:456(400ns) SwitchToState:178(391ns) ProcessPendingSessionEvents:495(17891825ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17874016ns max# 17781047ns}}}
2025-11-19T14:00:25.942456Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436070049120320:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.943785Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071234763480:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.943805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436071234763480:2111] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.944424Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071234763482:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.944428Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436071234763482:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.945213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436071234763471:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.945234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:25.945242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436071234763471:2107] [node 6] ICP22 created new session: [1:7574436071234763498:2107]
2025-11-19T14:00:25.945249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436071234763498:2107] [node 6] ICS09 handshake done sender: [1:7574436071234763477:2110] self: [1:7574436071234763472:0] peer: [6:7574436070054529359:0] socket: 331 qp: -1
2025-11-19T14:00:25.945257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436071234763498:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:25.945268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:25.946424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436070054529365:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48247
2025-11-19T14:00:25.946461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436070054529367:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48247
2025-11-19T14:00:25.946531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436070054529366:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48247
2025-11-19T14:00:25.946669Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436070054529353:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:25.946929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436070054529281:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.946948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:25.946954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436070054529281:2050] [node 3] ICP22 created new session: [6:7574436070054529368:2050]
2025-11-19T14:00:25.946960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436070054529368:2050] [node 3] ICS09 handshake done sender: [6:7574436070054529353:2111] self: [6:7574436070054529362:0] peer: [3:7574436070049120302:0] socket: 328 qp: -1
2025-11-19T14:00:25.946967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436070054529368:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:25.946975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.947243Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436069434735051:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.947255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436069434735051:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.948135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436069434735062:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21062
2025-11-19T14:00:25.948707Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436071234763480:2111] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:25.948850Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436071234763482:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:25.948964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436071234763410:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.948983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:25.948987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436071234763410:2048] [node 2] ICP22 created new session: [1:7574436071234763503:2048]
2025-11-19T14:00:25.948992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436071234763503:2048] [node 2] ICS09 handshake done sender: [1:7574436071234763480:2111] self: [1:7574436071234763478:0] peer: [2:7574436070916530027:0] socket: 332 qp: -1
2025-11-19T14:00:25.948997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436071234763503:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:25.949003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.950839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436071234763502:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35437
2025-11-19T14:00:25.950909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436071234763475:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.950919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:25.950926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436071234763475:2108] [node 4] ICP22 created new session: [1:7574436071234763505:2108]
2025-11-19T14:00:25.950932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436071234763505:2108] [node 4] ICS09 handshake done sender: [1:7574436071234763482:2112] self: [1:7574436071234763481:0] peer: [4:7574436067943112444:0] socket: 333 qp: -1
2025-11-19T14:00:25.950938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436071234763505:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:25.950957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:25.951733Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436069434735057:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.951896Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436069434735051:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:25.951964Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436070916530006:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:25.953472Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436067943112429:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.953491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436067943112429:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.953810Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436070049120305:2102] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.953826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436070049120305:2102] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.954045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436069434734999:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.954064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:25.954071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436069434734999:2066] [node 6] ICP22 created new session: [5:7574436069434735064:2066]
2025-11-19T14:00:25.954077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436069434735064:2066] [node 6] ICS09 handshake done sender: [5:7574436069434735051:2112] self: [5:7574436069434735033:0] peer: [6:7574436070054529364:0] socket: 358 qp: -1
2025-11-19T14:00:25.954083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436069434735064:2066] [node 6] ICS10 traffic start
2025-11-19T14:00:25.954092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.960063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436067943112447:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-19T14:00:25.960459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436067943112448:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-19T14:00:25.960826Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436067943112433:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:25.960579Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436070054529354:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:25.960930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436067943112361:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.960952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:25.960958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436067943112361:2048] [node 1] ICP22 created new session: [4:7574436067943112449:2048]
2025-11-19T14:00:25.960966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436067943112449:2048] [node 1] ICS09 handshake done sender: [4:7574436067943112433:2114] self: [4:7574436067943112444:0] peer: [1:7574436071234763481:0] socket: 347 qp: -1
2025-11-19T14:00:25.960973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436067943112449:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:25.960982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.960711Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071234763492:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.960746Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071234763502:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.960791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436071234763475:2108] [node 4] ICP18 handshake (actor [1:7574436071234763492:2118]) is too old
2025-11-19T14:00:25.961440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436069434735034:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.963218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436070916530001:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.963242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:25.963250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436070916530001:2108] [node 4] ICP22 created new session: [2:7574436070916530029:2108]
2025-11-19T14:00:25.963257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436070916530029:2108] [node 4] ICS09 handshake done sender: [2:7574436070916530006:2112] self: [2:7574436070916530005:0] peer: [4:7574436067943112445:0] socket: 324 qp: -1
2025-11-19T14:00:25.963265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436070916530029:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:25.963276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:25.963698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436070916529934:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28042) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436070916530015:2117] held: no
2025-11-19T14:00:25.965536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436070054529279:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28042) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436070054529354:2112] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.964709Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436067943112437:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.964722Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436067943112437:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.965408Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436067943112431:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.965952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436070054529279:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.965971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:25.965983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436070054529279:2048] [node 1] ICP22 created new session: [6:7574436070054529372:2048]
2025-11-19T14:00:25.965990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436070054529372:2048] [node 1] ICS09 handshake done sender: [6:7574436070054529354:2112] self: [6:7574436070054529359:0] peer: [1:7574436071234763472:0] socket: 338 qp: -1
2025-11-19T14:00:25.965998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436070054529372:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:25.966005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:25.966596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436070916530031:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6988
2025-11-19T14:00:25.968266Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436070916530015:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:25.968358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436070916529934:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.968372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:25.968386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436070916529934:2048] [node 1] ICP22 created new session: [2:7574436070916530032:2048]
2025-11-19T14:00:25.968391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436070916530032:2048] [node 1] ICS09 handshake done sender: [2:7574436070916530015:2117] self: [2:7574436070916530027:0] peer: [1:7574436071234763478:0] socket: 342 qp: -1
2025-11-19T14:00:25.968397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436070916530032:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:25.968403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:25.967344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436070049120335:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44121
2025-11-19T14:00:25.967669Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436070049120305:2102] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:25.967763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436070049120301:2087] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.967775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:25.967781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436070049120301:2087] [node 4] ICP22 created new session: [3:7574436070049120336:2087]
2025-11-19T14:00:25.967787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436070049120336:2087] [node 4] ICS09 handshake done sender: [3:7574436070049120305:2102] self: [3:7574436070049120304:0] peer: [4:7574436067943112446:0] socket: 327 qp: -1
2025-11-19T14:00:25.967794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436070049120336:2087] [node 4] ICS10 traffic start
2025-11-19T14:00:25.967802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:25.969089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436069434735066:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21062
2025-11-19T14:00:25.972343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436070054529370:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48247
2025-11-19T14:00:25.972482Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436070054529365:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.974058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436070054529280:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.974579Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436070054529361:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:25.974658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436070054529329:2087] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.974676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:25.974685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436070054529329:2087] [node 5] ICP22 created new session: [6:7574436070054529375:2087]
2025-11-19T14:00:25.974692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436070054529375:2087] [node 5] ICS09 handshake done sender: [6:7574436070054529361:2118] self: [6:7574436070054529364:0] peer: [5:7574436069434735033:0] socket: 359 qp: -1
2025-11-19T14:00:25.974700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436070054529375:2087] [node 5] ICS10 traffic start
2025-11-19T14:00:25.974710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:25.975647Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436070916530013:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.975667Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436070916530013:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.976758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436067943112451:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-19T14:00:25.976789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436067943112454:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7441
2025-11-19T14:00:25.977006Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436070916530013:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:25.977142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436070916529991:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.977161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:25.977170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436070916529991:2103] [node 6] ICP22 created new session: [2:7574436070916530036:2103]
2025-11-19T14:00:25.977176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436070916530036:2103] [node 6] ICS09 handshake done sender: [2:7574436070916530013:2115] self: [2:7574436070916530010:0] peer: [6:7574436070054529374:0] socket: 330 qp: -1
2025-11-19T14:00:25.977182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436070916530036:2103] [node 6] ICS10 traffic start
2025-11-19T14:00:25.977190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:25.977953Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436069434735066:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.978018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436069434734977:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.978250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436070916530035:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6988
2025-11-19T14:00:25.978670Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436067943112429:2110] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:25.978791Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436067943112437:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:25.978967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436067943112392:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.978982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:25.978987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436067943112392:2070] [node 6] ICP22 created new session: [4:7574436067943112456:2070]
2025-11-19T14:00:25.978992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436067943112456:2070] [node 6] ICS09 handshake done sender: [4:7574436067943112429:2110] self: [4:7574436067943112393:0] peer: [6:7574436070054529363:0] socket: 348 qp: -1
2025-11-19T14:00:25.978997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436067943112456:2070] [node 6] ICS10 traffic start
2025-11-19T14:00:25.979004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:25.979211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436067943112426:2067] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.979218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:25.979222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436067943112426:2067] [node 5] ICP22 created new session: [4:7574436067943112458:2067]
2025-11-19T14:00:25.979225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436067943112458:2067] [node 5] ICS09 handshake done sender: [4:7574436067943112437:2116] self: [4:7574436067943112436:0] peer: [5:7574436069434735067:0] socket: 350 qp: -1
2025-11-19T14:00:25.979229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436067943112458:2067] [node 5] ICS10 traffic start
2025-11-19T14:00:25.979233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:25.979389Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436067943112432:2113] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:25.980931Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436069434734999:2066] [node 6] ICP35 event processing took too much time PendingConnection:278(26801903ns) {HandleHandshakeStatus:329(26800963ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(217ns) SwitchToState:178(494ns) ProcessPendingSessionEvents:495(8591124ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8568339ns max# 8482737ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.980302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436067943112361:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(19260817ns) {HandleHandshakeStatus:329(19259707ns) {DropIncomingHandshake:441(413ns) DropOutgoingHandshake:456(2292ns) SwitchToState:178(315ns) ProcessPendingSessionEvents:495(11115767ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11092667ns max# 11016371ns}}}
2025-11-19T14:00:25.980791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436067943112362:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.980803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:25.980810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436067943112362:2049] [node 2] ICP22 created new session: [4:7574436067943112460:2049]
2025-11-19T14:00:25.980816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436067943112460:2049] [node 2] ICS09 handshake done sender: [4:7574436067943112431:2112] self: [4:7574436067943112445:0] peer: [2:7574436070916530005:0] socket: 345 qp: -1
2025-11-19T14:00:25.980821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436067943112460:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:25.980844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:25.983341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436069434735070:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21062
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.983645Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071234763484:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.983658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436071234763484:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.984922Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436071234763484:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:25.986831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436071234763476:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:25.986855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:25.986863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436071234763476:2109] [node 5] ICP22 created new session: [1:7574436071234763510:2109]
2025-11-19T14:00:25.986869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436071234763510:2109] [node 5] ICS09 handshake done sender: [1:7574436071234763484:2113] self: [1:7574436071234763483:0] peer: [5:7574436069434735069:0] socket: 373 qp: -1
2025-11-19T14:00:25.986875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436071234763510:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:25.986882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:25.987421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574436069434734999:2066] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:25.987482Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436069434735062:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.987514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436069434734979:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.987661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436069434735073:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21062
2025-11-19T14:00:25.987802Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436070049120335:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.987856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436070049120322:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.987860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436070049120322:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.988026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436067943112363:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:25.988051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:25.988056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436067943112363:2050] [node 3] ICP22 created new session: [4:7574436067943112463:2050]
2025-11-19T14:00:25.988061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436067943112463:2050] [node 3] ICS09 handshake done sender: [4:7574436067943112432:2113] self: [4:7574436067943112446:0] peer: [3:7574436070049120304:0] socket: 346 qp: -1
2025-11-19T14:00:25.988067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436067943112463:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:25.988073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.988234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436070049120243:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:25.988489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436070054529280:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19483) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436070054529365:2119] held: no
2025-11-19T14:00:25.988825Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436070054529309:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4381) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436070054529360:2117] held: no
2025-11-19T14:00:25.992586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436069434735071:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21062
2025-11-19T14:00:25.992637Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436071234763488:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.992651Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436071234763488:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:25.992678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436070054529377:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48247
2025-11-19T14:00:25.992136Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436067943112451:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:25.994315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574436067943112426:2067] [node 5] ICP18 handshake (actor [4:7574436067943112451:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:25.995162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436070049120340:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44121
2025-11-19T14:00:25.993599Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436069434734977:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28042) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436069434735066:2119] held: no
2025-11-19T14:00:25.993695Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436069434735066:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:25.995834Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436070049120322:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.000045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436069434734977:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.000073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:26.000085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436069434734977:2048] [node 1] ICP22 created new session: [5:7574436069434735075:2048]
2025-11-19T14:00:26.000092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436069434735075:2048] [node 1] ICS09 handshake done sender: [5:7574436069434735066:2119] self: [5:7574436069434735069:0] peer: [1:7574436071234763483:0] socket: 340 qp: -1
2025-11-19T14:00:26.000101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436069434735075:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:26.000111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:26.001390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436070054529378:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48247
2025-11-19T14:00:26.003403Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436070916530001:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:26.003492Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436070054529365:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.004552Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436071234763488:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:26.003667Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436069434735057:2116] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:26.004711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436070054529280:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.004725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:26.004730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436070054529280:2049] [node 2] ICP22 created new session: [6:7574436074349496675:2049]
2025-11-19T14:00:26.004735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436074349496675:2049] [node 2] ICS09 handshake done sender: [6:7574436070054529365:2119] self: [6:7574436070054529374:0] peer: [2:7574436070916530010:0] socket: 360 qp: -1
2025-11-19T14:00:26.004741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436074349496675:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:26.004748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.005286Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436069434735034:2075] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4381) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436069434735057:2116] held: no
2025-11-19T14:00:26.005369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436069434735034:2075] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.005376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:26.005381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436069434735034:2075] [node 4] ICP22 created new session: [5:7574436073729702373:2075]
2025-11-19T14:00:26.005385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436073729702373:2075] [node 4] ICS09 handshake done sender: [5:7574436069434735057:2116] self: [5:7574436069434735067:0] peer: [4:7574436067943112436:0] socket: 353 qp: -1
2025-11-19T14:00:26.005390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436073729702373:2075] [node 4] ICS10 traffic start
2025-11-19T14:00:26.005396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:26.005596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436069434735074:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21062
2025-11-19T14:00:26.006193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436071234763411:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.006214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:26.006219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436071234763411:2049] [node 3] ICP22 created new session: [1:7574436075529730809:2049]
2025-11-19T14:00:26.006225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436075529730809:2049] [node 3] ICS09 handshake done sender: [1:7574436071234763488:2114] self: [1:7574436071234763479:0] peer: [3:7574436070049120338:0] socket: 374 qp: -1
2025-11-19T14:00:26.006230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436075529730809:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:26.006238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:26.007069Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436070054529360:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:26.007336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436070054529309:2061] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.007351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:26.007357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436070054529309:2061] [node 4] ICP22 created new session: [6:7574436074349496677:2061]
2025-11-19T14:00:26.007362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436074349496677:2061] [node 4] ICS09 handshake done sender: [6:7574436070054529360:2117] self: [6:7574436070054529363:0] peer: [4:7574436067943112393:0] socket: 351 qp: -1
2025-11-19T14:00:26.007368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436074349496677:2061] [node 4] ICS10 traffic start
2025-11-19T14:00:26.007375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:26.004046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436074344087637:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44121
2025-11-19T14:00:26.004139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436070049120270:2071] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.004158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:26.004164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436070049120270:2071] [node 5] ICP22 created new session: [3:7574436074344087638:2071]
2025-11-19T14:00:26.004170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436074344087638:2071] [node 5] ICS09 handshake done sender: [3:7574436070049120322:2115] self: [3:7574436070049120272:0] peer: [5:7574436069434735072:0] socket: 325 qp: -1
2025-11-19T14:00:26.004178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436074344087638:2071] [node 5] ICS10 traffic start
2025-11-19T14:00:26.004188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:26.007806Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436070049120335:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:26.008017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436070049120243:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.008044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:26.008054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436070049120243:2048] [node 1] ICP22 created new session: [3:7574436074344087640:2048]
2025-11-19T14:00:26.008059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436074344087640:2048] [node 1] ICS09 handshake done sender: [3:7574436070049120335:2118] self: [3:7574436070049120338:0] peer: [1:7574436071234763479:0] socket: 375 qp: -1
2025-11-19T14:00:26.008066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436074344087640:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:26.008073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:26.010378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436069434734979:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22956) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436069434735062:2118] held: no
2025-11-19T14:00:26.010542Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436069434735062:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:26.010976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436069434734979:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.010994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:26.011001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436069434734979:2050] [node 3] ICP22 created new session: [5:7574436073729702375:2050]
2025-11-19T14:00:26.011006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436073729702375:2050] [node 3] ICS09 handshake done sender: [5:7574436069434735062:2118] self: [5:7574436069434735072:0] peer: [3:7574436070049120272:0] socket: 364 qp: -1
2025-11-19T14:00:26.011013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436073729702375:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:26.011021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:26.019715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:26.019851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436070916530032:2048] [node 1] ICS01 socket: 342 reason# EndOfStream
2025-11-19T14:00:26.019863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436070916529934:2048] [node 1] ICP30 unregister session Session# [2:7574436070916530032:2048] VirtualId# [2:7574436070916530027:0]
2025-11-19T14:00:26.019872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436070916530032:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.019917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:26.019940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:26.023865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:26.023982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436074344087640:2048] [node 1] ICS01 socket: 375 reason# EndOfStream
2025-11-19T14:00:26.024003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436070049120243:2048] [node 1] ICP30 unregister session Session# [3:7574436074344087640:2048] VirtualId# [3:7574436070049120338:0]
2025-11-19T14:00:26.024012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436074344087640:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.024071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:26.024099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:26.032958Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:26.033048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:26.033142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436067943112449:2048] [node 1] ICS07 socket disconnect 347 reason# EndOfStream
2025-11-19T14:00:26.033149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436067943112449:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.033193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:26.033448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:26.033511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436069434735075:2048] [node 1] ICS01 socket: 340 reason# EndOfStream
2025-11-19T14:00:26.033518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436069434734977:2048] [node 1] ICP30 unregister session Session# [5:7574436069434735075:2048] VirtualId# [5:7574436069434735069:0]
2025-11-19T14:00:26.033523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436069434735075:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.033558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:26.033581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:26.033215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436067943112449:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:26.033718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436070054529372:2048] [node 1] ICS01 socket: 338 reason# EndOfStream
2025-11-19T14:00:26.033726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436070054529279:2048] [node 1] ICP30 unregister session Session# [6:7574436070054529372:2048] VirtualId# [6:7574436070054529359:0]
2025-11-19T14:00:26.033730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436070054529372:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.033754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:26.033773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:26.034301Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436067943112361:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28042) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:26.034314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436067943112449:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:26.034319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436067943112361:2048] [node 1] ICP30 unregister session Session# [4:7574436067943112449:2048] VirtualId# [4:7574436067943112444:0]
2025-11-19T14:00:26.034364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:26.034371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436067943112361:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28042) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:26.034375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28042) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:26.047117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574436067943112361:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:26.071870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:26.071985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436070049120326:2049] [node 2] ICS01 socket: 357 reason# EndOfStream
2025-11-19T14:00:26.071998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436070049120244:2049] [node 2] ICP30 unregister session Session# [3:7574436070049120326:2049] VirtualId# [3:7574436070049120324:0]
2025-11-19T14:00:26.072008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436070049120326:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.072065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:26.072088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:26.086178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:26.086300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436067943112460:2049] [node 2] ICS07 socket disconnect 345 reason# EndOfStream
2025-11-19T14:00:26.086307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436067943112460:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.086361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:26.086381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436067943112460:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:26.086921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:26.087009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436074349496675:2049] [node 2] ICS01 socket: 360 reason# EndOfStream
2025-11-19T14:00:26.087017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436070054529280:2049] [node 2] ICP30 unregister session Session# [6:7574436074349496675:2049] VirtualId# [6:7574436070054529374:0]
2025-11-19T14:00:26.087025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436074349496675:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.087069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:26.087093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:26.090226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:26.090316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436069434735059:2049] [node 2] ICS01 socket: 334 reason# EndOfStream
2025-11-19T14:00:26.090326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436069434734978:2049] [node 2] ICP30 unregister session Session# [5:7574436069434735059:2049] VirtualId# [5:7574436069434735056:0]
2025-11-19T14:00:26.090334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436069434735059:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.090384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:26.090408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:26.092010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436067943112362:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19483) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:26.092029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436067943112460:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:26.092037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436067943112362:2049] [node 2] ICP30 unregister session Session# [4:7574436067943112460:2049] VirtualId# [4:7574436067943112445:0]
2025-11-19T14:00:26.092094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:26.092102Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436067943112362:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19483) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:26.092106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19483) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:26.104145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:26.104256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436073729702375:2050] [node 3] ICS01 socket: 364 reason# EndOfStream
2025-11-19T14:00:26.104267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436069434734979:2050] [node 3] ICP30 unregister session Session# [5:7574436073729702375:2050] VirtualId# [5:7574436069434735072:0]
2025-11-19T14:00:26.104275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436073729702375:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.104329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:26.104355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:26.104392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:26.104452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436070054529368:2050] [node 3] ICS01 socket: 328 reason# EndOfStream
2025-11-19T14:00:26.104458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436070054529281:2050] [node 3] ICP30 unregister session Session# [6:7574436070054529368:2050] VirtualId# [6:7574436070054529362:0]
2025-11-19T14:00:26.104462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436070054529368:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.104488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:26.104508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:26.161376Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:26.161497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436074349496677:2061] [node 4] ICS01 socket: 351 reason# EndOfStream
2025-11-19T14:00:26.161508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436070054529309:2061] [node 4] ICP30 unregister session Session# [6:7574436074349496677:2061] VirtualId# [6:7574436070054529363:0]
2025-11-19T14:00:26.161518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436074349496677:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.161573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:26.161598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:26.311506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436072361564470:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:26.311526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436072361564471:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:26.348501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436073432220398:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:26.348518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436073432220399:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:26.399868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436075318945275:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:26.399883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436075318945276:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:26.444448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436072226724333:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:26.444462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436072226724334:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:26.444467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436072226724335:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:26.479891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436073084968004:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:26.479903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436073084968005:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:26.479908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436073084968006:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:26.500455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436074482964940:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:26.500476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436074482964941:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:26.500481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436074482964942:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:26.532538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436075318945323:2094] [node 5] ICP01 ready to work
2025-11-19T14:00:26.533139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436073432220449:2097] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.535240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436075318945330:2097] [node 4] ICP01 ready to work
2025-11-19T14:00:26.535939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436073432220458:2103] [node 4] ICP01 ready to work
2025-11-19T14:00:26.540004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436073084968047:2089] [node 6] ICP01 ready to work
2025-11-19T14:00:26.539598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436072361564538:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:26.540496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436074482964992:2095] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.541370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436072226724381:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14088
2025-11-19T14:00:26.544481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436075318945336:2101] [node 6] ICP01 ready to work
2025-11-19T14:00:26.544628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436073432220452:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33008
2025-11-19T14:00:26.544790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436073432220464:2106] [node 6] ICP01 ready to work
2025-11-19T14:00:26.544856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436072361564541:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.553484Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436075318945330:2097] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18174944ns) {Configure:104(18174402ns) {ProcessConfigured:135(17807693ns) {StartInitialHandshake:157(17806623ns) {DropHandshakes:469(571ns) {DropIncomingHandshake:441(103ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(179ns) PrepareNewSessionHandshake:476(1322ns) {SwitchToState:178(851ns)}}}}}
2025-11-19T14:00:26.556275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436075318945350:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30453
2025-11-19T14:00:26.556301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436075318945351:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30453
2025-11-19T14:00:26.556545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436073432220458:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(20534341ns) {Configure:104(20533801ns) {ProcessConfigured:135(20210821ns) {StartInitialHandshake:157(20210122ns) {DropHandshakes:469(437ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(129ns) PrepareNewSessionHandshake:476(1840ns) {SwitchToState:178(1382ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.557092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436072226724397:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14088
2025-11-19T14:00:26.557116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436072226724398:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14088
2025-11-19T14:00:26.559950Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436072361564538:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(20260033ns) {Configure:104(20259325ns) {ProcessConfigured:135(19801553ns) {StartInitialHandshake:157(19800782ns) {DropHandshakes:469(708ns) {DropIncomingHandshake:441(100ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(329ns) PrepareNewSessionHandshake:476(2277ns) {SwitchToState:178(1835ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.563149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436075318945350:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.563319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436075318945275:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:26.563353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436075318945354:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30453
2025-11-19T14:00:26.562658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436072226724400:2110] [node 5] ICP01 ready to work
2025-11-19T14:00:26.560816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436073084968070:2102] [node 4] ICP01 ready to work
2025-11-19T14:00:26.563479Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436073084968047:2089] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(23410920ns) {Configure:104(23410127ns) {ProcessConfigured:135(20039394ns) {StartInitialHandshake:157(20039084ns) {DropHandshakes:469(410ns) {DropIncomingHandshake:441(21ns) DropOutgoingHandshake:456(180ns)} GenerateSessionVirtualId:988(303ns) PrepareNewSessionHandshake:476(886ns) {SwitchToState:178(396ns)}}}}}
2025-11-19T14:00:26.563740Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436073432220452:2099] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.563507Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436072361564533:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.563516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436072361564533:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:26.566087Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436072361564533:2107] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:26.566525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436072361564536:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:26.566560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436072361564471:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.566575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:26.566583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436072361564471:2049] [node 3] ICP22 created new session: [1:7574436072361564548:2049]
2025-11-19T14:00:26.566590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436072361564548:2049] [node 3] ICS09 handshake done sender: [1:7574436072361564533:2107] self: [1:7574436072361564532:0] peer: [3:7574436075318945355:0] socket: 327 qp: -1
2025-11-19T14:00:26.566598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436072361564548:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:26.566618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.577209Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436073084968005:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(32582272ns) {Configure:104(32580655ns) {ProcessConfigured:135(32279961ns) {StartInitialHandshake:157(32279878ns) {DropHandshakes:469(450ns) {DropIncomingHandshake:441(27ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(156ns) PrepareNewSessionHandshake:476(1203ns) {SwitchToState:178(91ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.577478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436075318945336:2101] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(32931659ns) {Configure:104(32930808ns) {ProcessConfigured:135(32506709ns) {StartInitialHandshake:157(32506050ns) {DropHandshakes:469(455ns) {DropIncomingHandshake:441(93ns) DropOutgoingHandshake:456(44ns)} GenerateSessionVirtualId:988(305ns) PrepareNewSessionHandshake:476(1215ns) {SwitchToState:178(20ns)}}}}}
2025-11-19T14:00:26.577839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436072361564551:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59959
2025-11-19T14:00:26.577891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436072361564552:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59959
2025-11-19T14:00:26.577923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436072361564553:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59959
2025-11-19T14:00:26.578023Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436073432220464:2106] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(33191413ns) {Configure:104(33190934ns) {ProcessConfigured:135(32645367ns) {StartInitialHandshake:157(32645140ns) {DropHandshakes:469(402ns) {DropIncomingHandshake:441(123ns) DropOutgoingHandshake:456(31ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:476(1199ns) {SwitchToState:178(226ns)}}}}}
2025-11-19T14:00:26.578296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436074482964940:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(33133038ns) {Configure:104(33131496ns) {ProcessConfigured:135(32807336ns) {StartInitialHandshake:157(32807022ns) {DropHandshakes:469(387ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:476(408ns) {SwitchToState:178(32ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.578645Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436073432220399:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(34486588ns) {Configure:104(34485400ns) {ProcessConfigured:135(32617972ns) {StartInitialHandshake:157(32617203ns) {DropHandshakes:469(805ns) {DropIncomingHandshake:441(136ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(1055ns) {SwitchToState:178(508ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.579949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436074482965017:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16905
2025-11-19T14:00:26.579351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436073084968081:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29542
2025-11-19T14:00:26.579399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436073084968082:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29542
2025-11-19T14:00:26.579429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436073084968083:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29542
2025-11-19T14:00:26.581769Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436073084968081:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.581797Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436073084968082:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.581830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436073084968006:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:26.581841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436073084968005:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:26.582071Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436072361564552:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.582100Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436072361564551:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.584074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436072226724397:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.584129Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436072226724398:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.586181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436074482965012:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16905
2025-11-19T14:00:26.586287Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436075318945325:2095] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.586302Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436075318945325:2095] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:26.586353Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436073432220451:2098] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.586368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436073432220451:2098] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.587799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436072226724394:2104] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.588781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436075318945358:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30453
2025-11-19T14:00:26.592034Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436074482965012:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.592093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436074482964940:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:26.592225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436072361564544:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.592236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436072361564544:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.592093Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436075318945351:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.592121Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436075318945350:2113] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:26.593341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436075318945275:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.593362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:26.593369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436075318945275:2048] [node 1] ICP22 created new session: [3:7574436075318945360:2048]
2025-11-19T14:00:26.593375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436075318945360:2048] [node 1] ICS09 handshake done sender: [3:7574436075318945350:2113] self: [3:7574436075318945355:0] peer: [1:7574436072361564532:0] socket: 330 qp: -1
2025-11-19T14:00:26.593383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436075318945360:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:26.593392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:26.593504Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436072361564544:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:26.592164Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436072226724334:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(58585337ns) {Configure:104(58584178ns) {ProcessConfigured:135(58190576ns) {StartInitialHandshake:157(58190007ns) {DropHandshakes:469(427ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(264ns) PrepareNewSessionHandshake:476(1138ns) {SwitchToState:178(438ns)}}}}}
2025-11-19T14:00:26.593950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436072361564541:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.593965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:26.593971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436072361564541:2110] [node 6] ICP22 created new session: [1:7574436072361564556:2110]
2025-11-19T14:00:26.593975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436072361564556:2110] [node 6] ICS09 handshake done sender: [1:7574436072361564544:2112] self: [1:7574436072361564543:0] peer: [6:7574436074482965018:0] socket: 336 qp: -1
2025-11-19T14:00:26.593981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436072361564556:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:26.593986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:26.593179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436074482965019:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16905
2025-11-19T14:00:26.593622Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436074482965012:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:26.593738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436074482964940:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.593748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:26.593755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436074482964940:2048] [node 1] ICP22 created new session: [6:7574436074482965020:2048]
2025-11-19T14:00:26.593760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436074482965020:2048] [node 1] ICS09 handshake done sender: [6:7574436074482965012:2112] self: [6:7574436074482965018:0] peer: [1:7574436072361564543:0] socket: 351 qp: -1
2025-11-19T14:00:26.593766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436074482965020:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:26.593796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:26.594425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436074482964985:2091] [node 5] ICP01 ready to work
2025-11-19T14:00:26.596120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436072226724410:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14088
2025-11-19T14:00:26.596199Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436072226724381:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.596533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436072226724409:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14088
2025-11-19T14:00:26.596641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436072226724334:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:26.596837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436072226724335:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(45842106ns) {Configure:104(45841462ns) {ProcessConfigured:135(45558131ns) {StartInitialHandshake:157(45557732ns) {DropHandshakes:469(482ns) {DropIncomingHandshake:441(81ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:476(1388ns) {SwitchToState:178(964ns)}}}}}
2025-11-19T14:00:26.596927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436072226724335:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:26.594061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436072361564536:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(26010803ns) {Configure:104(26009252ns) {ProcessConfigured:135(25517043ns) {StartInitialHandshake:157(25516429ns) {DropHandshakes:469(688ns) {DropIncomingHandshake:441(164ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(245ns) PrepareNewSessionHandshake:476(697ns) {SwitchToState:178(499ns)}}}}}
2025-11-19T14:00:26.601856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436073432220398:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(53874515ns) {Configure:104(53873829ns) {ProcessConfigured:135(53564728ns) {StartInitialHandshake:157(53564173ns) {DropHandshakes:469(450ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(864ns) {SwitchToState:178(660ns)}}}}}
2025-11-19T14:00:26.601907Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436073432220472:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.601910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436073432220472:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:26.602353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436073432220398:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.600216Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436072226724409:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.602718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436072226724417:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14088
2025-11-19T14:00:26.602797Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436073432220472:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:26.602893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436073432220458:2103] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.602908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:26.602914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436073432220458:2103] [node 4] ICP22 created new session: [2:7574436073432220480:2103]
2025-11-19T14:00:26.602920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436073432220480:2103] [node 4] ICS09 handshake done sender: [2:7574436073432220472:2112] self: [2:7574436073432220459:0] peer: [4:7574436072226724414:0] socket: 340 qp: -1
2025-11-19T14:00:26.602925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436073432220480:2103] [node 4] ICS10 traffic start
2025-11-19T14:00:26.602942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:26.603908Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436072361564530:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.603921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436072361564530:2106] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.604549Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436072361564471:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(37884831ns) {HandleHandshakeStatus:329(37883761ns) {DropIncomingHandshake:441(113ns) DropOutgoingHandshake:456(696ns) SwitchToState:178(175ns) ProcessPendingSessionEvents:495(19394474ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19374001ns max# 19303312ns}}}
2025-11-19T14:00:26.604610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436072361564471:2049] [node 3] ICP18 handshake (actor [1:7574436072361564552:2115]) is too old
2025-11-19T14:00:26.607971Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436072226724398:2109] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:26.604805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436073432220483:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33008
2025-11-19T14:00:26.610841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574436074482964940:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(17007764ns) {HandleHandshakeStatus:329(17007185ns) {DropIncomingHandshake:441(152ns) DropOutgoingHandshake:456(1088ns) SwitchToState:178(136ns) ProcessPendingSessionEvents:495(16896894ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16877959ns max# 16805050ns}}}
2025-11-19T14:00:26.611822Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436073432220451:2098] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:26.612231Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436073432220483:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.612247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436073432220449:2097] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.612259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:26.612266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436073432220449:2097] [node 5] ICP22 created new session: [2:7574436073432220485:2097]
2025-11-19T14:00:26.612272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436073432220485:2097] [node 5] ICS09 handshake done sender: [2:7574436073432220451:2098] self: [2:7574436073432220450:0] peer: [5:7574436073084968085:0] socket: 328 qp: -1
2025-11-19T14:00:26.612279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436073432220485:2097] [node 5] ICS10 traffic start
2025-11-19T14:00:26.612299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:26.599774Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436075318945276:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(31650383ns) {Configure:104(31649571ns) {ProcessConfigured:135(31259420ns) {StartInitialHandshake:157(31258955ns) {DropHandshakes:469(423ns) {DropIncomingHandshake:441(49ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:476(1009ns) {SwitchToState:178(582ns)}}}}}
2025-11-19T14:00:26.608334Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436075318945347:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.608341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436075318945347:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:26.612034Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436075318945325:2095] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:26.612650Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436074482965017:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.612673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436074482964941:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.614759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436072361564561:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59959
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.615228Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436074482964985:2091] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19129063ns) {Configure:104(19128225ns) {ProcessConfigured:135(18748059ns) {StartInitialHandshake:157(18747398ns) {DropHandshakes:469(610ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(77ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:476(919ns) {SwitchToState:178(513ns)}}}}}
2025-11-19T14:00:26.615708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436072226724334:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.615732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:26.615739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436072226724334:2049] [node 2] ICP22 created new session: [4:7574436072226724419:2049]
2025-11-19T14:00:26.615760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436072226724419:2049] [node 2] ICS09 handshake done sender: [4:7574436072226724398:2109] self: [4:7574436072226724414:0] peer: [2:7574436073432220459:0] socket: 342 qp: -1
2025-11-19T14:00:26.615767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436072226724419:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:26.615786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:26.616344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436072226724421:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14088
2025-11-19T14:00:26.618155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436072226724394:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21854474ns) {Configure:104(21853736ns) {ProcessConfigured:135(21510928ns) {StartInitialHandshake:157(21509648ns) {DropHandshakes:469(561ns) {DropIncomingHandshake:441(58ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(327ns) PrepareNewSessionHandshake:476(952ns) {SwitchToState:178(521ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.618733Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436075318945347:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.618956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436073084968087:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29542
2025-11-19T14:00:26.619004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436073084968088:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29542
2025-11-19T14:00:26.619554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436075318945323:2094] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.619567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:26.619574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436075318945323:2094] [node 5] ICP22 created new session: [3:7574436075318945366:2094]
2025-11-19T14:00:26.619581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436075318945366:2094] [node 5] ICS09 handshake done sender: [3:7574436075318945325:2095] self: [3:7574436075318945324:0] peer: [5:7574436073084968084:0] socket: 325 qp: -1
2025-11-19T14:00:26.619588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436075318945366:2094] [node 5] ICS10 traffic start
2025-11-19T14:00:26.619640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:26.619897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436075318945330:2097] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.619903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:26.619908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436075318945330:2097] [node 4] ICP22 created new session: [3:7574436075318945368:2097]
2025-11-19T14:00:26.619913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436075318945368:2097] [node 4] ICS09 handshake done sender: [3:7574436075318945347:2111] self: [3:7574436075318945331:0] peer: [4:7574436072226724415:0] socket: 338 qp: -1
2025-11-19T14:00:26.619918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436075318945368:2097] [node 4] ICS10 traffic start
2025-11-19T14:00:26.619922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.620792Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436073084968081:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:26.621283Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436072361564530:2106] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:26.621969Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436072361564546:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.621976Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436072361564546:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:26.622801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436072361564470:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.622815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:26.622821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436072361564470:2048] [node 2] ICP22 created new session: [1:7574436072361564563:2048]
2025-11-19T14:00:26.622827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436072361564563:2048] [node 2] ICS09 handshake done sender: [1:7574436072361564530:2106] self: [1:7574436072361564529:0] peer: [2:7574436073432220478:0] socket: 324 qp: -1
2025-11-19T14:00:26.622832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436072361564563:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:26.622840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:26.621843Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436072226724410:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.621879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436072226724333:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:26.622014Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436072226724397:2108] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:26.622096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436072226724335:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.622107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:26.622113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436072226724335:2050] [node 3] ICP22 created new session: [4:7574436072226724426:2050]
2025-11-19T14:00:26.622118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436072226724426:2050] [node 3] ICS09 handshake done sender: [4:7574436072226724397:2108] self: [4:7574436072226724415:0] peer: [3:7574436075318945331:0] socket: 341 qp: -1
2025-11-19T14:00:26.622124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436072226724426:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:26.622131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.625675Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436075318945354:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.625734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436075318945364:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30453
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.626182Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436073432220474:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.626198Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436073432220474:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:26.626405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436073084968089:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29542
2025-11-19T14:00:26.626448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436073084968090:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29542
2025-11-19T14:00:26.626464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436073084968006:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.626477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:26.626483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436073084968006:2050] [node 3] ICP22 created new session: [5:7574436073084968091:2050]
2025-11-19T14:00:26.626489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436073084968091:2050] [node 3] ICS09 handshake done sender: [5:7574436073084968081:2114] self: [5:7574436073084968084:0] peer: [3:7574436075318945324:0] socket: 329 qp: -1
2025-11-19T14:00:26.626494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436073084968091:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:26.626513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:26.626362Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436075318945275:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(25874733ns) {HandleHandshakeStatus:329(25873987ns) {DropIncomingHandshake:441(246ns) DropOutgoingHandshake:456(2789ns) SwitchToState:178(160ns) ProcessPendingSessionEvents:495(18812086ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18792151ns max# 18719507ns}}}
2025-11-19T14:00:26.626530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436074482965026:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16905
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.629329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436073432220489:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33008
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.630759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436074482965027:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16905
2025-11-19T14:00:26.630134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574436075318945323:2094] [node 5] ICP18 handshake (actor [3:7574436075318945354:2115]) is too old
2025-11-19T14:00:26.631149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436073084968087:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.632332Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436072361564546:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:26.633505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436072361564538:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.633520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:26.633527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436072361564538:2109] [node 4] ICP22 created new session: [1:7574436072361564566:2109]
2025-11-19T14:00:26.633534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436072361564566:2109] [node 4] ICS09 handshake done sender: [1:7574436072361564546:2113] self: [1:7574436072361564539:0] peer: [4:7574436072226724425:0] socket: 354 qp: -1
2025-11-19T14:00:26.633541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436072361564566:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:26.633559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:26.636172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436075318945365:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30453
2025-11-19T14:00:26.637747Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436072361564561:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.637780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436072361564470:2048] [node 2] ICP18 handshake (actor [1:7574436072361564561:2118]) is too old
2025-11-19T14:00:26.637539Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436073432220449:2097] [node 5] ICP35 event processing took too much time PendingConnection:278(25192853ns) {HandleHandshakeStatus:329(25191749ns) {DropIncomingHandshake:441(123ns) DropOutgoingHandshake:456(286ns) SwitchToState:178(179ns) ProcessPendingSessionEvents:495(11285768ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11265104ns max# 11196063ns}}}
2025-11-19T14:00:26.637719Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436073432220452:2099] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:26.638470Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436073084968082:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.638745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436074482965028:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16905
2025-11-19T14:00:26.639129Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436073084968090:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.639150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436073084968004:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:26.639278Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436075318945365:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.639304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436075318945276:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:26.639435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436073084968005:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.639451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:26.639456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436073084968005:2049] [node 2] ICP22 created new session: [5:7574436073084968095:2049]
2025-11-19T14:00:26.639461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436073084968095:2049] [node 2] ICS09 handshake done sender: [5:7574436073084968082:2115] self: [5:7574436073084968085:0] peer: [2:7574436073432220450:0] socket: 359 qp: -1
2025-11-19T14:00:26.639466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436073084968095:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:26.639474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.640391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436074482965029:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16905
2025-11-19T14:00:26.641858Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436073432220474:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:26.642080Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436074482965017:2115] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:26.642235Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436073432220476:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.642244Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436073432220476:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:26.642865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436073432220398:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.642881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:26.642887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436073432220398:2048] [node 1] ICP22 created new session: [2:7574436073432220491:2048]
2025-11-19T14:00:26.642893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436073432220491:2048] [node 1] ICS09 handshake done sender: [2:7574436073432220452:2099] self: [2:7574436073432220478:0] peer: [1:7574436072361564529:0] socket: 326 qp: -1
2025-11-19T14:00:26.642899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436073432220491:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:26.642907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:26.644307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436074482964941:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.644328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:26.644335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436074482964941:2049] [node 2] ICP22 created new session: [6:7574436074482965030:2049]
2025-11-19T14:00:26.644341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436074482965030:2049] [node 2] ICS09 handshake done sender: [6:7574436074482965017:2115] self: [6:7574436074482965023:0] peer: [2:7574436073432220466:0] socket: 358 qp: -1
2025-11-19T14:00:26.644349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436074482965030:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:26.644361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:26.644718Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436074482965029:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.644754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436074482964985:2091] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.648348Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436074482965027:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.648298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436072226724428:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14088
2025-11-19T14:00:26.648379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436074482964942:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:26.648479Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436075318945356:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.648490Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436075318945356:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.649339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436074482965034:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16905
2025-11-19T14:00:26.649436Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436075318945356:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:26.649567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436075318945336:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.649585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:26.649590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436075318945336:2101] [node 6] ICP22 created new session: [3:7574436075318945374:2101]
2025-11-19T14:00:26.649595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436075318945374:2101] [node 6] ICS09 handshake done sender: [3:7574436075318945356:2116] self: [3:7574436075318945341:0] peer: [6:7574436074482965033:0] socket: 366 qp: -1
2025-11-19T14:00:26.649600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436075318945374:2101] [node 6] ICS10 traffic start
2025-11-19T14:00:26.649607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:26.650159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436073432220464:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.650179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:26.650184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436073432220464:2106] [node 6] ICP22 created new session: [2:7574436073432220494:2106]
2025-11-19T14:00:26.650190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436073432220494:2106] [node 6] ICS09 handshake done sender: [2:7574436073432220474:2113] self: [2:7574436073432220466:0] peer: [6:7574436074482965023:0] socket: 355 qp: -1
2025-11-19T14:00:26.650195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436073432220494:2106] [node 6] ICS10 traffic start
2025-11-19T14:00:26.650200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.650522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436075318945376:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30453
2025-11-19T14:00:26.652461Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436072361564538:2109] [node 4] ICP35 event processing took too much time PendingConnection:278(18846994ns) {HandleHandshakeStatus:329(18846135ns) {DropIncomingHandshake:441(108ns) DropOutgoingHandshake:456(393ns) SwitchToState:178(203ns) ProcessPendingSessionEvents:495(18741218ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18719722ns max# 18658173ns}}}
2025-11-19T14:00:26.652669Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436072226724410:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:26.652783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436072226724333:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.652809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:26.652815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436072226724333:2048] [node 1] ICP22 created new session: [4:7574436072226724429:2048]
2025-11-19T14:00:26.652821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436072226724429:2048] [node 1] ICS09 handshake done sender: [4:7574436072226724410:2116] self: [4:7574436072226724425:0] peer: [1:7574436072361564539:0] socket: 361 qp: -1
2025-11-19T14:00:26.652827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436072226724429:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:26.652833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:26.654449Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436073432220476:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:26.654725Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436073084968006:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(28163678ns) {HandleHandshakeStatus:329(28163059ns) {DropIncomingHandshake:441(225ns) DropOutgoingHandshake:456(1843ns) SwitchToState:178(70ns) ProcessPendingSessionEvents:495(18672467ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18651001ns max# 18587191ns}}}
2025-11-19T14:00:26.655840Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436073084968077:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.655853Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436073084968077:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:26.660521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436073432220399:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.660541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:26.660547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436073432220399:2049] [node 3] ICP22 created new session: [2:7574436073432220497:2049]
2025-11-19T14:00:26.660552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436073432220497:2049] [node 3] ICS09 handshake done sender: [2:7574436073432220476:2114] self: [2:7574436073432220467:0] peer: [3:7574436075318945372:0] socket: 370 qp: -1
2025-11-19T14:00:26.660560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436073432220497:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:26.660566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:26.660688Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436075318945365:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:26.660341Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436072361564558:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.660362Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436072361564558:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.669461Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436073084968083:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.669510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436073084968070:2102] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:26.667852Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436074482965027:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:26.669682Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436072226724404:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.669691Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436072226724404:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.670318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436073084968099:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29542
2025-11-19T14:00:26.670727Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436072226724404:2112] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:26.670007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436075318945276:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.670032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:26.670040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436075318945276:2049] [node 2] ICP22 created new session: [3:7574436075318945377:2049]
2025-11-19T14:00:26.670048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436075318945377:2049] [node 2] ICS09 handshake done sender: [3:7574436075318945365:2120] self: [3:7574436075318945372:0] peer: [2:7574436073432220467:0] socket: 378 qp: -1
2025-11-19T14:00:26.670054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436075318945377:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:26.670064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:26.669611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436073432220496:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33008
2025-11-19T14:00:26.670935Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436073084968083:2116] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:26.671009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436073084968070:2102] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.671023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:26.671029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436073084968070:2102] [node 4] ICP22 created new session: [5:7574436073084968100:2102]
2025-11-19T14:00:26.671034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436073084968100:2102] [node 4] ICS09 handshake done sender: [5:7574436073084968083:2116] self: [5:7574436073084968098:0] peer: [4:7574436072226724403:0] socket: 360 qp: -1
2025-11-19T14:00:26.671040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436073084968100:2102] [node 4] ICS10 traffic start
2025-11-19T14:00:26.671047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:26.670958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436074482964942:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.670974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:26.670979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436074482964942:2050] [node 3] ICP22 created new session: [6:7574436074482965035:2050]
2025-11-19T14:00:26.670983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436074482965035:2050] [node 3] ICS09 handshake done sender: [6:7574436074482965027:2119] self: [6:7574436074482965033:0] peer: [3:7574436075318945341:0] socket: 385 qp: -1
2025-11-19T14:00:26.670988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436074482965035:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:26.671001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:26.672580Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574436074482964941:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(28171041ns) {HandleHandshakeStatus:329(28169863ns) {DropIncomingHandshake:441(357ns) DropOutgoingHandshake:456(4848ns) SwitchToState:178(236ns) ProcessPendingSessionEvents:495(16850292ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16834390ns max# 16773242ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.678034Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436074482965026:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.678140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436074482964992:2095] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:26.678164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436074482965037:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16905
2025-11-19T14:00:26.678306Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436072226724422:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:26.678318Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436072226724422:2121] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:26.679123Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436072226724422:2121] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:26.679209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436072226724394:2104] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.679226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:26.679233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436072226724394:2104] [node 6] ICP22 created new session: [4:7574436072226724433:2104]
2025-11-19T14:00:26.679239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436072226724433:2104] [node 6] ICS09 handshake done sender: [4:7574436072226724422:2121] self: [4:7574436072226724413:0] peer: [6:7574436074482965038:0] socket: 377 qp: -1
2025-11-19T14:00:26.679244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436072226724433:2104] [node 6] ICS10 traffic start
2025-11-19T14:00:26.679252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:26.679633Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436072361564558:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.679732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436074482965039:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16905
2025-11-19T14:00:26.679965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436072361564536:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.679984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:26.679991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436072361564536:2108] [node 5] ICP22 created new session: [1:7574436072361564569:2108]
2025-11-19T14:00:26.679997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436072361564569:2108] [node 5] ICS09 handshake done sender: [1:7574436072361564558:2117] self: [1:7574436072361564550:0] peer: [5:7574436073084968093:0] socket: 364 qp: -1
2025-11-19T14:00:26.680002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436072361564569:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:26.680009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:26.680545Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436073084968077:2110] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:26.679884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436072226724400:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.679905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:26.679912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436072226724400:2110] [node 5] ICP22 created new session: [4:7574436072226724435:2110]
2025-11-19T14:00:26.679918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436072226724435:2110] [node 5] ICS09 handshake done sender: [4:7574436072226724404:2112] self: [4:7574436072226724403:0] peer: [5:7574436073084968098:0] socket: 347 qp: -1
2025-11-19T14:00:26.679926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436072226724435:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:26.679932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:26.680849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436073084968047:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:26.680866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:26.680870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436073084968047:2089] [node 6] ICP22 created new session: [5:7574436073084968104:2089]
2025-11-19T14:00:26.680875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436073084968104:2089] [node 6] ICS09 handshake done sender: [5:7574436073084968077:2110] self: [5:7574436073084968054:0] peer: [6:7574436074482965032:0] socket: 339 qp: -1
2025-11-19T14:00:26.680882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436073084968104:2089] [node 6] ICS10 traffic start
2025-11-19T14:00:26.680886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:26.681324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436073084968103:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29542
2025-11-19T14:00:26.681504Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436073084968004:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14314) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436073084968087:2118] held: no
2025-11-19T14:00:26.681597Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436073084968087:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:26.682158Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436074482965026:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:26.682285Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436074482965029:2121] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:26.682345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436074482964992:2095] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.682358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:26.682364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436074482964992:2095] [node 4] ICP22 created new session: [6:7574436074482965040:2095]
2025-11-19T14:00:26.682370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436074482965040:2095] [node 4] ICS09 handshake done sender: [6:7574436074482965026:2118] self: [6:7574436074482965038:0] peer: [4:7574436072226724413:0] socket: 383 qp: -1
2025-11-19T14:00:26.682375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436074482965040:2095] [node 4] ICS10 traffic start
2025-11-19T14:00:26.682380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:26.683602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436074482964985:2091] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.683625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:26.683630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436074482964985:2091] [node 5] ICP22 created new session: [6:7574436074482965042:2091]
2025-11-19T14:00:26.683636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436074482965042:2091] [node 5] ICS09 handshake done sender: [6:7574436074482965029:2121] self: [6:7574436074482965032:0] peer: [5:7574436073084968054:0] socket: 390 qp: -1
2025-11-19T14:00:26.683641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436074482965042:2091] [node 5] ICS10 traffic start
2025-11-19T14:00:26.683647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:26.683949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436073084968004:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:26.683968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:26.683973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436073084968004:2048] [node 1] ICP22 created new session: [5:7574436073084968106:2048]
2025-11-19T14:00:26.683977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436073084968106:2048] [node 1] ICS09 handshake done sender: [5:7574436073084968087:2118] self: [5:7574436073084968093:0] peer: [1:7574436072361564550:0] socket: 367 qp: -1
2025-11-19T14:00:26.683983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436073084968106:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:26.683989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:26.684558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436073432220399:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:26.707431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:26.707526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436074482965020:2048] [node 1] ICS07 socket disconnect 351 reason# EndOfStream
2025-11-19T14:00:26.707533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436074482965020:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.707579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:26.707600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436074482965020:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:26.707819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:26.707891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574436075318945360:2048] [node 1] ICS07 socket disconnect 330 reason# EndOfStream
2025-11-19T14:00:26.707897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436075318945360:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.707928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:26.707948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436075318945360:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:26.708103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:26.708171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436073084968106:2048] [node 1] ICS01 socket: 367 reason# EndOfStream
2025-11-19T14:00:26.708177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436073084968004:2048] [node 1] ICP30 unregister session Session# [5:7574436073084968106:2048] VirtualId# [5:7574436073084968093:0]
2025-11-19T14:00:26.708182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436073084968106:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.708213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:26.708234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:26.710888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:26.710963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436072226724429:2048] [node 1] ICS01 socket: 361 reason# EndOfStream
2025-11-19T14:00:26.710970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436072226724333:2048] [node 1] ICP30 unregister session Session# [4:7574436072226724429:2048] VirtualId# [4:7574436072226724425:0]
2025-11-19T14:00:26.710974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436072226724429:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.711027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:26.711048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:26.708774Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436075318945275:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14314) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:26.708788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436075318945360:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:26.708792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436075318945275:2048] [node 1] ICP30 unregister session Session# [3:7574436075318945360:2048] VirtualId# [3:7574436075318945355:0]
2025-11-19T14:00:26.708829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:26.708834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436075318945275:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14314) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:26.708837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14314) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:26.716348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436074482964940:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14314) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:26.716366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436074482965020:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:26.716371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436074482964940:2048] [node 1] ICP30 unregister session Session# [6:7574436074482965020:2048] VirtualId# [6:7574436074482965018:0]
2025-11-19T14:00:26.716415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:26.716420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436074482964940:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14314) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:26.716424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14314) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:26.724189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574436075318945275:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:26.724312Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:26.724389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436075318945377:2049] [node 2] ICS01 socket: 378 reason# EndOfStream
2025-11-19T14:00:26.724397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436075318945276:2049] [node 2] ICP30 unregister session Session# [3:7574436075318945377:2049] VirtualId# [3:7574436075318945372:0]
2025-11-19T14:00:26.724403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436075318945377:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.724447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:26.724469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:26.730003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436074482964940:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:26.730046Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:26.730131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436074482965030:2049] [node 2] ICS07 socket disconnect 358 reason# EndOfStream
2025-11-19T14:00:26.730138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436074482965030:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.730199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:26.730221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436074482965030:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:26.734364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:26.734459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436072226724419:2049] [node 2] ICS01 socket: 342 reason# EndOfStream
2025-11-19T14:00:26.734468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436072226724334:2049] [node 2] ICP30 unregister session Session# [4:7574436072226724419:2049] VirtualId# [4:7574436072226724414:0]
2025-11-19T14:00:26.734475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436072226724419:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.734535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:26.734564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:26.746971Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:26.747067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436073084968095:2049] [node 2] ICS01 socket: 359 reason# EndOfStream
2025-11-19T14:00:26.747081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436073084968005:2049] [node 2] ICP30 unregister session Session# [5:7574436073084968095:2049] VirtualId# [5:7574436073084968085:0]
2025-11-19T14:00:26.747087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436073084968095:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.747163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:26.747192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:26.751673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436074482964941:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61568) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:26.751698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436074482965030:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:26.751704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436074482964941:2049] [node 2] ICP30 unregister session Session# [6:7574436074482965030:2049] VirtualId# [6:7574436074482965023:0]
2025-11-19T14:00:26.751762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:26.751768Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436074482964941:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61568) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:26.751771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61568) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:26.762520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436074482964941:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:26.772571Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:26.772658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574436073084968091:2050] [node 3] ICS07 socket disconnect 329 reason# EndOfStream
2025-11-19T14:00:26.772663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436073084968091:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.772724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:26.772749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574436073084968091:2050] [node 3] ICS15 start handshake
2025-11-19T14:00:26.771083Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:26.771176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436074482965035:2050] [node 3] ICS01 socket: 385 reason# EndOfStream
2025-11-19T14:00:26.771186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436074482964942:2050] [node 3] ICP30 unregister session Session# [6:7574436074482965035:2050] VirtualId# [6:7574436074482965033:0]
2025-11-19T14:00:26.771194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436074482965035:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.771260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:26.771286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:26.781160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436073084968006:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62838) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:26.781195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436073084968091:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:26.781202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436073084968006:2050] [node 3] ICP30 unregister session Session# [5:7574436073084968091:2050] VirtualId# [5:7574436073084968084:0]
2025-11-19T14:00:26.781245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:26.781254Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574436073084968006:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62838) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:26.781257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62838) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:26.793509Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:26.793612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436074482965040:2095] [node 4] ICS01 socket: 383 reason# EndOfStream
2025-11-19T14:00:26.793622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436074482964992:2095] [node 4] ICP30 unregister session Session# [6:7574436074482965040:2095] VirtualId# [6:7574436074482965038:0]
2025-11-19T14:00:26.793630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436074482965040:2095] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:26.793701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:26.793729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:26.933293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436073181093028:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:26.933316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436073181093029:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:26.996812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436072705356988:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:26.996824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436072705356989:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:27.076339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436080282905251:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:27.076368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436080282905252:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:27.128115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436076861849399:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:27.128129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436076861849400:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:27.128134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436076861849401:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:27.180587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436078196356557:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:27.180604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436078196356558:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:27.180610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436078196356559:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:27.251644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436079197282049:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:27.251659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436079197282050:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:27.251664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436079197282051:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:27.304307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436076861849428:2075] [node 6] ICP01 ready to work
2025-11-19T14:00:27.305404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436077476060387:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:27.304347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436079197282062:2059] [node 4] ICP01 ready to work
2025-11-19T14:00:27.320500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436077000324346:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:27.322054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436078196356557:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17343838ns) {Configure:104(17342913ns) {ProcessConfigured:135(17046318ns) {StartInitialHandshake:157(17046030ns) {DropHandshakes:469(366ns) {DropIncomingHandshake:441(66ns) DropOutgoingHandshake:456(29ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:476(1190ns) {SwitchToState:178(581ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.322791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436080282905312:2107] [node 6] ICP01 ready to work
2025-11-19T14:00:27.322251Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436072705356988:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(17532683ns) {Configure:104(17532078ns) {ProcessConfigured:135(16909140ns) {StartInitialHandshake:157(16908709ns) {DropHandshakes:469(314ns) {DropIncomingHandshake:441(25ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(253ns) PrepareNewSessionHandshake:476(514ns) {SwitchToState:178(302ns)}}}}}
2025-11-19T14:00:27.322537Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436073181093028:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17079475ns) {Configure:104(17079151ns) {ProcessConfigured:135(15480014ns) {StartInitialHandshake:157(15479681ns) {DropHandshakes:469(809ns) {DropIncomingHandshake:441(155ns) DropOutgoingHandshake:456(57ns)} GenerateSessionVirtualId:988(318ns) PrepareNewSessionHandshake:476(972ns) {SwitchToState:178(129ns)}}}}}
2025-11-19T14:00:27.322800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436077476060387:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(17068914ns) {Configure:104(17063497ns) {ProcessConfigured:135(15530660ns) {StartInitialHandshake:157(15529944ns) {DropHandshakes:469(317ns) {DropIncomingHandshake:441(94ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(284ns) PrepareNewSessionHandshake:476(694ns) {SwitchToState:178(486ns)}}}}}
2025-11-19T14:00:27.322929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436077476060396:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61014
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.325963Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436076861849401:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21940716ns) {Configure:104(21939181ns) {ProcessConfigured:135(18247603ns) {StartInitialHandshake:157(18246482ns) {DropHandshakes:469(405ns) {DropIncomingHandshake:441(32ns) DropOutgoingHandshake:456(28ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:476(655ns) {SwitchToState:178(350ns)}}}}}
2025-11-19T14:00:27.326251Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436076861849400:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(22209728ns) {Configure:104(22203481ns) {ProcessConfigured:135(18250098ns) {StartInitialHandshake:157(18249321ns) {DropHandshakes:469(285ns) {DropIncomingHandshake:441(24ns) DropOutgoingHandshake:456(18ns)} GenerateSessionVirtualId:988(184ns) PrepareNewSessionHandshake:476(894ns) {SwitchToState:178(670ns)}}}}}
2025-11-19T14:00:27.325968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436077000324353:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29015
2025-11-19T14:00:27.332941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436078196356589:2075] [node 4] ICP01 ready to work
2025-11-19T14:00:27.333662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436078196356601:2085] [node 6] ICP01 ready to work
2025-11-19T14:00:27.335822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436077476060386:2107] [node 6] ICP01 ready to work
2025-11-19T14:00:27.336077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436079197282084:2061] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.345244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436079197282121:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3104
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.348216Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436077000324353:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.348290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436072705356988:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:27.348711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436076861849472:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28750
2025-11-19T14:00:27.348763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436076861849473:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28750
2025-11-19T14:00:27.348773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436076861849474:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28750
2025-11-19T14:00:27.351913Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436076861849472:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.355516Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436079197282050:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(27507833ns) {Configure:104(27506596ns) {ProcessConfigured:135(27150425ns) {StartInitialHandshake:157(27149271ns) {DropHandshakes:469(584ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:476(971ns) {SwitchToState:178(355ns)}}}}}
2025-11-19T14:00:27.356527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436076861849399:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(27952880ns) {Configure:104(27952077ns) {ProcessConfigured:135(27577903ns) {StartInitialHandshake:157(27577497ns) {DropHandshakes:469(421ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:476(1407ns) {SwitchToState:178(1125ns)}}}}}
2025-11-19T14:00:27.357080Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436077476060393:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.357094Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436077476060393:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.360238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436077000324361:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29015
2025-11-19T14:00:27.360262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436077000324362:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29015
2025-11-19T14:00:27.360273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436077000324363:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29015
2025-11-19T14:00:27.363963Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436076861849474:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.364244Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436077476060393:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:27.366597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436077000324356:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.368045Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436077000324353:2111] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:27.367949Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436077476060396:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.368030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436077476060400:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61014
2025-11-19T14:00:27.368046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436077476060401:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61014
2025-11-19T14:00:27.368149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436072705356988:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.368169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:27.368178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436072705356988:2048] [node 1] ICP22 created new session: [2:7574436077000324366:2048]
2025-11-19T14:00:27.368194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436077000324366:2048] [node 1] ICS09 handshake done sender: [2:7574436077000324353:2111] self: [2:7574436077000324359:0] peer: [1:7574436077476060388:0] socket: 336 qp: -1
2025-11-19T14:00:27.368202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436077000324366:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:27.368222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:27.367515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436078196356589:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(34446170ns) {Configure:104(34445239ns) {ProcessConfigured:135(34083850ns) {StartInitialHandshake:157(34083312ns) {DropHandshakes:469(456ns) {DropIncomingHandshake:441(117ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:476(1520ns) {SwitchToState:178(978ns)}}}}}
2025-11-19T14:00:27.368211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436078196356601:2085] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(34457004ns) {Configure:104(34456537ns) {ProcessConfigured:135(34221374ns) {StartInitialHandshake:157(34220583ns) {DropHandshakes:469(405ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(275ns) PrepareNewSessionHandshake:476(981ns) {SwitchToState:178(453ns)}}}}}
2025-11-19T14:00:27.370696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436073181093028:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.370721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:27.370728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436073181093028:2048] [node 2] ICP22 created new session: [1:7574436077476060403:2048]
2025-11-19T14:00:27.370735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436077476060403:2048] [node 2] ICS09 handshake done sender: [1:7574436077476060393:2110] self: [1:7574436077476060388:0] peer: [2:7574436077000324359:0] socket: 333 qp: -1
2025-11-19T14:00:27.370741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436077476060403:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:27.370760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.371895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436079197282051:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(59824180ns) {Configure:104(59823437ns) {ProcessConfigured:135(59422962ns) {StartInitialHandshake:157(59422557ns) {DropHandshakes:469(883ns) {DropIncomingHandshake:441(116ns) DropOutgoingHandshake:456(181ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:476(1122ns) {SwitchToState:178(577ns)}}}}}
2025-11-19T14:00:27.373042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436076861849428:2075] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(68664529ns) {Configure:104(68663878ns) {ProcessConfigured:135(64933468ns) {StartInitialHandshake:157(64932752ns) {DropHandshakes:469(305ns) {DropIncomingHandshake:441(21ns) DropOutgoingHandshake:456(13ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:476(923ns) {SwitchToState:178(451ns)}}}}}
2025-11-19T14:00:27.373170Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436076861849473:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.373297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436076861849479:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28750
2025-11-19T14:00:27.373309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436076861849399:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.373511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436077476060386:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(37555252ns) {Configure:104(37554683ns) {ProcessConfigured:135(37219502ns) {StartInitialHandshake:157(37219078ns) {DropHandshakes:469(518ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(61ns)} GenerateSessionVirtualId:988(252ns) PrepareNewSessionHandshake:476(741ns) {SwitchToState:178(201ns)}}}}}
2025-11-19T14:00:27.373527Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436077476060395:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.373530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436077476060395:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:27.373812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436079197282084:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(37563159ns) {Configure:104(37562377ns) {ProcessConfigured:135(37192409ns) {StartInitialHandshake:157(37191960ns) {DropHandshakes:469(413ns) {DropIncomingHandshake:441(54ns) DropOutgoingHandshake:456(47ns)} GenerateSessionVirtualId:988(370ns) PrepareNewSessionHandshake:476(372ns) {SwitchToState:178(162ns)}}}}}
2025-11-19T14:00:27.371507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436078196356634:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39221
2025-11-19T14:00:27.372994Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436078196356634:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.373091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436080282905251:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(37606427ns) {Configure:104(37605395ns) {ProcessConfigured:135(37250691ns) {StartInitialHandshake:157(37250167ns) {DropHandshakes:469(698ns) {DropIncomingHandshake:441(90ns) DropOutgoingHandshake:456(102ns)} GenerateSessionVirtualId:988(248ns) PrepareNewSessionHandshake:476(1360ns) {SwitchToState:178(748ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.377408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436080282905324:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28217
2025-11-19T14:00:27.377460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436080282905325:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28217
2025-11-19T14:00:27.377474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436080282905326:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28217
2025-11-19T14:00:27.377517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436080282905327:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28217
2025-11-19T14:00:27.377756Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436080282905326:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.377790Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436080282905314:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.377793Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436080282905314:2108] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:27.378081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436080282905252:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(38814126ns) {Configure:104(38813251ns) {ProcessConfigured:135(38474868ns) {StartInitialHandshake:157(38474272ns) {DropHandshakes:469(536ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(254ns) PrepareNewSessionHandshake:476(546ns) {SwitchToState:178(33ns)}}}}}
2025-11-19T14:00:27.378162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436080282905252:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:27.375913Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436079197282121:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.377276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436079197282051:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:27.380195Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436080282905324:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.381525Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436077000324348:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.381549Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436077000324348:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:27.381996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436077476060406:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61014
2025-11-19T14:00:27.382469Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436077000324358:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.382481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436077000324358:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:27.383859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436076861849400:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:27.383903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436076861849400:2049] [node 2] ICP35 event processing took too much time PendingConnection:278(31935555ns) {IncomingHandshake:261(31934715ns) {DropIncomingHandshake:441(282ns) PrepareNewSessionHandshake:476(781ns) {SwitchToState:178(188ns)} GenerateSessionVirtualId:988(481ns) IssueIncomingHandshakeReply:193(31928511ns) {SwitchToState:178(336ns)}}}
2025-11-19T14:00:27.381612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436080282905317:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:27.384093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436080282905320:2110] [node 5] ICP01 ready to work
2025-11-19T14:00:27.383885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436077476060392:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.385276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436073181093029:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(69248505ns) {Configure:104(69247558ns) {ProcessConfigured:135(68922100ns) {StartInitialHandshake:157(68921693ns) {DropHandshakes:469(853ns) {DropIncomingHandshake:441(134ns) DropOutgoingHandshake:456(131ns)} GenerateSessionVirtualId:988(298ns) PrepareNewSessionHandshake:476(573ns) {SwitchToState:178(280ns)}}}}}
2025-11-19T14:00:27.383877Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436077000324362:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.384689Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436078196356558:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(52831843ns) {Configure:104(52830290ns) {ProcessConfigured:135(52415179ns) {StartInitialHandshake:157(52414826ns) {DropHandshakes:469(557ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(275ns) PrepareNewSessionHandshake:476(902ns) {SwitchToState:178(773ns)}}}}}
2025-11-19T14:00:27.384749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436078196356558:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:27.384893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436078196356637:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39221
2025-11-19T14:00:27.388231Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436080282905327:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.390416Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436077476060406:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.390510Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436077476060395:2111] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:27.390947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436077476060387:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.390963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:27.390972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436077476060387:2108] [node 4] ICP22 created new session: [1:7574436077476060414:2108]
2025-11-19T14:00:27.390979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436077476060414:2108] [node 4] ICS09 handshake done sender: [1:7574436077476060395:2111] self: [1:7574436077476060389:0] peer: [4:7574436076861849481:0] socket: 337 qp: -1
2025-11-19T14:00:27.390986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436077476060414:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:27.391008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:27.391968Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436080282905314:2108] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:27.392065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436080282905312:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.392080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:27.392085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436080282905312:2107] [node 6] ICP22 created new session: [3:7574436080282905336:2107]
2025-11-19T14:00:27.392092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436080282905336:2107] [node 6] ICS09 handshake done sender: [3:7574436080282905314:2108] self: [3:7574436080282905313:0] peer: [6:7574436079197282127:0] socket: 335 qp: -1
2025-11-19T14:00:27.392097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436080282905336:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:27.392113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:27.391879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436077000324355:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:27.391903Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436077000324365:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.391917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436077000324365:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:27.392337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436076861849454:2083] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.394596Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436077000324348:2108] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:27.393848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436079197282129:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3104
2025-11-19T14:00:27.393911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436079197282130:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3104
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.396466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436078196356638:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39221
2025-11-19T14:00:27.396783Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436078196356637:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.396816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436078196356557:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:27.397026Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436077476060409:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.397035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436077476060409:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.397760Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436077476060409:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:27.398749Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436077000324361:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.398829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436077476060392:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.398841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:27.398846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436077476060392:2109] [node 5] ICP22 created new session: [1:7574436077476060418:2109]
2025-11-19T14:00:27.398850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436077476060418:2109] [node 5] ICS09 handshake done sender: [1:7574436077476060409:2117] self: [1:7574436077476060408:0] peer: [5:7574436078196356639:0] socket: 368 qp: -1
2025-11-19T14:00:27.398855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436077476060418:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:27.398863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:27.399408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436079197282049:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(54289530ns) {Configure:104(54288740ns) {ProcessConfigured:135(53860004ns) {StartInitialHandshake:157(53859258ns) {DropHandshakes:469(819ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(342ns) PrepareNewSessionHandshake:476(660ns) {SwitchToState:178(357ns)}}}}}
2025-11-19T14:00:27.399695Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436077000324358:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.399928Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436079197282129:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.400010Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436079197282121:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:27.400040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436079197282049:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:27.400058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436079197282131:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3104
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.400218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436079197282051:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.400234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:27.400245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436079197282051:2050] [node 3] ICP22 created new session: [6:7574436079197282133:2050]
2025-11-19T14:00:27.400250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436079197282133:2050] [node 3] ICS09 handshake done sender: [6:7574436079197282121:2111] self: [6:7574436079197282127:0] peer: [3:7574436080282905313:0] socket: 340 qp: -1
2025-11-19T14:00:27.400255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436079197282133:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:27.400287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:27.403174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436077000324346:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.403194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:27.403200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436077000324346:2107] [node 4] ICP22 created new session: [2:7574436077000324373:2107]
2025-11-19T14:00:27.403206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436077000324373:2107] [node 4] ICS09 handshake done sender: [2:7574436077000324348:2108] self: [2:7574436077000324347:0] peer: [4:7574436076861849475:0] socket: 330 qp: -1
2025-11-19T14:00:27.403211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436077000324373:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:27.403218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:27.403690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436072705356989:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.403702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:27.403710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436072705356989:2049] [node 3] ICP22 created new session: [2:7574436077000324375:2049]
2025-11-19T14:00:27.403715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436077000324375:2049] [node 3] ICS09 handshake done sender: [2:7574436077000324358:2113] self: [2:7574436077000324357:0] peer: [3:7574436080282905329:0] socket: 347 qp: -1
2025-11-19T14:00:27.403719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436077000324375:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:27.403725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:27.397615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436078196356640:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39221
2025-11-19T14:00:27.397973Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436078196356637:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:27.398105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436078196356557:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.398120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:27.398126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436078196356557:2048] [node 1] ICP22 created new session: [5:7574436078196356641:2048]
2025-11-19T14:00:27.398132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436078196356641:2048] [node 1] ICS09 handshake done sender: [5:7574436078196356637:2116] self: [5:7574436078196356639:0] peer: [1:7574436077476060408:0] socket: 369 qp: -1
2025-11-19T14:00:27.398138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436078196356641:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:27.398158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:27.401176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436078196356644:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39221
2025-11-19T14:00:27.403086Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436078196356644:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.404089Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436077476060401:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.404877Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436077000324365:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:27.405645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436077000324356:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.405659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:27.405668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436077000324356:2112] [node 5] ICP22 created new session: [2:7574436077000324378:2112]
2025-11-19T14:00:27.405672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436077000324378:2112] [node 5] ICS09 handshake done sender: [2:7574436077000324365:2117] self: [2:7574436077000324364:0] peer: [5:7574436078196356636:0] socket: 357 qp: -1
2025-11-19T14:00:27.405677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436077000324378:2112] [node 5] ICS10 traffic start
2025-11-19T14:00:27.405684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:27.404758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436078196356646:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39221
2025-11-19T14:00:27.405067Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436078196356634:2114] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:27.405159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436078196356558:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.405169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:27.405174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436078196356558:2049] [node 2] ICP22 created new session: [5:7574436078196356647:2049]
2025-11-19T14:00:27.405180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436078196356647:2049] [node 2] ICS09 handshake done sender: [5:7574436078196356634:2114] self: [5:7574436078196356636:0] peer: [2:7574436077000324364:0] socket: 358 qp: -1
2025-11-19T14:00:27.405186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436078196356647:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:27.405204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:27.406806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436076861849400:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436076861849472:2112] held: no
2025-11-19T14:00:27.409931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436077476060405:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.409948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436077476060405:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.410733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436077000324380:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29015
2025-11-19T14:00:27.409954Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436078196356638:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.409999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436078196356559:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:27.410893Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436077476060387:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(19859047ns) {HandleHandshakeStatus:329(19858175ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(350ns) SwitchToState:178(108ns) ProcessPendingSessionEvents:495(11683359ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11658887ns max# 11587631ns}}}
2025-11-19T14:00:27.410943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436077476060387:2108] [node 4] ICP18 handshake (actor [1:7574436077476060401:2114]) is too old
2025-11-19T14:00:27.411596Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436080282905312:2107] [node 6] ICP35 event processing took too much time PendingConnection:278(19442112ns) {HandleHandshakeStatus:329(19441630ns) {DropIncomingHandshake:441(3772ns) DropOutgoingHandshake:456(197ns) SwitchToState:178(73ns) ProcessPendingSessionEvents:495(12257307ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12235794ns max# 12171608ns}}}
2025-11-19T14:00:27.412218Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436076861849399:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22254) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436076861849473:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.416211Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436077000324380:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.416279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436072705356989:2049] [node 3] ICP18 handshake (actor [2:7574436077000324380:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.415909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436076861849485:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28750
2025-11-19T14:00:27.419806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436076861849486:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28750
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.420156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436080282905341:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28217
2025-11-19T14:00:27.420215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436080282905342:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28217
2025-11-19T14:00:27.420688Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436076861849472:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:27.420867Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436076861849473:2113] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:27.420964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436076861849400:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.420983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:27.420991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436076861849400:2049] [node 2] ICP22 created new session: [4:7574436076861849487:2049]
2025-11-19T14:00:27.420997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436076861849487:2049] [node 2] ICS09 handshake done sender: [4:7574436076861849472:2112] self: [4:7574436076861849475:0] peer: [2:7574436077000324347:0] socket: 331 qp: -1
2025-11-19T14:00:27.421005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436076861849487:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:27.421014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:27.421345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436076861849399:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.421358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:27.421363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436076861849399:2048] [node 1] ICP22 created new session: [4:7574436076861849489:2048]
2025-11-19T14:00:27.421368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436076861849489:2048] [node 1] ICS09 handshake done sender: [4:7574436076861849473:2113] self: [4:7574436076861849481:0] peer: [1:7574436077476060389:0] socket: 348 qp: -1
2025-11-19T14:00:27.421373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436076861849489:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:27.421391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.422715Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436080282905333:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.422730Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436080282905333:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:27.422449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436079197282136:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3104
2025-11-19T14:00:27.424313Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436080282905341:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.426125Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436077476060411:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.426137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436077476060411:2118] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.424938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436080282905251:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:27.428321Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436076861849479:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.430044Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436077476060411:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:27.430200Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436077476060405:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:27.427425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436079197282137:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3104
2025-11-19T14:00:27.429438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436079197282138:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3104
2025-11-19T14:00:27.430422Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436079197282129:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:27.430526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436079197282049:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.430549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:27.430564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436079197282049:2048] [node 1] ICP22 created new session: [6:7574436079197282139:2048]
2025-11-19T14:00:27.430571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436079197282139:2048] [node 1] ICS09 handshake done sender: [6:7574436079197282129:2116] self: [6:7574436079197282132:0] peer: [1:7574436077476060398:0] socket: 362 qp: -1
2025-11-19T14:00:27.430578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436079197282139:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:27.430590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:27.431148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436073181093029:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.428942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436077000324355:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(36830047ns) {Configure:104(36829394ns) {ProcessConfigured:135(29744467ns) {StartInitialHandshake:157(29731942ns) {DropHandshakes:469(397ns) {DropIncomingHandshake:441(29ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(342ns) PrepareNewSessionHandshake:476(1107ns) {SwitchToState:178(685ns)}} IncomingHandshake:261(5110ns) {DropIncomingHandshake:441(56ns) PrepareNewSessionHandshake:476(253ns) {SwitchToState:178(171ns)} GenerateSessionVirtualId:988(210ns) IssueIncomingHandshakeReply:193(135ns)}}}}
2025-11-19T14:00:27.431165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:27.431172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436073181093029:2049] [node 3] ICP22 created new session: [1:7574436077476060422:2049]
2025-11-19T14:00:27.431177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436077476060422:2049] [node 3] ICS09 handshake done sender: [1:7574436077476060411:2118] self: [1:7574436077476060391:0] peer: [3:7574436080282905343:0] socket: 373 qp: -1
2025-11-19T14:00:27.431182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436077476060422:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:27.431188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:27.431635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436077476060386:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.431645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:27.431650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436077476060386:2107] [node 6] ICP22 created new session: [1:7574436077476060424:2107]
2025-11-19T14:00:27.431655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436077476060424:2107] [node 6] ICS09 handshake done sender: [1:7574436077476060405:2115] self: [1:7574436077476060398:0] peer: [6:7574436079197282132:0] socket: 361 qp: -1
2025-11-19T14:00:27.431660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436077476060424:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:27.431665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:27.431812Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436079197282131:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.431885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436079197282084:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:27.432149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436079197282136:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.432193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436079197282062:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.435486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436078196356650:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39221
2025-11-19T14:00:27.435541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436078196356651:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39221
2025-11-19T14:00:27.435717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436080282905252:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436080282905326:2114] held: no
2025-11-19T14:00:27.435893Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436080282905333:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:27.435975Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436080282905325:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.436102Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436080282905326:2114] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:27.435921Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436076861849478:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.435935Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436076861849478:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:27.435808Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436078196356633:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.435830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436078196356633:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:27.436849Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436078196356650:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.438077Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436078196356638:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:27.438184Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436076861849478:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:27.438291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436076861849428:2075] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.438314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:27.438320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436076861849428:2075] [node 6] ICP22 created new session: [4:7574436076861849494:2075]
2025-11-19T14:00:27.438326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436076861849494:2075] [node 6] ICS09 handshake done sender: [4:7574436076861849478:2116] self: [4:7574436076861849434:0] peer: [6:7574436079197282142:0] socket: 332 qp: -1
2025-11-19T14:00:27.438330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436076861849494:2075] [node 6] ICS10 traffic start
2025-11-19T14:00:27.438338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:27.438981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436080282905251:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22254) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436080282905341:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.439394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436076861849496:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28750
2025-11-19T14:00:27.440255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436080282905320:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.440280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:27.440287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436080282905320:2110] [node 5] ICP22 created new session: [3:7574436080282905347:2110]
2025-11-19T14:00:27.440293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436080282905347:2110] [node 5] ICS09 handshake done sender: [3:7574436080282905333:2117] self: [3:7574436080282905332:0] peer: [5:7574436078196356649:0] socket: 376 qp: -1
2025-11-19T14:00:27.440299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436080282905347:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:27.440306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:27.440534Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436079197282062:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436079197282136:2119] held: no
2025-11-19T14:00:27.439810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436078196356589:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:27.439833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436078196356559:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.439846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:27.439853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436078196356559:2050] [node 3] ICP22 created new session: [5:7574436078196356653:2050]
2025-11-19T14:00:27.439859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436078196356653:2050] [node 3] ICS09 handshake done sender: [5:7574436078196356638:2117] self: [5:7574436078196356649:0] peer: [3:7574436080282905332:0] socket: 378 qp: -1
2025-11-19T14:00:27.439868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436078196356653:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:27.439876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:27.441202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436080282905252:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.441221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:27.441229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436080282905252:2049] [node 2] ICP22 created new session: [3:7574436080282905349:2049]
2025-11-19T14:00:27.441245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436080282905349:2049] [node 2] ICS09 handshake done sender: [3:7574436080282905326:2114] self: [3:7574436080282905329:0] peer: [2:7574436077000324357:0] socket: 364 qp: -1
2025-11-19T14:00:27.441251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436080282905349:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:27.441271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:27.444297Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436076861849483:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.444310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436076861849483:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.445415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436078196356655:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39221
2025-11-19T14:00:27.445543Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436076861849483:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:27.445738Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436078196356650:2121] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:27.445814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436078196356589:2075] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.445829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:27.445835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436078196356589:2075] [node 4] ICP22 created new session: [5:7574436078196356656:2075]
2025-11-19T14:00:27.445841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436078196356656:2075] [node 4] ICS09 handshake done sender: [5:7574436078196356650:2121] self: [5:7574436078196356652:0] peer: [4:7574436076861849482:0] socket: 387 qp: -1
2025-11-19T14:00:27.445848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436078196356656:2075] [node 4] ICS10 traffic start
2025-11-19T14:00:27.445856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:27.447004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436076861849454:2083] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.447028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:27.447035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436076861849454:2083] [node 5] ICP22 created new session: [4:7574436076861849498:2083]
2025-11-19T14:00:27.447042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436076861849498:2083] [node 5] ICS09 handshake done sender: [4:7574436076861849483:2118] self: [4:7574436076861849482:0] peer: [5:7574436078196356652:0] socket: 385 qp: -1
2025-11-19T14:00:27.447048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436076861849498:2083] [node 5] ICS10 traffic start
2025-11-19T14:00:27.447060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.452084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436079197282143:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3104
2025-11-19T14:00:27.452282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436079197282144:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3104
2025-11-19T14:00:27.452323Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436079197282136:2119] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:27.452183Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436078196356633:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:27.452481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436078196356601:2085] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.452501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:27.452508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436078196356601:2085] [node 6] ICP22 created new session: [5:7574436078196356659:2085]
2025-11-19T14:00:27.452515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436078196356659:2085] [node 6] ICS09 handshake done sender: [5:7574436078196356633:2113] self: [5:7574436078196356630:0] peer: [6:7574436079197282141:0] socket: 381 qp: -1
2025-11-19T14:00:27.452520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436078196356659:2085] [node 6] ICS10 traffic start
2025-11-19T14:00:27.452530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:27.456816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436079197282084:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13721) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436079197282131:2118] held: no
2025-11-19T14:00:27.457631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436079197282062:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.457646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:27.457656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436079197282062:2059] [node 4] ICP22 created new session: [6:7574436079197282145:2059]
2025-11-19T14:00:27.457662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436079197282145:2059] [node 4] ICS09 handshake done sender: [6:7574436079197282136:2119] self: [6:7574436079197282142:0] peer: [4:7574436076861849434:0] socket: 339 qp: -1
2025-11-19T14:00:27.457670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436079197282145:2059] [node 4] ICS10 traffic start
2025-11-19T14:00:27.457681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:27.457942Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436079197282131:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.462375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436080282905351:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28217
2025-11-19T14:00:27.462501Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436079197282138:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.462523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436079197282084:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.462536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:27.462543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436079197282084:2061] [node 5] ICP22 created new session: [6:7574436079197282147:2061]
2025-11-19T14:00:27.462549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436079197282147:2061] [node 5] ICS09 handshake done sender: [6:7574436079197282131:2118] self: [6:7574436079197282141:0] peer: [5:7574436078196356630:0] socket: 343 qp: -1
2025-11-19T14:00:27.462556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436079197282147:2061] [node 5] ICS10 traffic start
2025-11-19T14:00:27.462566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:27.462838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436079197282050:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:27.468443Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436077000324381:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.468459Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436077000324381:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:27.470659Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436077000324381:2119] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:27.468373Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436080282905341:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:27.469474Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436080282905320:2110] [node 5] ICP35 event processing took too much time PendingConnection:278(29117932ns) {HandleHandshakeStatus:329(29117416ns) {DropIncomingHandshake:441(5568ns) DropOutgoingHandshake:456(136ns) SwitchToState:178(93ns) ProcessPendingSessionEvents:495(27514872ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27491482ns max# 27424220ns}}}
2025-11-19T14:00:27.470245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436080282905251:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.470260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:27.470268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436080282905251:2048] [node 1] ICP22 created new session: [3:7574436080282905352:2048]
2025-11-19T14:00:27.470274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436080282905352:2048] [node 1] ICS09 handshake done sender: [3:7574436080282905341:2119] self: [3:7574436080282905343:0] peer: [1:7574436077476060391:0] socket: 374 qp: -1
2025-11-19T14:00:27.470281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436080282905352:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:27.470292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:27.470765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436077000324355:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.470785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:27.470790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436077000324355:2110] [node 6] ICP22 created new session: [2:7574436077000324385:2110]
2025-11-19T14:00:27.470795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436077000324385:2110] [node 6] ICS09 handshake done sender: [2:7574436077000324381:2119] self: [2:7574436077000324372:0] peer: [6:7574436079197282149:0] socket: 390 qp: -1
2025-11-19T14:00:27.470799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436077000324385:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:27.470808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.472792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436079197282050:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436079197282138:2121] held: no
2025-11-19T14:00:27.473076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436079197282150:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3104
2025-11-19T14:00:27.474184Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436076861849496:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.474254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436076861849401:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:27.474473Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436080282905338:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:27.474484Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436080282905338:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:27.475211Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436080282905338:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:27.475321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436080282905317:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:27.475338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:27.475345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436080282905317:2109] [node 4] ICP22 created new session: [3:7574436080282905355:2109]
2025-11-19T14:00:27.475351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436080282905355:2109] [node 4] ICS09 handshake done sender: [3:7574436080282905338:2118] self: [3:7574436080282905331:0] peer: [4:7574436076861849500:0] socket: 349 qp: -1
2025-11-19T14:00:27.475356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436080282905355:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:27.475363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:27.475861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436076861849501:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28750
2025-11-19T14:00:27.476136Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436076861849496:2121] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:27.476345Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436079197282138:2121] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:27.476430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436076861849401:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.476447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:27.476451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436076861849401:2050] [node 3] ICP22 created new session: [4:7574436076861849502:2050]
2025-11-19T14:00:27.476457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436076861849502:2050] [node 3] ICS09 handshake done sender: [4:7574436076861849496:2121] self: [4:7574436076861849500:0] peer: [3:7574436080282905331:0] socket: 366 qp: -1
2025-11-19T14:00:27.476462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436076861849502:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:27.476474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:27.478302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436079197282050:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:27.478321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:27.478327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436079197282050:2049] [node 2] ICP22 created new session: [6:7574436079197282151:2049]
2025-11-19T14:00:27.478332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436079197282151:2049] [node 2] ICS09 handshake done sender: [6:7574436079197282138:2121] self: [6:7574436079197282149:0] peer: [2:7574436077000324372:0] socket: 391 qp: -1
2025-11-19T14:00:27.478337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436079197282151:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:27.478349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:27.485034Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:27.485117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436080282905352:2048] [node 1] ICS01 socket: 374 reason# EndOfStream
2025-11-19T14:00:27.485125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436080282905251:2048] [node 1] ICP30 unregister session Session# [3:7574436080282905352:2048] VirtualId# [3:7574436080282905343:0]
2025-11-19T14:00:27.485131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436080282905352:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:27.485162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:27.485179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:27.487531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:27.487632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436077000324366:2048] [node 1] ICS01 socket: 336 reason# EndOfStream
2025-11-19T14:00:27.487640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436072705356988:2048] [node 1] ICP30 unregister session Session# [2:7574436077000324366:2048] VirtualId# [2:7574436077000324359:0]
2025-11-19T14:00:27.487646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436077000324366:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:27.487697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:27.487723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:27.488351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:27.488428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436078196356641:2048] [node 1] ICS01 socket: 369 reason# EndOfStream
2025-11-19T14:00:27.488434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436078196356557:2048] [node 1] ICP30 unregister session Session# [5:7574436078196356641:2048] VirtualId# [5:7574436078196356639:0]
2025-11-19T14:00:27.488440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436078196356641:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:27.488483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:27.488503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:27.491976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:27.492087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436076861849489:2048] [node 1] ICS01 socket: 348 reason# EndOfStream
2025-11-19T14:00:27.492106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436076861849399:2048] [node 1] ICP30 unregister session Session# [4:7574436076861849489:2048] VirtualId# [4:7574436076861849481:0]
2025-11-19T14:00:27.492112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436076861849489:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:27.492168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:27.492192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:27.500257Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:27.500372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436079197282139:2048] [node 1] ICS01 socket: 362 reason# EndOfStream
2025-11-19T14:00:27.500381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436079197282049:2048] [node 1] ICP30 unregister session Session# [6:7574436079197282139:2048] VirtualId# [6:7574436079197282132:0]
2025-11-19T14:00:27.500390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436079197282139:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:27.500454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:27.500487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:27.509263Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:27.509355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436080282905349:2049] [node 2] ICS01 socket: 364 reason# EndOfStream
2025-11-19T14:00:27.509366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436080282905252:2049] [node 2] ICP30 unregister session Session# [3:7574436080282905349:2049] VirtualId# [3:7574436080282905329:0]
2025-11-19T14:00:27.509374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436080282905349:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:27.509410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:27.509431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:27.512062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:27.512152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436076861849487:2049] [node 2] ICS01 socket: 331 reason# EndOfStream
2025-11-19T14:00:27.512160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436076861849400:2049] [node 2] ICP30 unregister session Session# [4:7574436076861849487:2049] VirtualId# [4:7574436076861849475:0]
2025-11-19T14:00:27.512166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436076861849487:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:27.512215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:27.512243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:27.516245Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:27.516334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436078196356647:2049] [node 2] ICS01 socket: 358 reason# EndOfStream
2025-11-19T14:00:27.516341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436078196356558:2049] [node 2] ICP30 unregister session Session# [5:7574436078196356647:2049] VirtualId# [5:7574436078196356636:0]
2025-11-19T14:00:27.516347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436078196356647:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:27.516397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:27.516421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:27.521710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:27.521800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436079197282151:2049] [node 2] ICS01 socket: 391 reason# EndOfStream
2025-11-19T14:00:27.521808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436079197282050:2049] [node 2] ICP30 unregister session Session# [6:7574436079197282151:2049] VirtualId# [6:7574436079197282149:0]
2025-11-19T14:00:27.521816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436079197282151:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:27.521869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:27.521897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:27.542999Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:27.543107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436079197282133:2050] [node 3] ICS01 socket: 340 reason# EndOfStream
2025-11-19T14:00:27.543118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436079197282051:2050] [node 3] ICP30 unregister session Session# [6:7574436079197282133:2050] VirtualId# [6:7574436079197282127:0]
2025-11-19T14:00:27.543126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436079197282133:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:27.543186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:27.543215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:27.544193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:27.544271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436078196356653:2050] [node 3] ICS01 socket: 378 reason# EndOfStream
2025-11-19T14:00:27.544282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436078196356559:2050] [node 3] ICP30 unregister session Session# [5:7574436078196356653:2050] VirtualId# [5:7574436078196356649:0]
2025-11-19T14:00:27.544288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436078196356653:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:27.544340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:27.544375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:27.566574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:27.566668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436079197282145:2059] [node 4] ICS01 socket: 339 reason# EndOfStream
2025-11-19T14:00:27.566677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436079197282062:2059] [node 4] ICP30 unregister session Session# [6:7574436079197282145:2059] VirtualId# [6:7574436079197282142:0]
2025-11-19T14:00:27.566683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436079197282145:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:27.566730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:27.566754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:27.738987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436077262852006:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:27.739009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436077262852007:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:27.803931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436077964940943:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:27.803952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436077964940944:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:27.864579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436080111927630:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:27.864592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436080111927631:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:27.931206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436079200898429:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:27.931228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436079200898430:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:27.931232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436079200898431:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:27.968274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436079626819391:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:27.968287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436079626819392:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:27.968291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436079626819393:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:28.055981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436081479956891:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:28.056003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436081479956892:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:28.056008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436081479956893:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:28.120007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436081557819365:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:28.120108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436081557819366:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:28.120102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436083495865785:2104] [node 5] ICP01 ready to work
2025-11-19T14:00:28.120199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436083921786742:2101] [node 4] ICP01 ready to work
2025-11-19T14:00:28.119970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436081479956944:2095] [node 5] ICP01 ready to work
2025-11-19T14:00:28.120086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436081479956946:2098] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.120824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436081557819369:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.122971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436083495865789:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-19T14:00:28.122995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436083495865790:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-19T14:00:28.123007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436083495865791:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-19T14:00:28.121579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436084406894986:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52068
2025-11-19T14:00:28.122368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436084406894987:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52068
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.135456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436082259908299:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44079
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.153132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436083921786756:2105] [node 6] ICP01 ready to work
2025-11-19T14:00:28.154110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436083921786754:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9242
2025-11-19T14:00:28.154126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436083921786755:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9242
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.157038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436084406894993:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:28.157198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436084406894994:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:28.157423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436084406894995:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:28.158781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436084406894991:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52068
2025-11-19T14:00:28.158795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436084406894992:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52068
2025-11-19T14:00:28.159876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436082259908309:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:28.160051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436082259908310:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:28.160204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436082259908311:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.181185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436081557819381:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50707
2025-11-19T14:00:28.181245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436081557819382:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50707
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.185098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436081557819381:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.190044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436083921786765:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9242
2025-11-19T14:00:28.192779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436081479956967:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50491
2025-11-19T14:00:28.202971Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436083495865789:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.202368Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436084406894987:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.202444Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436084406894991:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.202479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436080111927630:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.202493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436080111927631:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.211649Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436081479956967:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.212171Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436081557819368:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.212189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436081557819368:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:28.212485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436084406894992:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.212659Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436083921786765:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.212687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436079626819393:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.212799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436083495865800:2114] [node 6] ICP01 ready to work
2025-11-19T14:00:28.212804Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436084406894997:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.212810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436084406894997:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:28.220057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436082259908303:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.220077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436082259908303:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:28.221213Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436081557819382:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.221537Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436081557819368:2110] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:28.221635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436077262852007:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.221656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:28.221665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436077262852007:2049] [node 3] ICP22 created new session: [1:7574436081557819386:2049]
2025-11-19T14:00:28.221671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436081557819386:2049] [node 3] ICS09 handshake done sender: [1:7574436081557819368:2110] self: [1:7574436081557819367:0] peer: [3:7574436084406895009:0] socket: 341 qp: -1
2025-11-19T14:00:28.221677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436081557819386:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:28.221685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:28.222173Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436082259908303:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.222658Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436083495865791:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.222698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436083495865801:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-19T14:00:28.222746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436083495865802:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-19T14:00:28.222849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436079200898429:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.224067Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436083921786755:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.224110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436083921786742:2101] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.224100Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436082259908299:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.224206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436077964940944:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.224219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:28.224226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436077964940944:2049] [node 3] ICP22 created new session: [2:7574436082259908321:2049]
2025-11-19T14:00:28.224235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436082259908321:2049] [node 3] ICS09 handshake done sender: [2:7574436082259908303:2109] self: [2:7574436082259908302:0] peer: [3:7574436084406895010:0] socket: 352 qp: -1
2025-11-19T14:00:28.224243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436082259908321:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:28.224252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:28.224656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436082259908320:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44079
2025-11-19T14:00:28.227879Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436083495865788:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.227894Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436083495865788:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:28.228297Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436083495865802:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.229544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436081479956970:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50491
2025-11-19T14:00:28.229595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436081479956891:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.229668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436081479956971:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50491
2025-11-19T14:00:28.229829Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436083921786754:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.235981Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436081479956970:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.236041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436081479956944:2095] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.236220Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436083921786760:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.236234Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436083921786760:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:28.237475Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436083921786760:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:28.237612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436083921786756:2105] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.237637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:28.237645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436083921786756:2105] [node 6] ICP22 created new session: [5:7574436083921786772:2105]
2025-11-19T14:00:28.237652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436083921786772:2105] [node 6] ICS09 handshake done sender: [5:7574436083921786760:2114] self: [5:7574436083921786759:0] peer: [6:7574436081479956973:0] socket: 358 qp: -1
2025-11-19T14:00:28.237660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436083921786772:2105] [node 6] ICS10 traffic start
2025-11-19T14:00:28.237670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:28.245053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436080111927630:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5062) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436084406894987:2108] held: no
2025-11-19T14:00:28.245199Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436084406894986:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.245451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436084406895012:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52068
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.246178Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436081479956944:2095] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6692) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436081479956970:2116] held: no
2025-11-19T14:00:28.246321Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436081479956971:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.246432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436081479956974:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50491
2025-11-19T14:00:28.246647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436081479956892:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.246771Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436081479956970:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.246915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436083921786775:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9242
2025-11-19T14:00:28.247103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436081479956944:2095] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.247120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:28.247126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436081479956944:2095] [node 5] ICP22 created new session: [6:7574436081479956977:2095]
2025-11-19T14:00:28.247131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436081479956977:2095] [node 5] ICS09 handshake done sender: [6:7574436081479956970:2116] self: [6:7574436081479956973:0] peer: [5:7574436083921786759:0] socket: 367 qp: -1
2025-11-19T14:00:28.247139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436081479956977:2095] [node 5] ICS10 traffic start
2025-11-19T14:00:28.247146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:28.247672Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436082259908315:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.247686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436082259908315:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.249487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436084406895013:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52068
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.249735Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436081557819371:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.249753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436081557819371:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:28.250954Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436081557819373:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.250960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436081557819373:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:28.251737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436081557819389:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50707
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.252178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436084406895014:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52068
2025-11-19T14:00:28.252266Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436084406894997:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:28.252489Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436083921786765:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:28.253444Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436082259908315:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:28.252624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436079626819393:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.252645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:28.252652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436079626819393:2050] [node 3] ICP22 created new session: [5:7574436083921786776:2050]
2025-11-19T14:00:28.252658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436083921786776:2050] [node 3] ICS09 handshake done sender: [5:7574436083921786765:2116] self: [5:7574436083921786768:0] peer: [3:7574436084406894996:0] socket: 364 qp: -1
2025-11-19T14:00:28.252664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436083921786776:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:28.252674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.254563Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436083495865790:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.254606Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436083495865801:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.259702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436081479956979:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50491
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.259941Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436082259908320:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.259962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436082259908310:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.259977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:28.259986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436082259908310:2111] [node 6] ICP22 created new session: [2:7574436082259908326:2111]
2025-11-19T14:00:28.259994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436082259908326:2111] [node 6] ICS09 handshake done sender: [2:7574436082259908315:2114] self: [2:7574436082259908314:0] peer: [6:7574436081479956976:0] socket: 374 qp: -1
2025-11-19T14:00:28.260002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436082259908326:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:28.260012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:28.260320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436077964940943:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.260892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436084406894993:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.260924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:28.260929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436084406894993:2109] [node 5] ICP22 created new session: [3:7574436084406895017:2109]
2025-11-19T14:00:28.260934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436084406895017:2109] [node 5] ICS09 handshake done sender: [3:7574436084406894997:2114] self: [3:7574436084406894996:0] peer: [5:7574436083921786768:0] socket: 360 qp: -1
2025-11-19T14:00:28.260939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436084406895017:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:28.260946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:28.260990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436081557819390:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50707
2025-11-19T14:00:28.261076Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436081557819371:2111] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:28.260544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436079200898430:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.260641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436079200898431:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.260659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436083495865811:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-19T14:00:28.261769Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436084406894991:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.262258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436083921786778:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9242
2025-11-19T14:00:28.262279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436083921786779:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9242
2025-11-19T14:00:28.262482Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436081557819373:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:28.262597Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436083495865788:2108] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:28.262737Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436081557819375:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.262745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436081557819375:2113] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:28.263296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436081557819365:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.263308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:28.263315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436081557819365:2107] [node 4] ICP22 created new session: [1:7574436081557819393:2107]
2025-11-19T14:00:28.263320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436081557819393:2107] [node 4] ICS09 handshake done sender: [1:7574436081557819371:2111] self: [1:7574436081557819370:0] peer: [4:7574436083495865809:0] socket: 342 qp: -1
2025-11-19T14:00:28.263326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436081557819393:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:28.263334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:28.263581Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436081557819389:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.263587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436081557819366:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.263592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:28.263596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436081557819366:2108] [node 6] ICP22 created new session: [1:7574436081557819395:2108]
2025-11-19T14:00:28.263600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436081557819395:2108] [node 6] ICS09 handshake done sender: [1:7574436081557819373:2112] self: [1:7574436081557819372:0] peer: [6:7574436081479956972:0] socket: 343 qp: -1
2025-11-19T14:00:28.263604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436081557819395:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:28.263608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:28.264477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436083495865785:2104] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.264499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:28.264505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436083495865785:2104] [node 5] ICP22 created new session: [4:7574436083495865815:2104]
2025-11-19T14:00:28.264511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436083495865815:2104] [node 5] ICS09 handshake done sender: [4:7574436083495865788:2108] self: [4:7574436083495865786:0] peer: [5:7574436083921786769:0] socket: 348 qp: -1
2025-11-19T14:00:28.264517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436083495865815:2104] [node 5] ICS10 traffic start
2025-11-19T14:00:28.264524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:28.266388Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436083921786755:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:28.266534Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436084406894987:2108] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:28.266863Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436084406894999:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.266870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436084406894999:2115] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:28.267356Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436084406895014:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.267363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436080111927631:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.267375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:28.267380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436080111927631:2049] [node 2] ICP22 created new session: [3:7574436084406895020:2049]
2025-11-19T14:00:28.267386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436084406895020:2049] [node 2] ICS09 handshake done sender: [3:7574436084406894991:2112] self: [3:7574436084406895010:0] peer: [2:7574436082259908302:0] socket: 355 qp: -1
2025-11-19T14:00:28.267392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436084406895020:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:28.267399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:28.267820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436080111927630:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.267828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:28.267832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436080111927630:2048] [node 1] ICP22 created new session: [3:7574436084406895022:2048]
2025-11-19T14:00:28.267836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436084406895022:2048] [node 1] ICS09 handshake done sender: [3:7574436084406894987:2108] self: [3:7574436084406895009:0] peer: [1:7574436081557819367:0] socket: 344 qp: -1
2025-11-19T14:00:28.267841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436084406895022:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:28.267847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:28.268499Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436083495865791:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.268611Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436082259908313:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.268622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436082259908313:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:28.270694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436082259908330:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44079
2025-11-19T14:00:28.270710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436082259908331:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44079
2025-11-19T14:00:28.270748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436082259908332:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44079
2025-11-19T14:00:28.271070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436079200898429:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.271089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:28.271095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436079200898429:2048] [node 1] ICP22 created new session: [4:7574436083495865819:2048]
2025-11-19T14:00:28.271101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436083495865819:2048] [node 1] ICS09 handshake done sender: [4:7574436083495865791:2111] self: [4:7574436083495865809:0] peer: [1:7574436081557819370:0] socket: 347 qp: -1
2025-11-19T14:00:28.271106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436083495865819:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:28.271113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:28.271892Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436081557819375:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:28.272395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436077262852006:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.272416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:28.272423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436077262852006:2048] [node 2] ICP22 created new session: [1:7574436081557819401:2048]
2025-11-19T14:00:28.272430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436081557819401:2048] [node 2] ICS09 handshake done sender: [1:7574436081557819375:2113] self: [1:7574436081557819374:0] peer: [2:7574436082259908329:0] socket: 366 qp: -1
2025-11-19T14:00:28.272434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436081557819401:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:28.272441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.273167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436081479956980:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50491
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.273431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436083495865818:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-19T14:00:28.273814Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436081479956971:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:28.275834Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436081479956967:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.276946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436081479956892:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.276968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:28.276977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436081479956892:2049] [node 2] ICP22 created new session: [6:7574436081479956982:2049]
2025-11-19T14:00:28.276983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436081479956982:2049] [node 2] ICS09 handshake done sender: [6:7574436081479956971:2117] self: [6:7574436081479956976:0] peer: [2:7574436082259908314:0] socket: 378 qp: -1
2025-11-19T14:00:28.276989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436081479956982:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:28.276996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:28.277372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436081479956891:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.277380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:28.277386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436081479956891:2048] [node 1] ICP22 created new session: [6:7574436081479956985:2048]
2025-11-19T14:00:28.277391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436081479956985:2048] [node 1] ICS09 handshake done sender: [6:7574436081479956967:2114] self: [6:7574436081479956972:0] peer: [1:7574436081557819372:0] socket: 354 qp: -1
2025-11-19T14:00:28.277396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436081479956985:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:28.277402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:28.277830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436081479956981:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50491
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.279905Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436081557819390:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.280133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436081557819400:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50707
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.280460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436083921786742:2101] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.280478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:28.280483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436083921786742:2101] [node 4] ICP22 created new session: [5:7574436083921786782:2101]
2025-11-19T14:00:28.280488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436083921786782:2101] [node 4] ICS09 handshake done sender: [5:7574436083921786755:2112] self: [5:7574436083921786769:0] peer: [4:7574436083495865786:0] socket: 356 qp: -1
2025-11-19T14:00:28.280493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436083921786782:2101] [node 4] ICS10 traffic start
2025-11-19T14:00:28.280502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:28.280628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574436083921786742:2101] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:28.280802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436083921786781:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9242
2025-11-19T14:00:28.281032Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436084406894999:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:28.281152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436084406894994:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.281170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:28.281199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436084406894994:2110] [node 4] ICP22 created new session: [3:7574436084406895027:2110]
2025-11-19T14:00:28.281205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436084406895027:2110] [node 4] ICS09 handshake done sender: [3:7574436084406894999:2115] self: [3:7574436084406894998:0] peer: [4:7574436083495865814:0] socket: 361 qp: -1
2025-11-19T14:00:28.281209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436084406895027:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:28.281215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:28.281613Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436082259908313:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.285602Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436081479956981:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.285718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436081479956984:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50491
2025-11-19T14:00:28.285770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436081479956946:2098] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.300771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436079200898431:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25803) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436083495865801:2116] held: no
2025-11-19T14:00:28.300924Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436083495865806:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.300929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436083495865806:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:28.301523Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436083495865802:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:28.302043Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436082259908331:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.302065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436082259908309:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.302083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:28.302091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436082259908309:2110] [node 4] ICP22 created new session: [2:7574436082259908334:2110]
2025-11-19T14:00:28.302098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436082259908334:2110] [node 4] ICS09 handshake done sender: [2:7574436082259908313:2113] self: [2:7574436082259908312:0] peer: [4:7574436083495865812:0] socket: 365 qp: -1
2025-11-19T14:00:28.302106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436082259908334:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:28.302119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:28.302382Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436077964940943:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5062) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436082259908320:2116] held: no
2025-11-19T14:00:28.302606Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436083921786779:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.302647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436079626819392:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.302935Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436081479956984:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.302970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436081479956893:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.303094Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436084406895005:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.303106Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436084406895005:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.304335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436081479956989:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50491
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.304438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436083495865822:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7258
2025-11-19T14:00:28.304674Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436084406895005:2118] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:28.304848Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436081479956984:2122] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:28.304911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436081479956893:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.304927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:28.304933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436081479956893:2050] [node 3] ICP22 created new session: [6:7574436081479956990:2050]
2025-11-19T14:00:28.304951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436081479956990:2050] [node 3] ICS09 handshake done sender: [6:7574436081479956984:2122] self: [6:7574436081479956988:0] peer: [3:7574436084406895004:0] socket: 405 qp: -1
2025-11-19T14:00:28.304957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436081479956990:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:28.304965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:28.306031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436084406894995:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.306051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:28.306057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436084406894995:2111] [node 6] ICP22 created new session: [3:7574436084406895030:2111]
2025-11-19T14:00:28.306062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436084406895030:2111] [node 6] ICS09 handshake done sender: [3:7574436084406895005:2118] self: [3:7574436084406895004:0] peer: [6:7574436081479956988:0] socket: 404 qp: -1
2025-11-19T14:00:28.306067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436084406895030:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:28.306074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:28.312754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436081557819366:2108] [node 6] ICP18 handshake (actor [1:7574436081557819390:2118]) is too old
2025-11-19T14:00:28.313261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436079200898430:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.313281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:28.313288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436079200898430:2049] [node 2] ICP22 created new session: [4:7574436083495865824:2049]
2025-11-19T14:00:28.313294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436083495865824:2049] [node 2] ICS09 handshake done sender: [4:7574436083495865802:2117] self: [4:7574436083495865812:0] peer: [2:7574436082259908312:0] socket: 371 qp: -1
2025-11-19T14:00:28.313302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436083495865824:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:28.313312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:28.314330Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436083921786781:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.314363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436079626819391:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:28.314901Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436082259908320:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.315030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436081479956992:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50491
2025-11-19T14:00:28.315256Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436082259908317:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.315262Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436082259908317:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:28.316077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436077964940943:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.316088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:28.316100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436077964940943:2048] [node 1] ICP22 created new session: [2:7574436082259908336:2048]
2025-11-19T14:00:28.316105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436082259908336:2048] [node 1] ICS09 handshake done sender: [2:7574436082259908320:2116] self: [2:7574436082259908329:0] peer: [1:7574436081557819374:0] socket: 373 qp: -1
2025-11-19T14:00:28.316111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436082259908336:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:28.316117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.316645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436083921786786:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9242
2025-11-19T14:00:28.316803Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436081557819377:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:28.316814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436081557819377:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.317722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436083921786787:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9242
2025-11-19T14:00:28.317959Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436083495865801:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:28.318072Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436081557819377:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:28.318258Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436083921786781:2121] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:28.318333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436079626819391:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.318351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:28.318356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436079626819391:2048] [node 1] ICP22 created new session: [5:7574436083921786788:2048]
2025-11-19T14:00:28.318362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436083921786788:2048] [node 1] ICS09 handshake done sender: [5:7574436083921786781:2121] self: [5:7574436083921786785:0] peer: [1:7574436081557819376:0] socket: 397 qp: -1
2025-11-19T14:00:28.318368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436083921786788:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:28.318375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:28.319582Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436083495865806:2119] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:28.319874Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436081479956981:2121] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:28.319979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436081479956946:2098] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.319995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:28.320002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436081479956946:2098] [node 4] ICP22 created new session: [6:7574436081479956993:2098]
2025-11-19T14:00:28.320007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436081479956993:2098] [node 4] ICS09 handshake done sender: [6:7574436081479956981:2121] self: [6:7574436081479956987:0] peer: [4:7574436083495865805:0] socket: 403 qp: -1
2025-11-19T14:00:28.320014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436081479956993:2098] [node 4] ICS10 traffic start
2025-11-19T14:00:28.320024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:28.321012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436081557819369:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.321032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:28.321038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436081557819369:2109] [node 5] ICP22 created new session: [1:7574436081557819404:2109]
2025-11-19T14:00:28.321043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436081557819404:2109] [node 5] ICS09 handshake done sender: [1:7574436081557819377:2114] self: [1:7574436081557819376:0] peer: [5:7574436083921786785:0] socket: 396 qp: -1
2025-11-19T14:00:28.321048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436081557819404:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:28.321057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:28.321882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436079200898431:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.321897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:28.321901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436079200898431:2050] [node 3] ICP22 created new session: [4:7574436083495865826:2050]
2025-11-19T14:00:28.321906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436083495865826:2050] [node 3] ICS09 handshake done sender: [4:7574436083495865801:2116] self: [4:7574436083495865814:0] peer: [3:7574436084406894998:0] socket: 368 qp: -1
2025-11-19T14:00:28.321912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436083495865826:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:28.321918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:28.322068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436083495865800:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.322076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:28.322079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436083495865800:2114] [node 6] ICP22 created new session: [4:7574436083495865828:2114]
2025-11-19T14:00:28.322083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436083495865828:2114] [node 6] ICS09 handshake done sender: [4:7574436083495865806:2119] self: [4:7574436083495865805:0] peer: [6:7574436081479956987:0] socket: 401 qp: -1
2025-11-19T14:00:28.322086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436083495865828:2114] [node 6] ICS10 traffic start
2025-11-19T14:00:28.322091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:28.322996Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436082259908317:2115] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:28.323133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436082259908311:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:28.323147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:28.323152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436082259908311:2112] [node 5] ICP22 created new session: [2:7574436082259908340:2112]
2025-11-19T14:00:28.323157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436082259908340:2112] [node 5] ICS09 handshake done sender: [2:7574436082259908317:2115] self: [2:7574436082259908316:0] peer: [5:7574436083921786784:0] socket: 390 qp: -1
2025-11-19T14:00:28.323161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436082259908340:2112] [node 5] ICS10 traffic start
2025-11-19T14:00:28.323169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:28.323608Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436083921786779:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:28.327949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436079626819392:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:28.327972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:28.327979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436079626819392:2049] [node 2] ICP22 created new session: [5:7574436083921786790:2049]
2025-11-19T14:00:28.327986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436083921786790:2049] [node 2] ICS09 handshake done sender: [5:7574436083921786779:2120] self: [5:7574436083921786784:0] peer: [2:7574436082259908316:0] socket: 393 qp: -1
2025-11-19T14:00:28.327993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436083921786790:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:28.328003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:28.353131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:28.353220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436082259908336:2048] [node 1] ICS01 socket: 373 reason# EndOfStream
2025-11-19T14:00:28.353229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436077964940943:2048] [node 1] ICP30 unregister session Session# [2:7574436082259908336:2048] VirtualId# [2:7574436082259908329:0]
2025-11-19T14:00:28.353236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436082259908336:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:28.353272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:28.353292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:28.353721Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:28.353785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436084406895022:2048] [node 1] ICS01 socket: 344 reason# EndOfStream
2025-11-19T14:00:28.353792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436080111927630:2048] [node 1] ICP30 unregister session Session# [3:7574436084406895022:2048] VirtualId# [3:7574436084406895009:0]
2025-11-19T14:00:28.353796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436084406895022:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:28.353822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:28.353840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:28.383349Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:28.383451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436083495865819:2048] [node 1] ICS01 socket: 347 reason# EndOfStream
2025-11-19T14:00:28.383461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436079200898429:2048] [node 1] ICP30 unregister session Session# [4:7574436083495865819:2048] VirtualId# [4:7574436083495865809:0]
2025-11-19T14:00:28.383469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436083495865819:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:28.383535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:28.383564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:28.383610Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:28.383698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436081479956985:2048] [node 1] ICS01 socket: 354 reason# EndOfStream
2025-11-19T14:00:28.383704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436081479956891:2048] [node 1] ICP30 unregister session Session# [6:7574436081479956985:2048] VirtualId# [6:7574436081479956972:0]
2025-11-19T14:00:28.383710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436081479956985:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:28.383757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:28.383784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:28.383824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:28.383903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436083921786788:2048] [node 1] ICS01 socket: 397 reason# EndOfStream
2025-11-19T14:00:28.383909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436079626819391:2048] [node 1] ICP30 unregister session Session# [5:7574436083921786788:2048] VirtualId# [5:7574436083921786785:0]
2025-11-19T14:00:28.383914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436083921786788:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:28.383948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:28.383972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:28.384965Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:28.385041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436084406895020:2049] [node 2] ICS01 socket: 355 reason# EndOfStream
2025-11-19T14:00:28.385048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436080111927631:2049] [node 2] ICP30 unregister session Session# [3:7574436084406895020:2049] VirtualId# [3:7574436084406895010:0]
2025-11-19T14:00:28.385054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436084406895020:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:28.385088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:28.385109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:28.400140Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:28.400248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436083495865824:2049] [node 2] ICS01 socket: 371 reason# EndOfStream
2025-11-19T14:00:28.400258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436079200898430:2049] [node 2] ICP30 unregister session Session# [4:7574436083495865824:2049] VirtualId# [4:7574436083495865812:0]
2025-11-19T14:00:28.400266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436083495865824:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:28.400332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:28.400357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:28.400399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:28.400483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436081479956982:2049] [node 2] ICS01 socket: 378 reason# EndOfStream
2025-11-19T14:00:28.400489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436081479956892:2049] [node 2] ICP30 unregister session Session# [6:7574436081479956982:2049] VirtualId# [6:7574436081479956976:0]
2025-11-19T14:00:28.400495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436081479956982:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:28.400529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:28.400553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:28.400590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:28.400650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436083921786790:2049] [node 2] ICS01 socket: 393 reason# EndOfStream
2025-11-19T14:00:28.400656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436079626819392:2049] [node 2] ICP30 unregister session Session# [5:7574436083921786790:2049] VirtualId# [5:7574436083921786784:0]
2025-11-19T14:00:28.400662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436083921786790:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:28.400692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:28.400713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:28.446876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:28.446978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436083921786776:2050] [node 3] ICS01 socket: 364 reason# EndOfStream
2025-11-19T14:00:28.446987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436079626819393:2050] [node 3] ICP30 unregister session Session# [5:7574436083921786776:2050] VirtualId# [5:7574436083921786768:0]
2025-11-19T14:00:28.446995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436083921786776:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:28.447060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:28.447089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:28.447135Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:28.447249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436081479956990:2050] [node 3] ICS01 socket: 405 reason# EndOfStream
2025-11-19T14:00:28.447255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436081479956893:2050] [node 3] ICP30 unregister session Session# [6:7574436081479956990:2050] VirtualId# [6:7574436081479956988:0]
2025-11-19T14:00:28.447261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436081479956990:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:28.447294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:28.447317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:28.478980Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:28.479088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436081479956993:2098] [node 4] ICS01 socket: 403 reason# EndOfStream
2025-11-19T14:00:28.479098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436081479956946:2098] [node 4] ICP30 unregister session Session# [6:7574436081479956993:2098] VirtualId# [6:7574436081479956987:0]
2025-11-19T14:00:28.479106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436081479956993:2098] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:28.479178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:28.479205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:28.704297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436080932018107:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:28.704308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436080932018108:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:28.763938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436082804054724:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:28.763952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436082804054725:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:28.816216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436084096572913:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:28.816232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436084096572914:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:28.880044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436082693242113:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:28.880062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436082693242114:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:28.880066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436082693242115:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:28.948657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436082573327579:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:28.948672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436082573327580:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:28.948677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436082573327581:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:28.986053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436084096572956:2086] [node 5] ICP01 ready to work
2025-11-19T14:00:28.986161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436084096572957:2089] [node 6] ICP01 ready to work
2025-11-19T14:00:28.986919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436084096572962:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21790
2025-11-19T14:00:28.988050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436084096572968:2094] [node 4] ICP01 ready to work
2025-11-19T14:00:28.980115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436084981840734:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:28.980130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436084981840735:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:28.980134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436084981840736:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:28.987403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436084981840747:2060] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52551
2025-11-19T14:00:28.992246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436082804054769:2090] [node 6] ICP01 ready to work
2025-11-19T14:00:28.994119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436082693242157:2087] [node 5] ICP01 ready to work
2025-11-19T14:00:28.994229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436082693242158:2090] [node 6] ICP01 ready to work
2025-11-19T14:00:29.000076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436084981840783:2071] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.006004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436080932018165:2100] [node 6] ICP01 ready to work
2025-11-19T14:00:29.006199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436085226985469:2102] [node 5] ICP01 ready to work
2025-11-19T14:00:29.006336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436085226985470:2106] [node 4] ICP01 ready to work
2025-11-19T14:00:29.004913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436088391540276:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21790
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.018147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436086868294938:2074] [node 4] ICP01 ready to work
2025-11-19T14:00:29.018286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436086868294939:2078] [node 6] ICP01 ready to work
2025-11-19T14:00:29.019457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436086868294937:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14624
2025-11-19T14:00:29.019963Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436082804054769:2090] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27628095ns) {Configure:104(27626860ns) {ProcessConfigured:135(27199609ns) {StartInitialHandshake:157(27198923ns) {DropHandshakes:469(892ns) {DropIncomingHandshake:441(137ns) DropOutgoingHandshake:456(64ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:476(1308ns) {SwitchToState:178(761ns)}}}}}
2025-11-19T14:00:29.020158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436087099022091:2097] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.029203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436087099022094:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47900
2025-11-19T14:00:29.029639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436085226985482:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7159
2025-11-19T14:00:29.029669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436085226985484:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7159
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.031492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436085226985483:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7159
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.033877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436088391540290:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21790
2025-11-19T14:00:29.037278Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436084096572962:2092] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.037328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436084096572913:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.039259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436089276808107:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52551
2025-11-19T14:00:29.039307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436089276808108:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52551
2025-11-19T14:00:29.039935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436087099022089:2095] [node 5] ICP01 ready to work
2025-11-19T14:00:29.040170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436089276808100:2063] [node 5] ICP01 ready to work
2025-11-19T14:00:29.040602Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085226985482:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.047906Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436089276808107:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.048059Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436084981840747:2060] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.048114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436084981840734:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.048137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436084981840736:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.051984Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436088391540276:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.052429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436080932018159:2098] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.052453Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436080932018159:2098] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.053226Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436086868294937:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.055927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436082693242157:2087] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(61215880ns) {Configure:104(43654901ns) {ProcessConfigured:135(43187509ns) {StartInitialHandshake:157(43186296ns) {DropHandshakes:469(1021ns) {DropIncomingHandshake:441(119ns) DropOutgoingHandshake:456(117ns)} GenerateSessionVirtualId:988(320ns) PrepareNewSessionHandshake:476(2091ns) {SwitchToState:178(1566ns)}}}}}
2025-11-19T14:00:29.057270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436086988209479:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27407
2025-11-19T14:00:29.057339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436086988209480:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27407
2025-11-19T14:00:29.057377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436086988209481:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27407
2025-11-19T14:00:29.057414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436086988209482:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27407
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.060161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436089276808112:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52551
2025-11-19T14:00:29.057831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436087099022089:2095] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(17737607ns) {Configure:104(17736339ns) {ProcessConfigured:135(17334439ns) {StartInitialHandshake:157(17333925ns) {DropHandshakes:469(790ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(128ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:476(1156ns) {SwitchToState:178(57ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.060151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436082693242113:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(65881468ns) {Configure:104(65880725ns) {ProcessConfigured:135(65501469ns) {StartInitialHandshake:157(65500884ns) {DropHandshakes:469(787ns) {DropIncomingHandshake:441(109ns) DropOutgoingHandshake:456(51ns)} GenerateSessionVirtualId:988(114ns) PrepareNewSessionHandshake:476(709ns) {SwitchToState:178(452ns)}}}}}
2025-11-19T14:00:29.060257Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436084096572961:2091] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.060276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436084096572961:2091] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.060330Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085226985473:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.060347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436085226985473:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.060497Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436086988209482:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.060522Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436086988209479:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.060538Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436086988209480:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.060582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436082693242115:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.060421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436086868294954:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14624
2025-11-19T14:00:29.063623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436082573327581:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(59646775ns) {Configure:104(59645985ns) {ProcessConfigured:135(59187802ns) {StartInitialHandshake:157(59187133ns) {DropHandshakes:469(749ns) {DropIncomingHandshake:441(95ns) DropOutgoingHandshake:456(76ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:476(733ns) {SwitchToState:178(552ns)}}}}}
2025-11-19T14:00:29.063718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436082573327581:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.061116Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085226985483:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.064213Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436080932018159:2098] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.064218Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436086988209481:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.064252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436082693242113:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.064339Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436088391540275:2105] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.064353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436088391540275:2105] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.064833Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436084096572959:2090] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.064837Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436084096572959:2090] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.065691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436088391540293:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21790
2025-11-19T14:00:29.065892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436086868294956:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14624
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.067842Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085226985475:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.067857Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436085226985475:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.068369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436080932018108:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.068387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:29.068395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436080932018108:2049] [node 3] ICP22 created new session: [1:7574436085226985488:2049]
2025-11-19T14:00:29.068403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436085226985488:2049] [node 3] ICS09 handshake done sender: [1:7574436080932018159:2098] self: [1:7574436080932018158:0] peer: [3:7574436088391540291:0] socket: 342 qp: -1
2025-11-19T14:00:29.068410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436085226985488:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:29.068421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:29.069367Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087099022094:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.070270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436089276808113:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52551
2025-11-19T14:00:29.070361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436089276808100:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(30041854ns) {Configure:104(30041144ns) {ProcessConfigured:135(29651666ns) {StartInitialHandshake:157(29650655ns) {DropHandshakes:469(497ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:476(303ns) {SwitchToState:178(108ns)}}}}}
2025-11-19T14:00:29.070705Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436085226985473:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:29.066963Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436088391540275:2105] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:29.070948Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436089276808107:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:29.072618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436087099022104:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47900
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.072996Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436084096572959:2090] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:29.074935Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436089276808108:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.074966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436084981840734:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.074986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:29.074998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436084981840734:2048] [node 1] ICP22 created new session: [6:7574436089276808115:2048]
2025-11-19T14:00:29.075005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436089276808115:2048] [node 1] ICS09 handshake done sender: [6:7574436089276808107:2115] self: [6:7574436089276808110:0] peer: [1:7574436085226985472:0] socket: 366 qp: -1
2025-11-19T14:00:29.075015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436089276808115:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:29.075043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:29.075248Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436085226985475:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:29.072783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436086868294957:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14624
2025-11-19T14:00:29.073211Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436086868294937:2108] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:29.073310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436082573327581:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.073324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:29.073330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436082573327581:2050] [node 3] ICP22 created new session: [5:7574436086868294958:2050]
2025-11-19T14:00:29.073336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436086868294958:2050] [node 3] ICS09 handshake done sender: [5:7574436086868294937:2108] self: [5:7574436086868294955:0] peer: [3:7574436084096572958:0] socket: 347 qp: -1
2025-11-19T14:00:29.073342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436086868294958:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:29.073350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:29.075462Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085226985484:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.075477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436080932018165:2100] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.075492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:29.075497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436080932018165:2100] [node 6] ICP22 created new session: [1:7574436085226985493:2100]
2025-11-19T14:00:29.075502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436085226985493:2100] [node 6] ICS09 handshake done sender: [1:7574436085226985473:2111] self: [1:7574436085226985472:0] peer: [6:7574436089276808110:0] socket: 354 qp: -1
2025-11-19T14:00:29.075508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436085226985493:2100] [node 6] ICS10 traffic start
2025-11-19T14:00:29.075515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:29.075795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436085226985470:2106] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.075803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:29.075807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436085226985470:2106] [node 4] ICP22 created new session: [1:7574436085226985495:2106]
2025-11-19T14:00:29.075811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436085226985495:2106] [node 4] ICS09 handshake done sender: [1:7574436085226985475:2112] self: [1:7574436085226985474:0] peer: [4:7574436086988209494:0] socket: 355 qp: -1
2025-11-19T14:00:29.075816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436085226985495:2106] [node 4] ICS10 traffic start
2025-11-19T14:00:29.075820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:29.076606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436084096572968:2094] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.076626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:29.076632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436084096572968:2094] [node 4] ICP22 created new session: [3:7574436088391540297:2094]
2025-11-19T14:00:29.076639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436088391540297:2094] [node 4] ICS09 handshake done sender: [3:7574436088391540275:2105] self: [3:7574436084096572969:0] peer: [4:7574436086988209493:0] socket: 352 qp: -1
2025-11-19T14:00:29.076646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436088391540297:2094] [node 4] ICS10 traffic start
2025-11-19T14:00:29.076656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:29.076901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436084096572956:2086] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.076907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:29.076912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436084096572956:2086] [node 5] ICP22 created new session: [3:7574436088391540299:2086]
2025-11-19T14:00:29.076916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436088391540299:2086] [node 5] ICS09 handshake done sender: [3:7574436084096572959:2090] self: [3:7574436084096572958:0] peer: [5:7574436086868294955:0] socket: 344 qp: -1
2025-11-19T14:00:29.076920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436088391540299:2086] [node 5] ICS10 traffic start
2025-11-19T14:00:29.076925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:29.078306Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436084096572961:2091] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:29.078543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436084096572957:2089] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.078563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:29.078568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436084096572957:2089] [node 6] ICP22 created new session: [3:7574436088391540303:2089]
2025-11-19T14:00:29.078573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436088391540303:2089] [node 6] ICS09 handshake done sender: [3:7574436084096572961:2091] self: [3:7574436084096572960:0] peer: [6:7574436089276808111:0] socket: 345 qp: -1
2025-11-19T14:00:29.078576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436088391540303:2089] [node 6] ICS10 traffic start
2025-11-19T14:00:29.078582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:29.078758Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436084096572962:2092] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.083228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436085226985499:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7159
2025-11-19T14:00:29.081665Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436086868294954:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.081711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436086868294938:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.083463Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436086868294957:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.083498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436082573327579:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.085165Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085226985477:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.085185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436085226985477:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.086051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436086868294963:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14624
2025-11-19T14:00:29.091717Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436085226985477:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:29.091947Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436086868294957:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:29.092126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436085226985469:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.092169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:29.092181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436085226985469:2102] [node 5] ICP22 created new session: [1:7574436085226985501:2102]
2025-11-19T14:00:29.092188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436085226985501:2102] [node 5] ICS09 handshake done sender: [1:7574436085226985477:2113] self: [1:7574436085226985476:0] peer: [5:7574436086868294962:0] socket: 384 qp: -1
2025-11-19T14:00:29.092197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436085226985501:2102] [node 5] ICS10 traffic start
2025-11-19T14:00:29.092208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.095858Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436088391540290:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.095880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436084096572913:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.095897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:29.095904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436084096572913:2048] [node 1] ICP22 created new session: [3:7574436088391540305:2048]
2025-11-19T14:00:29.095912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436088391540305:2048] [node 1] ICS09 handshake done sender: [3:7574436084096572962:2092] self: [3:7574436088391540291:0] peer: [1:7574436080932018158:0] socket: 343 qp: -1
2025-11-19T14:00:29.095918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436088391540305:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:29.095927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:29.096419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436084096572914:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.097384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436087099022106:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47900
2025-11-19T14:00:29.097438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436087099022107:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47900
2025-11-19T14:00:29.098030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574436084981840734:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22966166ns) {HandleHandshakeStatus:329(22965183ns) {DropIncomingHandshake:441(331ns) DropOutgoingHandshake:456(2098ns) SwitchToState:178(277ns) ProcessPendingSessionEvents:495(16532500ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16509548ns max# 16437596ns}}}
2025-11-19T14:00:29.099053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436086988209495:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27407
2025-11-19T14:00:29.099101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436086988209496:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27407
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.099482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436084981840735:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.099509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436084981840736:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7765) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436084981840747:2060] held: no
2025-11-19T14:00:29.099729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436089276808117:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52551
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.099894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436089276808118:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52551
2025-11-19T14:00:29.099340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436082573327579:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.099379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:29.099392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436082573327579:2048] [node 1] ICP22 created new session: [5:7574436086868294964:2048]
2025-11-19T14:00:29.099401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436086868294964:2048] [node 1] ICS09 handshake done sender: [5:7574436086868294957:2118] self: [5:7574436086868294962:0] peer: [1:7574436085226985476:0] socket: 389 qp: -1
2025-11-19T14:00:29.099415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436086868294964:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:29.099449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:29.099920Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087099022096:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.099931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436087099022096:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.102017Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087099022106:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.102061Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087099022090:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.102064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436087099022090:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.102564Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087099022107:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.102697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436082804054724:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.102941Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436087099022090:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:29.103032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436082804054769:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.103047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:29.103053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436082804054769:2090] [node 6] ICP22 created new session: [2:7574436087099022111:2090]
2025-11-19T14:00:29.103058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436087099022111:2090] [node 6] ICS09 handshake done sender: [2:7574436087099022090:2114] self: [2:7574436082804054770:0] peer: [6:7574436089276808119:0] socket: 360 qp: -1
2025-11-19T14:00:29.103064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436087099022111:2090] [node 6] ICS10 traffic start
2025-11-19T14:00:29.103071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:29.103527Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085226985479:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.103540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436085226985479:2114] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.106002Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436085226985479:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.106119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436080932018107:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.106138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:29.106144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436080932018107:2048] [node 2] ICP22 created new session: [1:7574436085226985504:2048]
2025-11-19T14:00:29.106149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436085226985504:2048] [node 2] ICS09 handshake done sender: [1:7574436085226985479:2114] self: [1:7574436085226985478:0] peer: [2:7574436087099022110:0] socket: 393 qp: -1
2025-11-19T14:00:29.106156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436085226985504:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:29.106164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:29.107817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436087099022114:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47900
2025-11-19T14:00:29.110729Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087099022104:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.110752Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436082804054724:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63259) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436087099022107:2122] held: no
2025-11-19T14:00:29.110866Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436087099022107:2122] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.111052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436082804054724:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.111066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:29.111072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436082804054724:2048] [node 1] ICP22 created new session: [2:7574436087099022116:2048]
2025-11-19T14:00:29.111077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436087099022116:2048] [node 1] ICS09 handshake done sender: [2:7574436087099022107:2122] self: [2:7574436087099022110:0] peer: [1:7574436085226985478:0] socket: 397 qp: -1
2025-11-19T14:00:29.111083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436087099022116:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:29.111089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:29.111270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436087099022115:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47900
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.108485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436086988209476:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.108497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436086988209476:2110] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.109581Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436086988209481:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:29.114763Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436084981840747:2060] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:29.111712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436086868294966:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14624
2025-11-19T14:00:29.113552Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436086868294966:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.113584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436082573327580:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.116232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436086868294968:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14624
2025-11-19T14:00:29.116728Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436087099022096:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:29.116840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436082804054725:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.116859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:29.116866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436082804054725:2049] [node 3] ICP22 created new session: [2:7574436087099022121:2049]
2025-11-19T14:00:29.116871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436087099022121:2049] [node 3] ICS09 handshake done sender: [2:7574436087099022096:2117] self: [2:7574436087099022095:0] peer: [3:7574436088391540307:0] socket: 362 qp: -1
2025-11-19T14:00:29.116876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436087099022121:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:29.116885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.116207Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436089276808117:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.116233Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436089276808112:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.117351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436089276808122:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52551
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.118137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436088391540308:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21790
2025-11-19T14:00:29.117479Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436086988209480:2113] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:29.118312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436082693242113:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.118325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:29.118331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436082693242113:2048] [node 1] ICP22 created new session: [4:7574436086988209499:2048]
2025-11-19T14:00:29.118338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436086988209499:2048] [node 1] ICS09 handshake done sender: [4:7574436086988209481:2114] self: [4:7574436086988209494:0] peer: [1:7574436085226985474:0] socket: 370 qp: -1
2025-11-19T14:00:29.118343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436086988209499:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:29.118349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:29.118505Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087099022103:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.118517Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436087099022103:2119] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.123469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436086868294969:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14624
2025-11-19T14:00:29.124299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436082693242115:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.124320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:29.124326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436082693242115:2050] [node 3] ICP22 created new session: [4:7574436086988209501:2050]
2025-11-19T14:00:29.124332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436086988209501:2050] [node 3] ICS09 handshake done sender: [4:7574436086988209480:2113] self: [4:7574436086988209493:0] peer: [3:7574436084096572969:0] socket: 369 qp: -1
2025-11-19T14:00:29.124338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436086988209501:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:29.124344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:29.127900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436089276808100:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.127937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436084981840783:2071] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.127964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436084981840736:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.127977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:29.127984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436084981840736:2050] [node 3] ICP22 created new session: [6:7574436089276808125:2050]
2025-11-19T14:00:29.127989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436089276808125:2050] [node 3] ICS09 handshake done sender: [6:7574436084981840747:2060] self: [6:7574436089276808111:0] peer: [3:7574436084096572960:0] socket: 346 qp: -1
2025-11-19T14:00:29.127995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436089276808125:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:29.128002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:29.128309Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436089276808108:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.130735Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436086868294947:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.130752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436086868294947:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.131413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436086868294970:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14624
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.132026Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436087099022103:2119] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:29.131812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436086988209497:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27407
2025-11-19T14:00:29.131909Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436086988209476:2110] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:29.132603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574436080932018108:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:29.132993Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436088391540290:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:29.134403Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087099022115:2124] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.134464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436087099022089:2095] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.134480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:29.134486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436087099022089:2095] [node 5] ICP22 created new session: [2:7574436087099022124:2095]
2025-11-19T14:00:29.134491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436087099022124:2095] [node 5] ICS09 handshake done sender: [2:7574436087099022103:2119] self: [2:7574436087099022102:0] peer: [5:7574436086868294967:0] socket: 361 qp: -1
2025-11-19T14:00:29.134498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436087099022124:2095] [node 5] ICS10 traffic start
2025-11-19T14:00:29.134506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:29.135113Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436086868294947:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:29.135341Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436086988209478:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.135350Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436086988209478:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.136135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436082804054769:2090] [node 6] ICP18 handshake (actor [2:7574436087099022115:2124]) is too old
2025-11-19T14:00:29.136289Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436086988209497:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.136315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436082693242157:2087] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.136331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:29.136339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436082693242157:2087] [node 5] ICP22 created new session: [4:7574436086988209503:2087]
2025-11-19T14:00:29.136345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436086988209503:2087] [node 5] ICS09 handshake done sender: [4:7574436086988209476:2110] self: [4:7574436086988209474:0] peer: [5:7574436086868294961:0] socket: 372 qp: -1
2025-11-19T14:00:29.136353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436086988209503:2087] [node 5] ICS10 traffic start
2025-11-19T14:00:29.136363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:29.136991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436084096572914:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7355) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436088391540290:2117] held: no
2025-11-19T14:00:29.137085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436084096572914:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.137095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:29.137100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436084096572914:2049] [node 2] ICP22 created new session: [3:7574436088391540309:2049]
2025-11-19T14:00:29.137105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436088391540309:2049] [node 2] ICS09 handshake done sender: [3:7574436088391540290:2117] self: [3:7574436088391540307:0] peer: [2:7574436087099022095:0] socket: 365 qp: -1
2025-11-19T14:00:29.137110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436088391540309:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:29.137117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:29.137653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436084981840735:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.137699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:29.137705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436084981840735:2049] [node 2] ICP22 created new session: [6:7574436089276808128:2049]
2025-11-19T14:00:29.137709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436089276808128:2049] [node 2] ICS09 handshake done sender: [6:7574436089276808108:2116] self: [6:7574436089276808119:0] peer: [2:7574436082804054770:0] socket: 367 qp: -1
2025-11-19T14:00:29.137722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436089276808128:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:29.137729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:29.139074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436082693242114:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.139198Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436086988209478:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:29.139265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436082693242158:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.139278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:29.139283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436082693242158:2090] [node 6] ICP22 created new session: [4:7574436086988209507:2090]
2025-11-19T14:00:29.139289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436086988209507:2090] [node 6] ICS09 handshake done sender: [4:7574436086988209478:2111] self: [4:7574436086988209477:0] peer: [6:7574436089276808124:0] socket: 373 qp: -1
2025-11-19T14:00:29.139294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436086988209507:2090] [node 6] ICS10 traffic start
2025-11-19T14:00:29.139299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:29.139990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436086868294939:2078] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.140021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:29.140029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436086868294939:2078] [node 6] ICP22 created new session: [5:7574436086868294972:2078]
2025-11-19T14:00:29.140035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436086868294972:2078] [node 6] ICS09 handshake done sender: [5:7574436086868294947:2112] self: [5:7574436086868294946:0] peer: [6:7574436089276808123:0] socket: 390 qp: -1
2025-11-19T14:00:29.140043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436086868294972:2078] [node 6] ICS10 traffic start
2025-11-19T14:00:29.140052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.140882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436089276808127:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52551
2025-11-19T14:00:29.141041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436084981840783:2071] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3947) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436089276808112:2117] held: no
2025-11-19T14:00:29.141246Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436089276808117:2120] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.141322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436089276808130:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52551
2025-11-19T14:00:29.141435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436089276808100:2063] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.141449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:29.141453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436089276808100:2063] [node 5] ICP22 created new session: [6:7574436089276808131:2063]
2025-11-19T14:00:29.141460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436089276808131:2063] [node 5] ICS09 handshake done sender: [6:7574436089276808117:2120] self: [6:7574436089276808123:0] peer: [5:7574436086868294946:0] socket: 398 qp: -1
2025-11-19T14:00:29.141464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436089276808131:2063] [node 5] ICS10 traffic start
2025-11-19T14:00:29.141470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:29.142460Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436089276808112:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:29.142533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436084981840783:2071] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.142546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:29.142550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436084981840783:2071] [node 4] ICP22 created new session: [6:7574436089276808133:2071]
2025-11-19T14:00:29.142555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436089276808133:2071] [node 4] ICS09 handshake done sender: [6:7574436089276808112:2117] self: [6:7574436089276808124:0] peer: [4:7574436086988209477:0] socket: 376 qp: -1
2025-11-19T14:00:29.142559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436089276808133:2071] [node 4] ICS10 traffic start
2025-11-19T14:00:29.142564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:29.147005Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087099022098:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.147019Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436087099022098:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.148602Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436087099022098:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:29.148477Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436086868294968:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.148912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436087099022091:2097] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.148930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:29.148937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436087099022091:2097] [node 4] ICP22 created new session: [2:7574436087099022127:2097]
2025-11-19T14:00:29.148942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436087099022127:2097] [node 4] ICS09 handshake done sender: [2:7574436087099022098:2118] self: [2:7574436087099022097:0] peer: [4:7574436086988209506:0] socket: 349 qp: -1
2025-11-19T14:00:29.148948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436087099022127:2097] [node 4] ICS10 traffic start
2025-11-19T14:00:29.148956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:29.151295Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436086868294966:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:29.148297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436086988209509:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27407
2025-11-19T14:00:29.148769Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436086988209497:2121] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:29.149403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436082693242114:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.149413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:29.151394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436082573327580:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7355) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436086868294966:2120] held: no
2025-11-19T14:00:29.149418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436082693242114:2049] [node 2] ICP22 created new session: [4:7574436086988209510:2049]
2025-11-19T14:00:29.151594Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436086868294954:2116] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:29.149423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436086988209510:2049] [node 2] ICS09 handshake done sender: [4:7574436086988209497:2121] self: [4:7574436086988209506:0] peer: [2:7574436087099022097:0] socket: 364 qp: -1
2025-11-19T14:00:29.149428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436086988209510:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:29.149433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:29.151675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436086868294938:2074] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3947) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436086868294954:2116] held: no
2025-11-19T14:00:29.151864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436082573327580:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.151875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:29.151881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436082573327580:2049] [node 2] ICP22 created new session: [5:7574436086868294974:2049]
2025-11-19T14:00:29.151886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436086868294974:2049] [node 2] ICS09 handshake done sender: [5:7574436086868294966:2120] self: [5:7574436086868294967:0] peer: [2:7574436087099022102:0] socket: 404 qp: -1
2025-11-19T14:00:29.151892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436086868294974:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:29.151899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:29.152182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574436086868294939:2078] [node 6] ICP18 handshake (actor [5:7574436086868294968:2121]) is too old
2025-11-19T14:00:29.152526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436086868294938:2074] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.152539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:29.152544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436086868294938:2074] [node 4] ICP22 created new session: [5:7574436086868294976:2074]
2025-11-19T14:00:29.152549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436086868294976:2074] [node 4] ICS09 handshake done sender: [5:7574436086868294954:2116] self: [5:7574436086868294961:0] peer: [4:7574436086988209474:0] socket: 375 qp: -1
2025-11-19T14:00:29.152554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436086868294976:2074] [node 4] ICS10 traffic start
2025-11-19T14:00:29.152560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:29.165010Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:29.165082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436088391540305:2048] [node 1] ICS01 socket: 343 reason# EndOfStream
2025-11-19T14:00:29.165089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436084096572913:2048] [node 1] ICP30 unregister session Session# [3:7574436088391540305:2048] VirtualId# [3:7574436088391540291:0]
2025-11-19T14:00:29.165095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436088391540305:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.165123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:29.165142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:29.164493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:29.164591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436087099022116:2048] [node 1] ICS01 socket: 397 reason# EndOfStream
2025-11-19T14:00:29.164600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436082804054724:2048] [node 1] ICP30 unregister session Session# [2:7574436087099022116:2048] VirtualId# [2:7574436087099022110:0]
2025-11-19T14:00:29.164607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436087099022116:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.164644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:29.164664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:29.174698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:29.174799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436089276808115:2048] [node 1] ICS07 socket disconnect 366 reason# EndOfStream
2025-11-19T14:00:29.174807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436089276808115:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.174876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:29.174898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436089276808115:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:29.175354Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436084981840734:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63259) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:29.175364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436089276808115:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:29.175370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436084981840734:2048] [node 1] ICP30 unregister session Session# [6:7574436089276808115:2048] VirtualId# [6:7574436089276808110:0]
2025-11-19T14:00:29.175412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:29.175420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436084981840734:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63259) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:29.175422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63259) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:29.178321Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:29.178405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436086988209499:2048] [node 1] ICS07 socket disconnect 370 reason# EndOfStream
2025-11-19T14:00:29.178410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436086988209499:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.178463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:29.178484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436086988209499:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:29.178852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:29.178913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436086868294964:2048] [node 1] ICS01 socket: 389 reason# EndOfStream
2025-11-19T14:00:29.178918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436082573327579:2048] [node 1] ICP30 unregister session Session# [5:7574436086868294964:2048] VirtualId# [5:7574436086868294962:0]
2025-11-19T14:00:29.178924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436086868294964:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.178951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:29.178977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:29.179281Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436082693242113:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63259) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:29.179292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436086988209499:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:29.179296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436082693242113:2048] [node 1] ICP30 unregister session Session# [4:7574436086988209499:2048] VirtualId# [4:7574436086988209494:0]
2025-11-19T14:00:29.179333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:29.179338Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436082693242113:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63259) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:29.179340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63259) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:29.181701Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:29.181794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436088391540309:2049] [node 2] ICS01 socket: 365 reason# EndOfStream
2025-11-19T14:00:29.181802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436084096572914:2049] [node 2] ICP30 unregister session Session# [3:7574436088391540309:2049] VirtualId# [3:7574436088391540307:0]
2025-11-19T14:00:29.181808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436088391540309:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.181841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:29.181861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:29.184084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436084981840734:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:29.190026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:29.190121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436086868294974:2049] [node 2] ICS01 socket: 404 reason# EndOfStream
2025-11-19T14:00:29.190128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436082573327580:2049] [node 2] ICP30 unregister session Session# [5:7574436086868294974:2049] VirtualId# [5:7574436086868294967:0]
2025-11-19T14:00:29.190136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436086868294974:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.190179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:29.190208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:29.190241Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:29.190303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436086988209510:2049] [node 2] ICS01 socket: 364 reason# EndOfStream
2025-11-19T14:00:29.190308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436082693242114:2049] [node 2] ICP30 unregister session Session# [4:7574436086988209510:2049] VirtualId# [4:7574436086988209506:0]
2025-11-19T14:00:29.190312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436086988209510:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.190331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:29.190351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:29.191804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:29.191885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436089276808128:2049] [node 2] ICS01 socket: 367 reason# EndOfStream
2025-11-19T14:00:29.191892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436084981840735:2049] [node 2] ICP30 unregister session Session# [6:7574436089276808128:2049] VirtualId# [6:7574436089276808119:0]
2025-11-19T14:00:29.191897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436089276808128:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.191948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:29.191979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:29.195832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574436082693242113:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:29.218843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:29.218934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436089276808125:2050] [node 3] ICS01 socket: 346 reason# EndOfStream
2025-11-19T14:00:29.218943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436084981840736:2050] [node 3] ICP30 unregister session Session# [6:7574436089276808125:2050] VirtualId# [6:7574436089276808111:0]
2025-11-19T14:00:29.218950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436089276808125:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.218991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:29.219012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:29.223425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:29.223558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436086868294958:2050] [node 3] ICS01 socket: 347 reason# EndOfStream
2025-11-19T14:00:29.223565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436082573327581:2050] [node 3] ICP30 unregister session Session# [5:7574436086868294958:2050] VirtualId# [5:7574436086868294955:0]
2025-11-19T14:00:29.223572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436086868294958:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.223627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:29.223663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:29.240104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:29.240239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436089276808133:2071] [node 4] ICS01 socket: 376 reason# EndOfStream
2025-11-19T14:00:29.240254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436084981840783:2071] [node 4] ICP30 unregister session Session# [6:7574436089276808133:2071] VirtualId# [6:7574436089276808124:0]
2025-11-19T14:00:29.240263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436089276808133:2071] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.240324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:29.240353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:29.324587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436085136504910:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:29.324604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436085136504911:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:29.379979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436087532351054:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:29.379996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436087532351055:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:29.416141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436085293964554:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:29.416159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436085293964555:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:29.455917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436088471604018:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:29.455941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436088471604019:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:29.455945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436088471604020:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:29.503900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436087826137577:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:29.503917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436087826137578:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:29.503923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436087826137579:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:29.535857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436086816588769:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:29.535871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436086816588770:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:29.535876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436086816588771:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:29.587317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436088471604045:2072] [node 5] ICP01 ready to work
2025-11-19T14:00:29.587323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436085293964607:2096] [node 5] ICP01 ready to work
2025-11-19T14:00:29.587468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436085293964608:2099] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.593513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436087826137596:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22133
2025-11-19T14:00:29.593540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436087826137597:2067] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22133
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.594116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436085136504970:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40787
2025-11-19T14:00:29.594513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436087826137610:2070] [node 6] ICP01 ready to work
2025-11-19T14:00:29.594616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436087826137611:2072] [node 4] ICP01 ready to work
2025-11-19T14:00:29.595139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436087532351115:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.594604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436086816588787:2061] [node 5] ICP01 ready to work
2025-11-19T14:00:29.594709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436086816588788:2063] [node 4] ICP01 ready to work
2025-11-19T14:00:29.600734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436085293964624:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:29.611778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436086816588806:2076] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21542
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.617033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436087826137647:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22133
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.622650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436085293964632:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57887
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.620237Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436088471604019:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(31780621ns) {Configure:104(31779726ns) {ProcessConfigured:135(31487330ns) {StartInitialHandshake:157(31486485ns) {DropHandshakes:469(1092ns) {DropIncomingHandshake:441(244ns) DropOutgoingHandshake:456(70ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:476(1566ns) {SwitchToState:178(925ns)}}}}}
2025-11-19T14:00:29.621202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436088471604092:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46839
2025-11-19T14:00:29.621264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436088471604093:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46839
2025-11-19T14:00:29.621297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436088471604094:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46839
2025-11-19T14:00:29.624058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436085136504977:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:29.625172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436085136504976:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40787
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.625513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436087826137610:2070] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(30847824ns) {Configure:104(30847384ns) {ProcessConfigured:135(30586439ns) {StartInitialHandshake:157(30586035ns) {DropHandshakes:469(524ns) {DropIncomingHandshake:441(80ns) DropOutgoingHandshake:456(85ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:476(544ns) {SwitchToState:178(365ns)}}}}}
2025-11-19T14:00:29.625655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436087532351055:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(28189053ns) {Configure:104(28188324ns) {ProcessConfigured:135(27878457ns) {StartInitialHandshake:157(27878341ns) {DropHandshakes:469(687ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(166ns) PrepareNewSessionHandshake:476(1112ns) {SwitchToState:178(84ns)}}}}}
2025-11-19T14:00:29.625763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436087532351124:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:29.630211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436085136504973:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:29.625734Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436088471604092:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.625766Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436088471604093:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.625871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436088471604020:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.625887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436088471604097:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46839
2025-11-19T14:00:29.630386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436088471604059:2081] [node 6] ICP01 ready to work
2025-11-19T14:00:29.631800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436087532351120:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:29.630965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436085136504974:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:29.636453Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436085293964608:2099] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(48186377ns) {Configure:104(47961326ns) {ProcessConfigured:135(47685053ns) {StartInitialHandshake:157(47684574ns) {DropHandshakes:469(502ns) {DropIncomingHandshake:441(150ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:476(1419ns) {SwitchToState:178(773ns)}}}}}
2025-11-19T14:00:29.636518Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436085293964626:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.636523Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436085293964626:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.636011Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436086816588806:2076] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.636070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436086816588770:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.638885Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087532351117:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.638899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436087532351117:2108] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.639531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436087532351132:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20513
2025-11-19T14:00:29.639569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436087532351133:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20513
2025-11-19T14:00:29.639609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436087532351134:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20513
2025-11-19T14:00:29.639620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436087532351135:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20513
2025-11-19T14:00:29.640537Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436085293964626:2111] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:29.647868Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087532351133:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.647961Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436087532351117:2108] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:29.648400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436085293964624:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.648430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:29.648441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436085293964624:2108] [node 4] ICP22 created new session: [3:7574436085293964635:2108]
2025-11-19T14:00:29.648449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436085293964635:2108] [node 4] ICS09 handshake done sender: [3:7574436085293964626:2111] self: [3:7574436085293964625:0] peer: [4:7574436088471604099:0] socket: 361 qp: -1
2025-11-19T14:00:29.648459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436085293964635:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:29.648471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:29.649287Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436085136504973:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(18934923ns) {Configure:104(18934478ns) {ProcessConfigured:135(18501977ns) {StartInitialHandshake:157(18501543ns) {DropHandshakes:469(768ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(161ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(1195ns) {SwitchToState:178(143ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.650584Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436088471604059:2081] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(20023597ns) {Configure:104(20022934ns) {ProcessConfigured:135(19675956ns) {StartInitialHandshake:157(19675411ns) {DropHandshakes:469(504ns) {DropIncomingHandshake:441(73ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(866ns) {SwitchToState:178(374ns)}}}}}
2025-11-19T14:00:29.650799Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085136504970:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.650879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436085136504990:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40787
2025-11-19T14:00:29.650919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436085136504991:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40787
2025-11-19T14:00:29.651043Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436085136504974:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19946181ns) {Configure:104(19945849ns) {ProcessConfigured:135(19672073ns) {StartInitialHandshake:157(19671487ns) {DropHandshakes:469(290ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:476(883ns) {SwitchToState:178(79ns)}}}}}
2025-11-19T14:00:29.649047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436086816588771:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(52832000ns) {Configure:104(52831224ns) {ProcessConfigured:135(52606619ns) {StartInitialHandshake:157(52605736ns) {DropHandshakes:469(416ns) {DropIncomingHandshake:441(29ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(147ns) PrepareNewSessionHandshake:476(1168ns) {SwitchToState:178(366ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.651948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436085293964638:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57887
2025-11-19T14:00:29.651970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436085293964639:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57887
2025-11-19T14:00:29.653209Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436087826137597:2067] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.653279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436087826137579:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.653635Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087532351132:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.653672Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085136504991:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.653799Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085136504990:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.653797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436087532351120:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21641466ns) {Configure:104(21640489ns) {ProcessConfigured:135(21310727ns) {StartInitialHandshake:157(21310113ns) {DropHandshakes:469(581ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(47ns)} GenerateSessionVirtualId:988(271ns) PrepareNewSessionHandshake:476(1257ns) {SwitchToState:178(971ns)}}}}}
2025-11-19T14:00:29.654317Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436085293964613:2101] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.654328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436085293964613:2101] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.659918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436087532351115:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.659963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:29.659976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436087532351115:2107] [node 6] ICP22 created new session: [2:7574436087532351141:2107]
2025-11-19T14:00:29.659995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436087532351141:2107] [node 6] ICS09 handshake done sender: [2:7574436087532351117:2108] self: [2:7574436087532351116:0] peer: [6:7574436086816588847:0] socket: 354 qp: -1
2025-11-19T14:00:29.660010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436087532351141:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:29.660025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.676689Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436085293964613:2101] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:29.676785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436085293964607:2096] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.676802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:29.676809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436085293964607:2096] [node 5] ICP22 created new session: [3:7574436085293964641:2096]
2025-11-19T14:00:29.676816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436085293964641:2096] [node 5] ICS09 handshake done sender: [3:7574436085293964613:2101] self: [3:7574436085293964609:0] peer: [5:7574436087826137658:0] socket: 349 qp: -1
2025-11-19T14:00:29.676824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436085293964641:2096] [node 5] ICS10 traffic start
2025-11-19T14:00:29.676834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:29.676343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436088471604102:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46839
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.678962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436088471604103:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46839
2025-11-19T14:00:29.679019Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436088471604093:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.686301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436087826137659:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22133
2025-11-19T14:00:29.686668Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436085293964638:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.686718Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436085293964639:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.686757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436085293964554:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.686769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436085293964555:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.688907Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436087826137597:2067] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:29.689275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436086816588849:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21542
2025-11-19T14:00:29.690045Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085136504976:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.689762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436087826137579:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.689788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:29.689797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436087826137579:2050] [node 3] ICP22 created new session: [5:7574436087826137660:2050]
2025-11-19T14:00:29.689805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436087826137660:2050] [node 3] ICS09 handshake done sender: [5:7574436087826137597:2067] self: [5:7574436087826137658:0] peer: [3:7574436085293964609:0] socket: 351 qp: -1
2025-11-19T14:00:29.689814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436087826137660:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:29.689834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:29.690743Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087532351123:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.690758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436087532351123:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.691367Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087532351135:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.691392Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087532351134:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.691530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436087532351054:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.691910Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436087532351123:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:29.690249Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436088471604097:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.690329Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436088471604103:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.690335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436088471604020:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.690346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:29.690351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436088471604020:2050] [node 3] ICP22 created new session: [4:7574436088471604104:2050]
2025-11-19T14:00:29.690358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436088471604104:2050] [node 3] ICS09 handshake done sender: [4:7574436088471604093:2115] self: [4:7574436088471604099:0] peer: [3:7574436085293964625:0] socket: 367 qp: -1
2025-11-19T14:00:29.690364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436088471604104:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:29.690372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:29.692053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436087532351055:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.692072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:29.692078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436087532351055:2049] [node 3] ICP22 created new session: [2:7574436087532351146:2049]
2025-11-19T14:00:29.692085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436087532351146:2049] [node 3] ICS09 handshake done sender: [2:7574436087532351123:2111] self: [2:7574436087532351121:0] peer: [3:7574436085293964645:0] socket: 376 qp: -1
2025-11-19T14:00:29.692090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436087532351146:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:29.692097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:29.692967Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436085293964555:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8528) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436085293964639:2117] held: no
2025-11-19T14:00:29.692740Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085136504979:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.692752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436085136504979:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.693378Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436086816588806:2076] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:29.694754Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436088471604094:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.694887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436088471604018:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.694900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436088471604019:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.696808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436086816588770:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.696831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:29.696841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436086816588770:2049] [node 2] ICP22 created new session: [6:7574436086816588851:2049]
2025-11-19T14:00:29.696851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436086816588851:2049] [node 2] ICS09 handshake done sender: [6:7574436086816588806:2076] self: [6:7574436086816588847:0] peer: [2:7574436087532351116:0] socket: 357 qp: -1
2025-11-19T14:00:29.696858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436086816588851:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:29.696895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.697087Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436087826137647:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.697359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436087826137662:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22133
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.698591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436086816588850:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21542
2025-11-19T14:00:29.698698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436086816588853:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21542
2025-11-19T14:00:29.699534Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085136504981:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.699551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436085136504981:2113] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.699959Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085136504983:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.699968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436085136504983:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.700855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436085293964646:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57887
2025-11-19T14:00:29.700993Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436086816588850:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.702287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436086816588771:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.700357Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436087826137596:2066] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-11-19T14:00:29.701639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436087826137611:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-11-19T14:00:29.703667Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436085136504979:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:29.703838Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436085136504983:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:29.703788Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436088471604047:2074] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.703800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436088471604047:2074] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.704347Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436085136504981:2113] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:29.704516Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087532351128:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.704533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436087532351128:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.704615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436085293964647:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57887
2025-11-19T14:00:29.704648Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436085293964633:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.704652Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436085293964633:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.704878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436085136504911:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.704901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:29.704908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436085136504911:2049] [node 3] ICP22 created new session: [1:7574436085136505000:2049]
2025-11-19T14:00:29.704915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436085136505000:2049] [node 3] ICS09 handshake done sender: [1:7574436085136504981:2113] self: [1:7574436085136504980:0] peer: [3:7574436085293964644:0] socket: 373 qp: -1
2025-11-19T14:00:29.704921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436085136505000:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:29.704928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.705037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436087826137666:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22133
2025-11-19T14:00:29.705058Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436086816588853:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.705085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436086816588787:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.705216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436087532351149:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20513
2025-11-19T14:00:29.705358Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436087532351135:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:29.705555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436087532351054:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.705568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:29.705574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436087532351054:2048] [node 1] ICP22 created new session: [2:7574436087532351150:2048]
2025-11-19T14:00:29.705579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436087532351150:2048] [node 1] ICS09 handshake done sender: [2:7574436087532351135:2117] self: [2:7574436087532351144:0] peer: [1:7574436085136504978:0] socket: 383 qp: -1
2025-11-19T14:00:29.705584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436087532351150:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:29.705590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:29.705467Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436088471604047:2074] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:29.705672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436088471604045:2072] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.705687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:29.705692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436088471604045:2072] [node 5] ICP22 created new session: [4:7574436088471604111:2072]
2025-11-19T14:00:29.705697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436088471604111:2072] [node 5] ICS09 handshake done sender: [4:7574436088471604047:2074] self: [4:7574436088471604046:0] peer: [5:7574436087826137665:0] socket: 348 qp: -1
2025-11-19T14:00:29.705701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436088471604111:2072] [node 5] ICS10 traffic start
2025-11-19T14:00:29.705725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:29.705389Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436087826137651:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.705398Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436087826137651:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.705764Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436087826137662:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.705896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436087826137578:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.705925Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436087826137596:2066] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:29.706073Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436087826137651:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:29.706128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436087826137611:2072] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.706136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:29.706140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436087826137611:2072] [node 4] ICP22 created new session: [5:7574436087826137669:2072]
2025-11-19T14:00:29.706145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436087826137669:2072] [node 4] ICS09 handshake done sender: [5:7574436087826137596:2066] self: [5:7574436087826137665:0] peer: [4:7574436088471604046:0] socket: 350 qp: -1
2025-11-19T14:00:29.706149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436087826137669:2072] [node 4] ICS10 traffic start
2025-11-19T14:00:29.706153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:29.706859Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436085293964639:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:29.707131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436087826137579:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(17948835ns) {HandleHandshakeStatus:329(17947945ns) {DropIncomingHandshake:441(372ns) DropOutgoingHandshake:456(2802ns) SwitchToState:178(392ns) ProcessPendingSessionEvents:495(7096028ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7072456ns max# 6983046ns}}}
2025-11-19T14:00:29.707164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436087826137610:2070] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.707178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:29.707183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436087826137610:2070] [node 6] ICP22 created new session: [5:7574436087826137671:2070]
2025-11-19T14:00:29.707187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436087826137671:2070] [node 6] ICS09 handshake done sender: [5:7574436087826137651:2111] self: [5:7574436087826137612:0] peer: [6:7574436086816588856:0] socket: 399 qp: -1
2025-11-19T14:00:29.707193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436087826137671:2070] [node 6] ICS10 traffic start
2025-11-19T14:00:29.707199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:29.707994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436085136504910:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.708014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:29.708020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436085136504910:2048] [node 2] ICP22 created new session: [1:7574436085136505002:2048]
2025-11-19T14:00:29.708025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436085136505002:2048] [node 2] ICS09 handshake done sender: [1:7574436085136504979:2112] self: [1:7574436085136504978:0] peer: [2:7574436087532351144:0] socket: 372 qp: -1
2025-11-19T14:00:29.708030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436085136505002:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:29.708035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:29.708236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436085136504977:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.708242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:29.708246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436085136504977:2110] [node 4] ICP22 created new session: [1:7574436085136505004:2110]
2025-11-19T14:00:29.708251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436085136505004:2110] [node 4] ICS09 handshake done sender: [1:7574436085136504983:2114] self: [1:7574436085136504982:0] peer: [4:7574436088471604106:0] socket: 374 qp: -1
2025-11-19T14:00:29.708254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436085136505004:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:29.708259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.709548Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436085293964647:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.709563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436085293964555:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.709572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:29.709579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436085293964555:2049] [node 2] ICP22 created new session: [3:7574436085293964649:2049]
2025-11-19T14:00:29.709584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436085293964649:2049] [node 2] ICS09 handshake done sender: [3:7574436085293964639:2117] self: [3:7574436085293964645:0] peer: [2:7574436087532351121:0] socket: 389 qp: -1
2025-11-19T14:00:29.709590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436085293964649:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:29.709597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:29.709996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436085293964648:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57887
2025-11-19T14:00:29.711936Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436085293964638:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:29.713307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436085293964554:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.713325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:29.713331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436085293964554:2048] [node 1] ICP22 created new session: [3:7574436085293964652:2048]
2025-11-19T14:00:29.713337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436085293964652:2048] [node 1] ICS09 handshake done sender: [3:7574436085293964638:2116] self: [3:7574436085293964644:0] peer: [1:7574436085136504980:0] socket: 379 qp: -1
2025-11-19T14:00:29.713343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436085293964652:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:29.713349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:29.714484Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436087532351137:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.714497Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436087532351137:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.716671Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436087532351128:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:29.716806Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436087532351137:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:29.716922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436087532351124:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.716940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:29.716945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436087532351124:2110] [node 4] ICP22 created new session: [2:7574436087532351154:2110]
2025-11-19T14:00:29.716949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436087532351154:2110] [node 4] ICS09 handshake done sender: [2:7574436087532351128:2113] self: [2:7574436087532351127:0] peer: [4:7574436088471604107:0] socket: 390 qp: -1
2025-11-19T14:00:29.716955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436087532351154:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:29.716962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:29.717146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436087532351120:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.717156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:29.717159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436087532351120:2109] [node 5] ICP22 created new session: [2:7574436087532351156:2109]
2025-11-19T14:00:29.717162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436087532351156:2109] [node 5] ICS09 handshake done sender: [2:7574436087532351137:2118] self: [2:7574436087532351131:0] peer: [5:7574436087826137668:0] socket: 397 qp: -1
2025-11-19T14:00:29.717165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436087532351156:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:29.717168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:29.717530Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436085293964633:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:29.717785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436085293964608:2099] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.717800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:29.717804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436085293964608:2099] [node 6] ICP22 created new session: [3:7574436085293964655:2099]
2025-11-19T14:00:29.717809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436085293964655:2099] [node 6] ICS09 handshake done sender: [3:7574436085293964633:2115] self: [3:7574436085293964610:0] peer: [6:7574436086816588855:0] socket: 393 qp: -1
2025-11-19T14:00:29.717813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436085293964655:2099] [node 6] ICS10 traffic start
2025-11-19T14:00:29.717818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.716460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436087826137673:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22133
2025-11-19T14:00:29.716505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436087826137674:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22133
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.719137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436086816588857:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21542
2025-11-19T14:00:29.719445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436088471604110:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46839
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.719697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436086816588858:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21542
2025-11-19T14:00:29.719716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436086816588859:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21542
2025-11-19T14:00:29.721231Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436087826137673:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.721255Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436087826137662:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:29.721450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436087826137577:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.721591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436087826137578:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.721605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:29.721612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436087826137578:2049] [node 2] ICP22 created new session: [5:7574436087826137676:2049]
2025-11-19T14:00:29.721617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436087826137676:2049] [node 2] ICS09 handshake done sender: [5:7574436087826137662:2119] self: [5:7574436087826137668:0] peer: [2:7574436087532351131:0] socket: 398 qp: -1
2025-11-19T14:00:29.721624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436087826137676:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:29.721632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:29.722848Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085136504992:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.722861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436085136504992:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.723873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436087826137678:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22133
2025-11-19T14:00:29.724474Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436085136504992:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:29.724950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574436085293964624:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:29.725595Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436088471604019:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8528) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436088471604103:2120] held: no
2025-11-19T14:00:29.725692Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436088471604097:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:29.726156Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436087826137673:2121] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:29.726262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436087826137577:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.726275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:29.726280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436087826137577:2048] [node 1] ICP22 created new session: [5:7574436087826137679:2048]
2025-11-19T14:00:29.726285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436087826137679:2048] [node 1] ICS09 handshake done sender: [5:7574436087826137673:2121] self: [5:7574436087826137675:0] peer: [1:7574436085136504988:0] socket: 388 qp: -1
2025-11-19T14:00:29.726290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436087826137679:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:29.726296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:29.726601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436085136504974:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.726616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:29.726620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436085136504974:2109] [node 5] ICP22 created new session: [1:7574436085136505009:2109]
2025-11-19T14:00:29.726625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436085136505009:2109] [node 5] ICS09 handshake done sender: [1:7574436085136504992:2118] self: [1:7574436085136504988:0] peer: [5:7574436087826137675:0] socket: 413 qp: -1
2025-11-19T14:00:29.726629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436085136505009:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:29.726634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:29.726783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436086816588787:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30038) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436086816588853:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.734402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436088471604018:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.734435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:29.734445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436088471604018:2048] [node 1] ICP22 created new session: [4:7574436088471604116:2048]
2025-11-19T14:00:29.734451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436088471604116:2048] [node 1] ICS09 handshake done sender: [4:7574436088471604097:2117] self: [4:7574436088471604106:0] peer: [1:7574436085136504982:0] socket: 375 qp: -1
2025-11-19T14:00:29.734460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436088471604116:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:29.734468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:29.734852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436088471604115:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46839
2025-11-19T14:00:29.731836Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436086816588850:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:29.734997Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436086816588858:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.735119Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436086816588853:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.735903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436086816588771:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8162) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436086816588850:2117] held: no
2025-11-19T14:00:29.735988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436086816588771:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.735998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:29.736002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436086816588771:2050] [node 3] ICP22 created new session: [6:7574436086816588861:2050]
2025-11-19T14:00:29.736007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436086816588861:2050] [node 3] ICS09 handshake done sender: [6:7574436086816588850:2117] self: [6:7574436086816588855:0] peer: [3:7574436085293964610:0] socket: 395 qp: -1
2025-11-19T14:00:29.736013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436086816588861:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:29.736019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:29.736323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436086816588860:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21542
2025-11-19T14:00:29.736374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436086816588769:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.737850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436086816588787:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.737863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:29.737869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436086816588787:2061] [node 5] ICP22 created new session: [6:7574436086816588865:2061]
2025-11-19T14:00:29.737875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436086816588865:2061] [node 5] ICS09 handshake done sender: [6:7574436086816588853:2118] self: [6:7574436086816588856:0] peer: [5:7574436087826137612:0] socket: 400 qp: -1
2025-11-19T14:00:29.737880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436086816588865:2061] [node 5] ICS10 traffic start
2025-11-19T14:00:29.737885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:29.738101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436086816588864:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21542
2025-11-19T14:00:29.738389Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436086816588858:2120] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:29.738468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436086816588769:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.738480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:29.738484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436086816588769:2048] [node 1] ICP22 created new session: [6:7574436086816588867:2048]
2025-11-19T14:00:29.738488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436086816588867:2048] [node 1] ICS09 handshake done sender: [6:7574436086816588858:2120] self: [6:7574436086816588863:0] peer: [1:7574436085136504987:0] socket: 377 qp: -1
2025-11-19T14:00:29.738492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436086816588867:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:29.738497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:29.736925Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436085136504989:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.736938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436085136504989:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:29.737503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436085136505011:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40787
2025-11-19T14:00:29.738233Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436085136504989:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:29.739606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436085136504973:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.739619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:29.739625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436085136504973:2108] [node 6] ICP22 created new session: [1:7574436085136505013:2108]
2025-11-19T14:00:29.739631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436085136505013:2108] [node 6] ICS09 handshake done sender: [1:7574436085136504989:2115] self: [1:7574436085136504987:0] peer: [6:7574436086816588863:0] socket: 412 qp: -1
2025-11-19T14:00:29.739635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436085136505013:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:29.739641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:29.741485Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436086816588860:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.741523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436086816588788:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:29.741713Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436088471604101:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:29.741725Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436088471604101:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:29.742611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436086816588870:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21542
2025-11-19T14:00:29.742781Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436088471604101:2118] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:29.743196Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436086816588860:2122] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:29.742965Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574436085136504977:2110] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:29.743273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436086816588788:2063] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.743286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:29.743293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436086816588788:2063] [node 4] ICP22 created new session: [6:7574436086816588871:2063]
2025-11-19T14:00:29.743298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436086816588871:2063] [node 4] ICS09 handshake done sender: [6:7574436086816588860:2122] self: [6:7574436086816588869:0] peer: [4:7574436088471604100:0] socket: 381 qp: -1
2025-11-19T14:00:29.743303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436086816588871:2063] [node 4] ICS10 traffic start
2025-11-19T14:00:29.743311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:29.744400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436088471604059:2081] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:29.744419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:29.744425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436088471604059:2081] [node 6] ICP22 created new session: [4:7574436088471604119:2081]
2025-11-19T14:00:29.744431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436088471604119:2081] [node 6] ICS09 handshake done sender: [4:7574436088471604101:2118] self: [4:7574436088471604100:0] peer: [6:7574436086816588869:0] socket: 366 qp: -1
2025-11-19T14:00:29.744436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436088471604119:2081] [node 6] ICS10 traffic start
2025-11-19T14:00:29.744459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:29.751984Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436088471604103:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:29.752173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436088471604019:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:29.752191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:29.752202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436088471604019:2049] [node 2] ICP22 created new session: [4:7574436088471604121:2049]
2025-11-19T14:00:29.752208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436088471604121:2049] [node 2] ICS09 handshake done sender: [4:7574436088471604103:2120] self: [4:7574436088471604107:0] peer: [2:7574436087532351127:0] socket: 392 qp: -1
2025-11-19T14:00:29.752216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436088471604121:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:29.752230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:29.760517Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:29.760626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436087532351150:2048] [node 1] ICS07 socket disconnect 383 reason# EndOfStream
2025-11-19T14:00:29.760632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436087532351150:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.760666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:29.760683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436087532351150:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:29.761224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:29.761306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436085293964652:2048] [node 1] ICS01 socket: 379 reason# EndOfStream
2025-11-19T14:00:29.761312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436085293964554:2048] [node 1] ICP30 unregister session Session# [3:7574436085293964652:2048] VirtualId# [3:7574436085293964644:0]
2025-11-19T14:00:29.761317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436085293964652:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.761340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:29.761360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:29.763426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436087532351054:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21407) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T14:00:29.763440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436087532351150:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:29.763798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436087532351054:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21407) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:29.763810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436087532351150:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:29.763816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436087532351054:2048] [node 1] ICP30 unregister session Session# [2:7574436087532351150:2048] VirtualId# [2:7574436087532351144:0]
2025-11-19T14:00:29.763876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:29.763883Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574436087532351054:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21407) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:29.763887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21407) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:29.767051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:29.767120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436087826137679:2048] [node 1] ICS01 socket: 388 reason# EndOfStream
2025-11-19T14:00:29.767130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436087826137577:2048] [node 1] ICP30 unregister session Session# [5:7574436087826137679:2048] VirtualId# [5:7574436087826137675:0]
2025-11-19T14:00:29.767134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436087826137679:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.767161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:29.766845Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:29.766931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436086816588867:2048] [node 1] ICS01 socket: 377 reason# EndOfStream
2025-11-19T14:00:29.766938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436086816588769:2048] [node 1] ICP30 unregister session Session# [6:7574436086816588867:2048] VirtualId# [6:7574436086816588863:0]
2025-11-19T14:00:29.766944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436086816588867:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.766991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:29.767016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:29.767215Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:29.767298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436088471604116:2048] [node 1] ICS01 socket: 375 reason# EndOfStream
2025-11-19T14:00:29.767303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436088471604018:2048] [node 1] ICP30 unregister session Session# [4:7574436088471604116:2048] VirtualId# [4:7574436088471604106:0]
2025-11-19T14:00:29.767308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436088471604116:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.767343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:29.767366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:29.767180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:29.832533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:29.832649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436085293964649:2049] [node 2] ICS01 socket: 389 reason# EndOfStream
2025-11-19T14:00:29.832661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436085293964555:2049] [node 2] ICP30 unregister session Session# [3:7574436085293964649:2049] VirtualId# [3:7574436085293964645:0]
2025-11-19T14:00:29.832671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436085293964649:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.832704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:29.832723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:29.848939Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:29.849050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436086816588851:2049] [node 2] ICS01 socket: 357 reason# EndOfStream
2025-11-19T14:00:29.849061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436086816588770:2049] [node 2] ICP30 unregister session Session# [6:7574436086816588851:2049] VirtualId# [6:7574436086816588847:0]
2025-11-19T14:00:29.849070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436086816588851:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.849119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:29.849142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:29.849450Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:29.849508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436088471604121:2049] [node 2] ICS01 socket: 392 reason# EndOfStream
2025-11-19T14:00:29.849512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436088471604019:2049] [node 2] ICP30 unregister session Session# [4:7574436088471604121:2049] VirtualId# [4:7574436088471604107:0]
2025-11-19T14:00:29.849516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436088471604121:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.849542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:29.849562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:29.849272Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:29.849336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436087826137676:2049] [node 2] ICS01 socket: 398 reason# EndOfStream
2025-11-19T14:00:29.849361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436087826137578:2049] [node 2] ICP30 unregister session Session# [5:7574436087826137676:2049] VirtualId# [5:7574436087826137668:0]
2025-11-19T14:00:29.849365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436087826137676:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.849397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:29.849417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:29.869344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:29.869459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574436087826137660:2050] [node 3] ICS07 socket disconnect 351 reason# EndOfStream
2025-11-19T14:00:29.869467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436087826137660:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.869522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:29.869544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574436087826137660:2050] [node 3] ICS15 start handshake
2025-11-19T14:00:29.872963Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:29.873074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436086816588861:2050] [node 3] ICS01 socket: 395 reason# EndOfStream
2025-11-19T14:00:29.873083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436086816588771:2050] [node 3] ICP30 unregister session Session# [6:7574436086816588861:2050] VirtualId# [6:7574436086816588855:0]
2025-11-19T14:00:29.873093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436086816588861:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.873147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:29.873174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:29.873731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436087826137579:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8162) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:29.873750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436087826137660:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:29.873755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436087826137579:2050] [node 3] ICP30 unregister session Session# [5:7574436087826137660:2050] VirtualId# [5:7574436087826137658:0]
2025-11-19T14:00:29.873820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:29.873827Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574436087826137579:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8162) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:29.873830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8162) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:29.887863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574436087826137579:2050] [node 3] ICP33 wake up from error state
2025-11-19T14:00:29.903672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:29.903811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436086816588871:2063] [node 4] ICS01 socket: 381 reason# EndOfStream
2025-11-19T14:00:29.903823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436086816588788:2063] [node 4] ICP30 unregister session Session# [6:7574436086816588871:2063] VirtualId# [6:7574436086816588869:0]
2025-11-19T14:00:29.903835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436086816588871:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.903907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:29.903932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:29.916021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T14:00:29.916145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436086816588865:2061] [node 5] ICS01 socket: 400 reason# EndOfStream
2025-11-19T14:00:29.916164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436086816588787:2061] [node 5] ICP30 unregister session Session# [6:7574436086816588865:2061] VirtualId# [6:7574436086816588856:0]
2025-11-19T14:00:29.916175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436086816588865:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:29.916229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T14:00:29.916258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:30.027989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436090547748355:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:30.028002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436090547748356:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:30.092167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436093243927485:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:30.092179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436093243927486:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:30.167981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436093298127204:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:30.167996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436093298127205:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:30.213129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436091636361002:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:30.213141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436091636361003:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:30.213146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436091636361004:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:30.292180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436091705063593:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:30.292195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436091705063594:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:30.292200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436091705063595:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:30.362972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436091047138496:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:30.362984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436091047138497:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:30.362988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436091047138498:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:30.418108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436091047138509:2059] [node 5] ICP01 ready to work
2025-11-19T14:00:30.419546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436091705063646:2099] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.420121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436091705063650:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57616
2025-11-19T14:00:30.420354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436091636361064:2102] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.427870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436091636361066:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62752
2025-11-19T14:00:30.424590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436090547748418:2107] [node 6] ICP01 ready to work
2025-11-19T14:00:30.425199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436090547748423:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:30.425313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436090547748426:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.428586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436093243927548:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:30.429723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436093298127268:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17958
2025-11-19T14:00:30.432490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436091705063660:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:30.432179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436093298127272:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.436239Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436091705063646:2099] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(16379613ns) {Configure:104(16352917ns) {ProcessConfigured:135(15620820ns) {StartInitialHandshake:157(15620223ns) {DropHandshakes:469(855ns) {DropIncomingHandshake:441(162ns) DropOutgoingHandshake:456(146ns)} GenerateSessionVirtualId:988(585ns) PrepareNewSessionHandshake:476(949ns) {SwitchToState:178(400ns)}}}}}
2025-11-19T14:00:30.436996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436091705063594:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17394372ns) {Configure:104(17393945ns) {ProcessConfigured:135(16208419ns) {StartInitialHandshake:157(16208127ns) {DropHandshakes:469(194ns) {DropIncomingHandshake:441(105ns) DropOutgoingHandshake:456(21ns)} GenerateSessionVirtualId:988(64ns) PrepareNewSessionHandshake:476(780ns) {SwitchToState:178(630ns)}}}}}
2025-11-19T14:00:30.439057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436091705063669:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57616
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.437148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436091636361064:2102] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(16694666ns) {Configure:104(16694184ns) {ProcessConfigured:135(16268173ns) {StartInitialHandshake:157(16267926ns) {DropHandshakes:469(700ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:476(526ns) {SwitchToState:178(353ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.439963Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436091047138496:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(20051374ns) {Configure:104(20050574ns) {ProcessConfigured:135(19723486ns) {StartInitialHandshake:157(19723192ns) {DropHandshakes:469(745ns) {DropIncomingHandshake:441(165ns) DropOutgoingHandshake:456(164ns)} GenerateSessionVirtualId:988(607ns) PrepareNewSessionHandshake:476(929ns) {SwitchToState:178(231ns)}}}}}
2025-11-19T14:00:30.440007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436091047138531:2075] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24581
2025-11-19T14:00:30.440600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436091047138563:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24581
2025-11-19T14:00:30.440610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436091047138564:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24581
2025-11-19T14:00:30.440877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436091047138497:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17049285ns) {Configure:104(17048601ns) {ProcessConfigured:135(16722359ns) {StartInitialHandshake:157(16722071ns) {DropHandshakes:469(487ns) {DropIncomingHandshake:441(33ns) DropOutgoingHandshake:456(120ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(646ns) {SwitchToState:178(383ns)}}}}}
2025-11-19T14:00:30.441425Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436091705063650:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.452688Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436091636361004:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(31772826ns) {Configure:104(22580871ns) {ProcessConfigured:135(22177325ns) {StartInitialHandshake:157(22176989ns) {DropHandshakes:469(657ns) {DropIncomingHandshake:441(74ns) DropOutgoingHandshake:456(115ns)} GenerateSessionVirtualId:988(425ns) PrepareNewSessionHandshake:476(657ns) {SwitchToState:178(484ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.455225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436093243927560:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14148
2025-11-19T14:00:30.455275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436093243927561:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14148
2025-11-19T14:00:30.455286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436093243927562:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14148
2025-11-19T14:00:30.455298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436093243927563:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14148
2025-11-19T14:00:30.455588Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436091047138531:2075] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.455631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436091047138496:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:30.455800Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093243927560:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.455826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436093243927485:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:30.455881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436093298127272:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(23648728ns) {Configure:104(23648111ns) {ProcessConfigured:135(23338476ns) {StartInitialHandshake:157(23338224ns) {DropHandshakes:469(440ns) {DropIncomingHandshake:441(53ns) DropOutgoingHandshake:456(99ns)} GenerateSessionVirtualId:988(56ns) PrepareNewSessionHandshake:476(881ns) {SwitchToState:178(569ns)}}}}}
2025-11-19T14:00:30.456033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436093298127278:2112] [node 6] ICP01 ready to work
2025-11-19T14:00:30.455937Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436090547748420:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.455949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436090547748420:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:30.456320Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436090547748417:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.456324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436090547748417:2108] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.459861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436093243927558:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:30.460374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436093243927559:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:30.458525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436090547748436:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
2025-11-19T14:00:30.459217Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436090547748420:2109] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:30.460030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436091636361069:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:30.460200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436091047138537:2065] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.468373Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436093298127268:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.468569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436093298127280:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17958
2025-11-19T14:00:30.468580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436093298127281:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17958
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.471864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436091636361002:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(23706079ns) {Configure:104(23704629ns) {ProcessConfigured:135(23313952ns) {StartInitialHandshake:157(23313380ns) {DropHandshakes:469(777ns) {DropIncomingHandshake:441(157ns) DropOutgoingHandshake:456(151ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(1330ns) {SwitchToState:178(161ns)}}}}}
2025-11-19T14:00:30.472126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436091705063595:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(23375148ns) {Configure:104(23374256ns) {ProcessConfigured:135(22886295ns) {StartInitialHandshake:157(22885478ns) {DropHandshakes:469(683ns) {DropIncomingHandshake:441(112ns) DropOutgoingHandshake:456(107ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:476(1251ns) {SwitchToState:178(893ns)}}}}}
2025-11-19T14:00:30.472973Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436091705063660:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(40406322ns) {Configure:104(40405823ns) {ProcessConfigured:135(40037653ns) {StartInitialHandshake:157(40036837ns) {DropHandshakes:469(644ns) {DropIncomingHandshake:441(81ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:476(687ns) {SwitchToState:178(444ns)}}}}}
2025-11-19T14:00:30.479926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436093298127276:2111] [node 4] ICP01 ready to work
2025-11-19T14:00:30.479885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436090547748437:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
2025-11-19T14:00:30.479993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436090547748438:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
2025-11-19T14:00:30.480094Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436090547748417:2108] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:30.480124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436090547748439:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
2025-11-19T14:00:30.480280Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436091636361066:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.480338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436091636361002:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.483305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436090547748355:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.483327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:30.483339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436090547748355:2048] [node 2] ICP22 created new session: [1:7574436090547748441:2048]
2025-11-19T14:00:30.483346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436090547748441:2048] [node 2] ICS09 handshake done sender: [1:7574436090547748417:2108] self: [1:7574436090547748416:0] peer: [2:7574436093243927564:0] socket: 356 qp: -1
2025-11-19T14:00:30.483354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436090547748441:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:30.483377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:30.482927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436091705063673:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57616
2025-11-19T14:00:30.483213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436091047138577:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24581
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.485493Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436091705063673:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.485549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436091705063675:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57616
2025-11-19T14:00:30.485602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436091705063595:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:30.485953Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436091047138531:2075] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:30.486121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436091047138496:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.486140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:30.486146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436091047138496:2048] [node 1] ICP22 created new session: [6:7574436091047138578:2048]
2025-11-19T14:00:30.486152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436091047138578:2048] [node 1] ICS09 handshake done sender: [6:7574436091047138531:2075] self: [6:7574436091047138575:0] peer: [1:7574436090547748419:0] socket: 362 qp: -1
2025-11-19T14:00:30.486158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436091047138578:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:30.486166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:30.487349Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436093243927558:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(27331594ns) {Configure:104(27330774ns) {ProcessConfigured:135(26951189ns) {StartInitialHandshake:157(26950388ns) {DropHandshakes:469(554ns) {DropIncomingHandshake:441(119ns) DropOutgoingHandshake:456(138ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(1412ns) {SwitchToState:178(925ns)}}}}}
2025-11-19T14:00:30.487579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436091636361069:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27411777ns) {Configure:104(27411239ns) {ProcessConfigured:135(27055300ns) {StartInitialHandshake:157(27054881ns) {DropHandshakes:469(551ns) {DropIncomingHandshake:441(91ns) DropOutgoingHandshake:456(123ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:476(1387ns) {SwitchToState:178(710ns)}}}}}
2025-11-19T14:00:30.487711Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436091047138537:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(27392587ns) {Configure:104(27392260ns) {ProcessConfigured:135(27045911ns) {StartInitialHandshake:157(27045677ns) {DropHandshakes:469(435ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:476(732ns) {SwitchToState:178(35ns)}}}}}
2025-11-19T14:00:30.488095Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436090547748428:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.488114Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436090547748428:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.489328Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093243927561:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.489590Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436091705063675:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.489679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436091705063593:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:30.488317Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436090547748439:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.488361Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436090547748436:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.491674Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436090547748433:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.491680Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436090547748433:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:30.488455Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436093298127277:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.488468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436093298127277:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:30.489001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436093298127286:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17958
2025-11-19T14:00:30.489595Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436093298127277:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:30.489727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436093298127272:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.489737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:30.489744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436093298127272:2110] [node 5] ICP22 created new session: [3:7574436093298127288:2110]
2025-11-19T14:00:30.489750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436093298127288:2110] [node 5] ICS09 handshake done sender: [3:7574436093298127277:2113] self: [3:7574436093298127273:0] peer: [5:7574436091705063676:0] socket: 392 qp: -1
2025-11-19T14:00:30.489756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436093298127288:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:30.489775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:30.491510Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436093298127286:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.491536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436093298127204:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.502328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436090547748418:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.502357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:30.502371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436090547748418:2107] [node 6] ICP22 created new session: [1:7574436090547748445:2107]
2025-11-19T14:00:30.502379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436090547748445:2107] [node 6] ICS09 handshake done sender: [1:7574436090547748420:2109] self: [1:7574436090547748419:0] peer: [6:7574436091047138575:0] socket: 357 qp: -1
2025-11-19T14:00:30.502389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436090547748445:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:30.502401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:30.502423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436091047138582:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24581
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.503106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436091705063677:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57616
2025-11-19T14:00:30.503527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436093243927569:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14148
2025-11-19T14:00:30.503545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436093243927570:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14148
2025-11-19T14:00:30.503776Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436093243927560:2113] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:30.504016Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436091705063673:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:30.504233Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093243927563:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.504256Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093243927562:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.504242Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436091705063669:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.504276Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436091705063662:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.504279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436091705063662:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:30.504508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436091705063660:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:30.504709Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093243927550:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.504714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436093243927550:2110] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:30.505138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436093243927485:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.505152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:30.505163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436093243927485:2048] [node 1] ICP22 created new session: [2:7574436093243927573:2048]
2025-11-19T14:00:30.505170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436093243927573:2048] [node 1] ICS09 handshake done sender: [2:7574436093243927560:2113] self: [2:7574436093243927564:0] peer: [1:7574436090547748416:0] socket: 369 qp: -1
2025-11-19T14:00:30.505176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436093243927573:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:30.505183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:30.505713Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436091705063662:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:30.505821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436091705063595:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.505835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:30.505839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436091705063595:2050] [node 3] ICP22 created new session: [5:7574436091705063681:2050]
2025-11-19T14:00:30.505845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436091705063681:2050] [node 3] ICS09 handshake done sender: [5:7574436091705063673:2116] self: [5:7574436091705063676:0] peer: [3:7574436093298127273:0] socket: 393 qp: -1
2025-11-19T14:00:30.505849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436091705063681:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:30.505856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:30.506191Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436093243927550:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.506483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436091705063646:2099] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.506500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:30.506504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436091705063646:2099] [node 6] ICP22 created new session: [5:7574436091705063684:2099]
2025-11-19T14:00:30.506509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436091705063684:2099] [node 6] ICS09 handshake done sender: [5:7574436091705063662:2111] self: [5:7574436091705063652:0] peer: [6:7574436091047138584:0] socket: 371 qp: -1
2025-11-19T14:00:30.506514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436091705063684:2099] [node 6] ICS10 traffic start
2025-11-19T14:00:30.506531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.506961Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436091636361070:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.506974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436091636361070:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:30.507618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436091636361082:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62752
2025-11-19T14:00:30.503722Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436091047138582:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.503831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436091047138498:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:30.504142Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436091047138564:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.504166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436091047138509:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:30.504619Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436091047138563:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.504635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436091047138497:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:30.508160Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436091636361070:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:30.508347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436091636361064:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.508356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:30.508362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436091636361064:2102] [node 5] ICP22 created new session: [4:7574436091636361084:2102]
2025-11-19T14:00:30.508366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436091636361084:2102] [node 5] ICS09 handshake done sender: [4:7574436091636361070:2111] self: [4:7574436091636361065:0] peer: [5:7574436091705063679:0] socket: 374 qp: -1
2025-11-19T14:00:30.508370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436091636361084:2102] [node 5] ICS10 traffic start
2025-11-19T14:00:30.508385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:30.510171Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574436090547748355:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:30.510289Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436090547748430:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.510296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436090547748430:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:30.509668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436091047138509:2059] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436091047138564:2107] held: no
2025-11-19T14:00:30.510626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436091047138586:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24581
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.508223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436093243927548:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.508241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:30.508248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436093243927548:2109] [node 6] ICP22 created new session: [2:7574436093243927575:2109]
2025-11-19T14:00:30.508257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436093243927575:2109] [node 6] ICS09 handshake done sender: [2:7574436093243927550:2110] self: [2:7574436093243927549:0] peer: [6:7574436091047138585:0] socket: 363 qp: -1
2025-11-19T14:00:30.508262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436093243927575:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:30.508280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:30.511292Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436093243927559:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(50818528ns) {Configure:104(50817986ns) {ProcessConfigured:135(50603914ns) {StartInitialHandshake:157(50603491ns) {DropHandshakes:469(188ns) {DropIncomingHandshake:441(41ns) DropOutgoingHandshake:456(15ns)} GenerateSessionVirtualId:988(514ns) PrepareNewSessionHandshake:476(464ns) {SwitchToState:178(188ns)}}}}}
2025-11-19T14:00:30.508163Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436093298127280:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.511486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436091705063686:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57616
2025-11-19T14:00:30.511832Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436091705063669:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:30.508268Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436093298127281:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.508355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436093298127205:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:30.510676Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436093298127283:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.510683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436093298127283:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.512436Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093243927552:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.512448Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436093243927552:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.513162Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436093243927552:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:30.513251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436093243927486:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.513269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:30.513274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436093243927486:2049] [node 3] ICP22 created new session: [2:7574436093243927582:2049]
2025-11-19T14:00:30.513281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436093243927582:2049] [node 3] ICS09 handshake done sender: [2:7574436093243927552:2111] self: [2:7574436093243927551:0] peer: [3:7574436093298127293:0] socket: 365 qp: -1
2025-11-19T14:00:30.513287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436093243927582:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:30.513292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.511973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436091705063660:2110] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.511987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:30.511993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436091705063660:2110] [node 4] ICP22 created new session: [5:7574436091705063688:2110]
2025-11-19T14:00:30.511998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436091705063688:2110] [node 4] ICS09 handshake done sender: [5:7574436091705063669:2114] self: [5:7574436091705063679:0] peer: [4:7574436091636361065:0] socket: 375 qp: -1
2025-11-19T14:00:30.512005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436091705063688:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:30.512013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:30.512263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436091705063687:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57616
2025-11-19T14:00:30.512307Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436091047138564:2107] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:30.513097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436091047138587:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24581
2025-11-19T14:00:30.514113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436093298127295:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17958
2025-11-19T14:00:30.514475Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436093298127280:2114] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:30.514577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436093298127205:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.514590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:30.514596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436093298127205:2049] [node 2] ICP22 created new session: [3:7574436093298127296:2049]
2025-11-19T14:00:30.514601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436093298127296:2049] [node 2] ICS09 handshake done sender: [3:7574436093298127280:2114] self: [3:7574436093298127293:0] peer: [2:7574436093243927551:0] socket: 368 qp: -1
2025-11-19T14:00:30.514606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436093298127296:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:30.514611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:30.516563Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093243927570:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.516589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436093243927548:2109] [node 6] ICP18 handshake (actor [2:7574436093243927570:2119]) is too old
2025-11-19T14:00:30.515734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436091047138509:2059] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.515766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:30.515772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436091047138509:2059] [node 5] ICP22 created new session: [6:7574436091047138588:2059]
2025-11-19T14:00:30.515777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436091047138588:2059] [node 5] ICS09 handshake done sender: [6:7574436091047138564:2107] self: [6:7574436091047138584:0] peer: [5:7574436091705063652:0] socket: 377 qp: -1
2025-11-19T14:00:30.515782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436091047138588:2059] [node 5] ICS10 traffic start
2025-11-19T14:00:30.515789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:30.516052Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436091047138563:2106] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:30.516977Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436091705063687:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.517008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436091705063594:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:30.520793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436091047138497:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.520815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:30.520823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436091047138497:2049] [node 2] ICP22 created new session: [6:7574436091047138590:2049]
2025-11-19T14:00:30.520829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436091047138590:2049] [node 2] ICS09 handshake done sender: [6:7574436091047138563:2106] self: [6:7574436091047138585:0] peer: [2:7574436093243927549:0] socket: 376 qp: -1
2025-11-19T14:00:30.520835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436091047138590:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:30.520844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:30.522036Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093243927578:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.522046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436093243927578:2120] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:30.523432Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436093243927578:2120] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:30.523539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436093243927559:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.523558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:30.523575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436093243927559:2108] [node 5] ICP22 created new session: [2:7574436093243927585:2108]
2025-11-19T14:00:30.523582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436093243927585:2108] [node 5] ICS09 handshake done sender: [2:7574436093243927578:2120] self: [2:7574436093243927566:0] peer: [5:7574436091705063690:0] socket: 411 qp: -1
2025-11-19T14:00:30.523588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436093243927585:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:30.523595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.528011Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436090547748438:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.528218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436090547748447:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
2025-11-19T14:00:30.528978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436091705063594:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17463) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436091705063687:2121] held: no
2025-11-19T14:00:30.529138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436091705063691:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57616
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.530435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436093298127299:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17958
2025-11-19T14:00:30.530284Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436090547748418:2107] [node 6] ICP35 event processing took too much time PendingConnection:278(50123176ns) {HandleHandshakeStatus:329(50122225ns) {DropIncomingHandshake:441(374ns) DropOutgoingHandshake:456(614ns) SwitchToState:178(322ns) ProcessPendingSessionEvents:495(19815576ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19792639ns max# 19724899ns}}}
2025-11-19T14:00:30.530337Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436090547748428:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:30.532219Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436090547748433:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:30.531769Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436091636361064:2102] [node 5] ICP35 event processing took too much time PendingConnection:278(23302292ns) {HandleHandshakeStatus:329(23301924ns) {DropIncomingHandshake:441(34ns) DropOutgoingHandshake:456(76ns) SwitchToState:178(25ns) ProcessPendingSessionEvents:495(20789246ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20766855ns max# 20700411ns}}}
2025-11-19T14:00:30.534196Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436090547748430:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:30.534834Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436091705063687:2121] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.533014Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436093298127283:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:30.533059Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436093298127286:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:30.533110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436093298127204:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.533124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:30.533130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436093298127204:2048] [node 1] ICP22 created new session: [3:7574436093298127301:2048]
2025-11-19T14:00:30.533137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436093298127301:2048] [node 1] ICS09 handshake done sender: [3:7574436093298127286:2117] self: [3:7574436093298127291:0] peer: [1:7574436090547748432:0] socket: 378 qp: -1
2025-11-19T14:00:30.533143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436093298127301:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:30.533150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:30.535125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436090547748418:2107] [node 6] ICP18 handshake (actor [1:7574436090547748439:2118]) is too old
2025-11-19T14:00:30.535415Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436090547748447:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.535422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436090547748356:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.535437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:30.535441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436090547748356:2049] [node 3] ICP22 created new session: [1:7574436090547748452:2049]
2025-11-19T14:00:30.535447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436090547748452:2049] [node 3] ICS09 handshake done sender: [1:7574436090547748433:2114] self: [1:7574436090547748432:0] peer: [3:7574436093298127291:0] socket: 395 qp: -1
2025-11-19T14:00:30.535452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436090547748452:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:30.535459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:30.534954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436091636361087:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62752
2025-11-19T14:00:30.534978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436091636361088:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62752
2025-11-19T14:00:30.535002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436091636361089:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62752
2025-11-19T14:00:30.535018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436091636361090:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62752
2025-11-19T14:00:30.536016Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436091636361088:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.536037Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436091636361066:2109] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:30.536085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436091636361004:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:30.536138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436091636361002:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.536147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:30.536152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436091636361002:2048] [node 1] ICP22 created new session: [4:7574436091636361092:2048]
2025-11-19T14:00:30.536157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436091636361092:2048] [node 1] ICS09 handshake done sender: [4:7574436091636361066:2109] self: [4:7574436091636361080:0] peer: [1:7574436090547748427:0] socket: 359 qp: -1
2025-11-19T14:00:30.536162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436091636361092:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:30.536168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:30.535361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436091705063692:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57616
2025-11-19T14:00:30.536226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436090547748426:2111] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.536240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:30.536245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436090547748426:2111] [node 5] ICP22 created new session: [1:7574436090547748454:2111]
2025-11-19T14:00:30.536249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436090547748454:2111] [node 5] ICS09 handshake done sender: [1:7574436090547748430:2113] self: [1:7574436090547748429:0] peer: [5:7574436091705063678:0] socket: 394 qp: -1
2025-11-19T14:00:30.536253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436090547748454:2111] [node 5] ICS10 traffic start
2025-11-19T14:00:30.536258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.536230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436093298127278:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.536255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:30.536264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436093298127278:2112] [node 6] ICP22 created new session: [3:7574436093298127303:2112]
2025-11-19T14:00:30.536272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436093298127303:2112] [node 6] ICS09 handshake done sender: [3:7574436093298127283:2116] self: [3:7574436093298127279:0] peer: [6:7574436091047138583:0] socket: 400 qp: -1
2025-11-19T14:00:30.536280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436093298127303:2112] [node 6] ICS10 traffic start
2025-11-19T14:00:30.536288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:30.536568Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436093298127292:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.536572Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436093298127292:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:30.538228Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436091636361087:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.538370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436091636361003:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:30.538404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436091636361094:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62752
2025-11-19T14:00:30.540098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436090547748423:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.540112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:30.540118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436090547748423:2110] [node 4] ICP22 created new session: [1:7574436090547748456:2110]
2025-11-19T14:00:30.540125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436090547748456:2110] [node 4] ICS09 handshake done sender: [1:7574436090547748428:2112] self: [1:7574436090547748427:0] peer: [4:7574436091636361080:0] socket: 358 qp: -1
2025-11-19T14:00:30.540131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436090547748456:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:30.540140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:30.540280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436090547748423:2110] [node 4] ICP18 handshake (actor [1:7574436090547748447:2119]) is too old
2025-11-19T14:00:30.540536Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436093298127292:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:30.541550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436093298127276:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.541565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:30.541571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436093298127276:2111] [node 4] ICP22 created new session: [3:7574436093298127306:2111]
2025-11-19T14:00:30.541576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436093298127306:2111] [node 4] ICS09 handshake done sender: [3:7574436093298127292:2118] self: [3:7574436093298127285:0] peer: [4:7574436091636361091:0] socket: 416 qp: -1
2025-11-19T14:00:30.541580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436093298127306:2111] [node 4] ICS10 traffic start
2025-11-19T14:00:30.541587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:30.544410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436091047138498:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436091047138582:2119] held: no
2025-11-19T14:00:30.547925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436091705063594:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.547951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:30.547963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436091705063594:2049] [node 2] ICP22 created new session: [5:7574436091705063693:2049]
2025-11-19T14:00:30.547970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436091705063693:2049] [node 2] ICS09 handshake done sender: [5:7574436091705063687:2121] self: [5:7574436091705063690:0] peer: [2:7574436093243927566:0] socket: 412 qp: -1
2025-11-19T14:00:30.547977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436091705063693:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:30.548002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:30.549532Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093243927567:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.549556Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436093243927567:2117] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:30.549091Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436091705063675:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:30.553729Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436091636361088:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:30.553847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436091636361004:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.553865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:30.553873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436091636361004:2050] [node 3] ICP22 created new session: [4:7574436091636361097:2050]
2025-11-19T14:00:30.553879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436091636361097:2050] [node 3] ICS09 handshake done sender: [4:7574436091636361088:2118] self: [4:7574436091636361091:0] peer: [3:7574436093298127285:0] socket: 422 qp: -1
2025-11-19T14:00:30.553886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436091636361097:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:30.553908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.555246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436091705063593:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3361) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436091705063675:2118] held: no
2025-11-19T14:00:30.555334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436091705063593:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.555341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:30.555346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436091705063593:2048] [node 1] ICP22 created new session: [5:7574436091705063695:2048]
2025-11-19T14:00:30.555351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436091705063695:2048] [node 1] ICS09 handshake done sender: [5:7574436091705063675:2118] self: [5:7574436091705063678:0] peer: [1:7574436090547748429:0] socket: 396 qp: -1
2025-11-19T14:00:30.555355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436091705063695:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:30.555361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:30.555989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436091636361002:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(19746650ns) {HandleHandshakeStatus:329(19745757ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(829ns) SwitchToState:178(40ns) ProcessPendingSessionEvents:495(14880197ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14855343ns max# 14782834ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.558215Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436091636361089:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.558492Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574436091636361064:2102] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:30.559293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436091636361099:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62752
2025-11-19T14:00:30.555954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436091047138593:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24581
2025-11-19T14:00:30.556553Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436091047138582:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:30.558592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436091047138594:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24581
2025-11-19T14:00:30.558628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436091047138498:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.558639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:30.558644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436091047138498:2050] [node 3] ICP22 created new session: [6:7574436091047138595:2050]
2025-11-19T14:00:30.558649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436091047138595:2050] [node 3] ICS09 handshake done sender: [6:7574436091047138582:2119] self: [6:7574436091047138583:0] peer: [3:7574436093298127279:0] socket: 401 qp: -1
2025-11-19T14:00:30.558654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436091047138595:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:30.558662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:30.560024Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436093243927567:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:30.560145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436093243927558:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.560178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:30.560185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436093243927558:2107] [node 4] ICP22 created new session: [2:7574436093243927588:2107]
2025-11-19T14:00:30.560191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436093243927588:2107] [node 4] ICS09 handshake done sender: [2:7574436093243927567:2117] self: [2:7574436093243927565:0] peer: [4:7574436091636361096:0] socket: 387 qp: -1
2025-11-19T14:00:30.560196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436093243927588:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:30.560205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:30.560566Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436091047138594:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.560610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436091047138537:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:30.560826Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436091636361081:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:30.560835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436091636361081:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:30.561398Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436091636361087:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:30.562637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436091047138598:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24581
2025-11-19T14:00:30.562778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436091636361003:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17463) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436091636361087:2117] held: no
2025-11-19T14:00:30.563016Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436091636361081:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:30.563182Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436091047138594:2123] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:30.563290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436091047138537:2065] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.563306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:30.563311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436091047138537:2065] [node 4] ICP22 created new session: [6:7574436091047138599:2065]
2025-11-19T14:00:30.563317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436091047138599:2065] [node 4] ICS09 handshake done sender: [6:7574436091047138594:2123] self: [6:7574436091047138597:0] peer: [4:7574436091636361078:0] socket: 355 qp: -1
2025-11-19T14:00:30.563322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436091047138599:2065] [node 4] ICS10 traffic start
2025-11-19T14:00:30.563331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:30.564345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436091636361069:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:30.564363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:30.564368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436091636361069:2110] [node 6] ICP22 created new session: [4:7574436091636361102:2110]
2025-11-19T14:00:30.564373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436091636361102:2110] [node 6] ICS09 handshake done sender: [4:7574436091636361081:2115] self: [4:7574436091636361078:0] peer: [6:7574436091047138597:0] socket: 381 qp: -1
2025-11-19T14:00:30.564378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436091636361102:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:30.564385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:30.565246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436091636361003:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:30.565257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:30.565262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436091636361003:2049] [node 2] ICP22 created new session: [4:7574436091636361104:2049]
2025-11-19T14:00:30.565267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436091636361104:2049] [node 2] ICS09 handshake done sender: [4:7574436091636361087:2117] self: [4:7574436091636361096:0] peer: [2:7574436093243927565:0] socket: 413 qp: -1
2025-11-19T14:00:30.565271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436091636361104:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:30.565276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:30.599675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:30.599800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436093298127301:2048] [node 1] ICS01 socket: 378 reason# EndOfStream
2025-11-19T14:00:30.599807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436093298127204:2048] [node 1] ICP30 unregister session Session# [3:7574436093298127301:2048] VirtualId# [3:7574436093298127291:0]
2025-11-19T14:00:30.599813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436093298127301:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:30.599849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:30.599872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:30.598187Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:30.598291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436093243927573:2048] [node 1] ICS01 socket: 369 reason# EndOfStream
2025-11-19T14:00:30.598301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436093243927485:2048] [node 1] ICP30 unregister session Session# [2:7574436093243927573:2048] VirtualId# [2:7574436093243927564:0]
2025-11-19T14:00:30.598309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436093243927573:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:30.598363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:30.598389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:30.607985Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:30.608076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436091705063695:2048] [node 1] ICS01 socket: 396 reason# EndOfStream
2025-11-19T14:00:30.608083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436091705063593:2048] [node 1] ICP30 unregister session Session# [5:7574436091705063695:2048] VirtualId# [5:7574436091705063678:0]
2025-11-19T14:00:30.608089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436091705063695:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:30.608137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:30.608159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:30.608205Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:30.608263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436091047138578:2048] [node 1] ICS01 socket: 362 reason# EndOfStream
2025-11-19T14:00:30.608268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436091047138496:2048] [node 1] ICP30 unregister session Session# [6:7574436091047138578:2048] VirtualId# [6:7574436091047138575:0]
2025-11-19T14:00:30.608272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436091047138578:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:30.608299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:30.608320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:30.608347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:30.608407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436091636361092:2048] [node 1] ICS07 socket disconnect 359 reason# EndOfStream
2025-11-19T14:00:30.608411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436091636361092:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:30.608434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:30.608454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436091636361092:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:30.609764Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436091636361002:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3361) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:30.609776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436091636361092:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:30.609781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436091636361002:2048] [node 1] ICP30 unregister session Session# [4:7574436091636361092:2048] VirtualId# [4:7574436091636361080:0]
2025-11-19T14:00:30.609816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:30.609821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436091636361002:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3361) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:30.609823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3361) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:30.623832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7574436091636361002:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:30.624591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:30.624686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436093298127296:2049] [node 2] ICS01 socket: 368 reason# EndOfStream
2025-11-19T14:00:30.624695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436093298127205:2049] [node 2] ICP30 unregister session Session# [3:7574436093298127296:2049] VirtualId# [3:7574436093298127293:0]
2025-11-19T14:00:30.624703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436093298127296:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:30.624744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:30.624764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:30.638786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:30.638873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436091636361104:2049] [node 2] ICS01 socket: 413 reason# EndOfStream
2025-11-19T14:00:30.638879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436091636361003:2049] [node 2] ICP30 unregister session Session# [4:7574436091636361104:2049] VirtualId# [4:7574436091636361096:0]
2025-11-19T14:00:30.638885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436091636361104:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:30.638940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:30.638964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:30.638997Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:30.639057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436091705063693:2049] [node 2] ICS01 socket: 412 reason# EndOfStream
2025-11-19T14:00:30.639062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436091705063594:2049] [node 2] ICP30 unregister session Session# [5:7574436091705063693:2049] VirtualId# [5:7574436091705063690:0]
2025-11-19T14:00:30.639066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436091705063693:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:30.639089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:30.639109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:30.639139Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:30.639200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436091047138590:2049] [node 2] ICS01 socket: 376 reason# EndOfStream
2025-11-19T14:00:30.639205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436091047138497:2049] [node 2] ICP30 unregister session Session# [6:7574436091047138590:2049] VirtualId# [6:7574436091047138585:0]
2025-11-19T14:00:30.639208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436091047138590:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:30.639234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:30.639254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:30.662279Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:30.662369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436091047138595:2050] [node 3] ICS01 socket: 401 reason# EndOfStream
2025-11-19T14:00:30.662377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436091047138498:2050] [node 3] ICP30 unregister session Session# [6:7574436091047138595:2050] VirtualId# [6:7574436091047138583:0]
2025-11-19T14:00:30.662384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436091047138595:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:30.662441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:30.662464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:30.662760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:30.662825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574436091705063681:2050] [node 3] ICS07 socket disconnect 393 reason# EndOfStream
2025-11-19T14:00:30.662829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436091705063681:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:30.662858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:30.662878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574436091705063681:2050] [node 3] ICS15 start handshake
2025-11-19T14:00:30.672013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436091705063595:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9798) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:30.672033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436091705063681:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:30.672038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436091705063595:2050] [node 3] ICP30 unregister session Session# [5:7574436091705063681:2050] VirtualId# [5:7574436091705063676:0]
2025-11-19T14:00:30.672082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:30.672088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574436091705063595:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9798) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:30.672090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9798) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:30.690249Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:30.690341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436091047138599:2065] [node 4] ICS01 socket: 355 reason# EndOfStream
2025-11-19T14:00:30.690349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436091047138537:2065] [node 4] ICP30 unregister session Session# [6:7574436091047138599:2065] VirtualId# [6:7574436091047138597:0]
2025-11-19T14:00:30.690355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436091047138599:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:30.690413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:30.690437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:30.784454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436090952562328:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:30.784472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436090952562329:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:30.803950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436093053298499:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:30.807796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436093053298498:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:30.808010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436091685507977:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:30.808019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436091685507978:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:30.887860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436091620770078:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:30.887871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436091620770079:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:30.887876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436091620770080:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:30.975838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436089416409328:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:30.975852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436089416409329:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:30.975857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436089416409330:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:31.035857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436093939279608:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:31.035869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436093939279609:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:31.035874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436093939279610:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:31.088781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436095247529685:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.092062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436095915737435:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29705
2025-11-19T14:00:31.092084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436093939279665:2090] [node 5] ICP01 ready to work
2025-11-19T14:00:31.092255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436093939279666:2094] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.097172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436095915737436:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29705
2025-11-19T14:00:31.097312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436093711376685:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25887
2025-11-19T14:00:31.102944Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436095915737435:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.103437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436091620770078:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.103834Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436095915737436:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.104002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436095915737439:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:31.104251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436095915737439:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.103967Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436093711376685:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.104163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436093711376687:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:31.104568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436093711376687:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.104144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436097348265859:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:31.104442Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436093939279676:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.104454Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574436093939279676:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.105051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436095247529696:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:31.105639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436095247529699:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38919
2025-11-19T14:00:31.105823Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436093939279673:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.105829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574436093939279673:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.106388Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436093939279676:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.106604Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436093939279673:2109] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:31.106604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436093711376689:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25887
2025-11-19T14:00:31.106887Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436093711376685:2108] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:31.106681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574436093939279665:2090] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.106707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:31.106712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436093939279665:2090] [node 5] ICP22 created new session: [6:7574436093939279681:2090]
2025-11-19T14:00:31.106719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436093939279681:2090] [node 5] ICS09 handshake done sender: [6:7574436093939279673:2109] self: [6:7574436093939279672:0] peer: [5:7574436093711376688:0] socket: 366 qp: -1
2025-11-19T14:00:31.106726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436093939279681:2090] [node 5] ICS10 traffic start
2025-11-19T14:00:31.106744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.107166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574436093939279666:2094] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.107178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:31.107183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436093939279666:2094] [node 4] ICP22 created new session: [6:7574436093939279683:2094]
2025-11-19T14:00:31.107187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436093939279683:2094] [node 4] ICS09 handshake done sender: [6:7574436093939279676:2110] self: [6:7574436093939279675:0] peer: [4:7574436095915737440:0] socket: 368 qp: -1
2025-11-19T14:00:31.107193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436093939279683:2094] [node 4] ICS10 traffic start
2025-11-19T14:00:31.107198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:31.109362Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436095247529687:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.109384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436095247529687:2108] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.110019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436097348265858:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57417
2025-11-19T14:00:31.110711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436095247529701:2114] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.112338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436095247529703:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38919
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.112507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436095247529704:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38919
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.115113Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436097348265858:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.115510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436095915737441:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29705
2025-11-19T14:00:31.115544Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436095247529704:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.116169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436095980475342:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58424
2025-11-19T14:00:31.115315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436093711376696:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25887
2025-11-19T14:00:31.116076Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436095915737436:2109] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:31.116359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436095980475336:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58424
2025-11-19T14:00:31.116445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436095980475334:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.117116Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436095980475336:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.117139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436091685507977:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.117282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436095915737439:2110] [node 6] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.117293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:31.117299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436095915737439:2110] [node 6] ICP22 created new session: [4:7574436095915737442:2110]
2025-11-19T14:00:31.117305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436095915737442:2110] [node 6] ICS09 handshake done sender: [4:7574436095915737436:2109] self: [4:7574436095915737440:0] peer: [6:7574436093939279675:0] socket: 369 qp: -1
2025-11-19T14:00:31.117310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436095915737442:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:31.117329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.126855Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436095247529693:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.126868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436095247529693:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.124759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436095980475346:2112] [node 6] ICP01 ready to work
2025-11-19T14:00:31.126753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436095980475350:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58424
2025-11-19T14:00:31.128604Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436095247529693:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:31.128820Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436095247529687:2108] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:31.129064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436095980475352:2115] [node 5] ICP01 ready to work
2025-11-19T14:00:31.132649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436095915737448:2113] [node 5] ICP01 ready to work
2025-11-19T14:00:31.132273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436097348265871:2114] [node 6] ICP01 ready to work
2025-11-19T14:00:31.135977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436095915737439:2110] [node 6] ICP35 event processing took too much time PendingConnection:278(18673383ns) {HandleHandshakeStatus:329(18672371ns) {DropIncomingHandshake:441(189ns) DropOutgoingHandshake:456(52ns) SwitchToState:178(62ns) ProcessPendingSessionEvents:495(489ns)}}
2025-11-19T14:00:31.136131Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436095247529690:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.136146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436095247529690:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.136263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436095247529685:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.136279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:31.136288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436095247529685:2107] [node 4] ICP22 created new session: [1:7574436095247529707:2107]
2025-11-19T14:00:31.136296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436095247529707:2107] [node 4] ICS09 handshake done sender: [1:7574436095247529687:2108] self: [1:7574436095247529686:0] peer: [4:7574436095915737437:0] socket: 360 qp: -1
2025-11-19T14:00:31.136305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436095247529707:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:31.136314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:31.136216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574436093939279666:2094] [node 4] ICP35 event processing took too much time PendingConnection:278(28947608ns) {HandleHandshakeStatus:329(28947021ns) {DropIncomingHandshake:441(34ns) DropOutgoingHandshake:456(135ns) SwitchToState:178(154ns) ProcessPendingSessionEvents:495(28889659ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 28873419ns max# 28807401ns}}}
2025-11-19T14:00:31.136448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436097348265868:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57417
2025-11-19T14:00:31.136901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436097348265865:2111] [node 4] ICP01 ready to work
2025-11-19T14:00:31.136977Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436097348265868:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.137355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436093053298498:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.137379Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436093053298498:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(22127664ns) {IncomingHandshake:261(22127229ns) {DropIncomingHandshake:441(397ns) PrepareNewSessionHandshake:476(509ns) {SwitchToState:178(113ns)} GenerateSessionVirtualId:988(240ns) IssueIncomingHandshakeReply:193(22122597ns) {SwitchToState:178(172ns)}}}
2025-11-19T14:00:31.137474Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436095980475342:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.137499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436091685507978:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.141903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436097348265876:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57417
2025-11-19T14:00:31.141948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436097348265877:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57417
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.142036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436093939279688:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30250
2025-11-19T14:00:31.142199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436090952562328:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.142221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:31.142229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436090952562328:2048] [node 2] ICP22 created new session: [1:7574436095247529709:2048]
2025-11-19T14:00:31.142236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436095247529709:2048] [node 2] ICS09 handshake done sender: [1:7574436095247529693:2110] self: [1:7574436095247529692:0] peer: [2:7574436097348265869:0] socket: 364 qp: -1
2025-11-19T14:00:31.142243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436095247529709:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:31.142252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:31.142670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436095247529711:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38919
2025-11-19T14:00:31.140952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436095980475334:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24448255ns) {Configure:104(24447574ns) {ProcessConfigured:135(24153241ns) {StartInitialHandshake:157(24153057ns) {DropHandshakes:469(584ns) {DropIncomingHandshake:441(14ns) DropOutgoingHandshake:456(117ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(983ns) {SwitchToState:178(33ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.143445Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436097348265855:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.143460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436097348265855:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.143212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436093939279689:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30250
2025-11-19T14:00:31.143854Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436097348265858:2109] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:31.143927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436093711376687:2109] [node 6] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.143954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:31.143962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436093711376687:2109] [node 6] ICP22 created new session: [5:7574436093711376697:2109]
2025-11-19T14:00:31.143970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436093711376697:2109] [node 6] ICS09 handshake done sender: [5:7574436093711376685:2108] self: [5:7574436093711376688:0] peer: [6:7574436093939279672:0] socket: 367 qp: -1
2025-11-19T14:00:31.143977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436093711376697:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:31.144004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.144152Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436093939279688:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.146385Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436095247529699:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.146466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436090952562328:2048] [node 2] ICP18 handshake (actor [1:7574436095247529699:2113]) is too old
2025-11-19T14:00:31.146717Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436095247529685:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(17756182ns) {HandleHandshakeStatus:329(17755707ns) {DropIncomingHandshake:441(356ns) DropOutgoingHandshake:456(256ns) SwitchToState:178(226ns) ProcessPendingSessionEvents:495(10317464ns) {unordered ForwardSessionEventToSession:612 num# 20 sum# 10311995ns max# 10297148ns}}}
2025-11-19T14:00:31.144904Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436095980475348:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.144912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436095980475348:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.145498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436095980475357:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58424
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.147348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436093939279691:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30250
2025-11-19T14:00:31.147467Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436095247529703:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.147512Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436095247529690:2109] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:31.147830Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436095247529711:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.147851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436090952562329:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.147861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:31.147867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436090952562329:2049] [node 3] ICP22 created new session: [1:7574436095247529714:2049]
2025-11-19T14:00:31.147872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436095247529714:2049] [node 3] ICS09 handshake done sender: [1:7574436095247529690:2109] self: [1:7574436095247529689:0] peer: [3:7574436095980475344:0] socket: 362 qp: -1
2025-11-19T14:00:31.147877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436095247529714:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:31.147883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:31.147192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436093053298498:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.147216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:31.147223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436093053298498:2048] [node 1] ICP22 created new session: [2:7574436097348265878:2048]
2025-11-19T14:00:31.147229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436097348265878:2048] [node 1] ICS09 handshake done sender: [2:7574436097348265858:2109] self: [2:7574436097348265869:0] peer: [1:7574436095247529692:0] socket: 370 qp: -1
2025-11-19T14:00:31.147236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436097348265878:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:31.147243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.149607Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436095980475348:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:31.149602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436093939279695:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30250
2025-11-19T14:00:31.151340Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436093711376696:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.151380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436089416409329:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.149741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436095980475346:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.149755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:31.149762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436095980475346:2112] [node 6] ICP22 created new session: [3:7574436095980475360:2112]
2025-11-19T14:00:31.149769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436095980475360:2112] [node 6] ICS09 handshake done sender: [3:7574436095980475348:2113] self: [3:7574436095980475347:0] peer: [6:7574436093939279690:0] socket: 389 qp: -1
2025-11-19T14:00:31.149775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436095980475360:2112] [node 6] ICS10 traffic start
2025-11-19T14:00:31.149793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:31.151198Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436095980475357:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.151481Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436097348265866:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.151489Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436097348265866:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.153703Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436093711376687:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(46244861ns) {HandleHandshakeStatus:329(46244247ns) {DropIncomingHandshake:441(1222ns) DropOutgoingHandshake:456(60ns) SwitchToState:178(211ns) ProcessPendingSessionEvents:495(494ns)}}
2025-11-19T14:00:31.153866Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436095980475350:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.159149Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436097348265876:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.159350Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436097348265855:2107] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:31.159986Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436093939279695:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.160061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436093939279609:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.158104Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436095247529701:2114] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(47317880ns) {Configure:104(47316878ns) {ProcessConfigured:135(46899645ns) {StartInitialHandshake:157(46898476ns) {DropHandshakes:469(739ns) {DropIncomingHandshake:441(75ns) DropOutgoingHandshake:456(79ns)} GenerateSessionVirtualId:988(137ns) PrepareNewSessionHandshake:476(1220ns) {SwitchToState:178(536ns)}}}}}
2025-11-19T14:00:31.160166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436090952562329:2049] [node 3] ICP18 handshake (actor [1:7574436095247529703:2115]) is too old
2025-11-19T14:00:31.160618Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436093939279689:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.160662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436093939279608:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.160245Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436097348265873:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.160253Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436097348265873:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.161646Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436097348265873:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:31.164086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436093053298499:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.164111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:31.164119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436093053298499:2049] [node 3] ICP22 created new session: [2:7574436097348265885:2049]
2025-11-19T14:00:31.164127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436097348265885:2049] [node 3] ICS09 handshake done sender: [2:7574436097348265855:2107] self: [2:7574436097348265854:0] peer: [3:7574436095980475354:0] socket: 365 qp: -1
2025-11-19T14:00:31.164133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436097348265885:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:31.164141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:31.164339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436097348265871:2114] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.164344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:31.164347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436097348265871:2114] [node 6] ICP22 created new session: [2:7574436097348265887:2114]
2025-11-19T14:00:31.164351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436097348265887:2114] [node 6] ICS09 handshake done sender: [2:7574436097348265873:2115] self: [2:7574436097348265872:0] peer: [6:7574436093939279696:0] socket: 407 qp: -1
2025-11-19T14:00:31.164355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436097348265887:2114] [node 6] ICS10 traffic start
2025-11-19T14:00:31.164360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:31.165006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436093711376705:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25887
2025-11-19T14:00:31.165094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436093711376700:2114] [node 4] ICP01 ready to work
2025-11-19T14:00:31.165521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436093939279610:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.165567Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436093939279610:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(21348365ns) {Configure:104(21347688ns) {ProcessConfigured:135(20840770ns) {IncomingHandshake:261(20828647ns) {DropIncomingHandshake:441(88ns) PrepareNewSessionHandshake:476(293ns) {SwitchToState:178(93ns)} GenerateSessionVirtualId:988(996ns) IssueIncomingHandshakeReply:193(20823567ns) {SwitchToState:178(239ns)}}}}}
2025-11-19T14:00:31.165591Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436093939279688:2113] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:31.165913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436093939279610:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.165925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:31.165932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436093939279610:2050] [node 3] ICP22 created new session: [6:7574436093939279698:2050]
2025-11-19T14:00:31.165937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436093939279698:2050] [node 3] ICS09 handshake done sender: [6:7574436093939279688:2113] self: [6:7574436093939279690:0] peer: [3:7574436095980475347:0] socket: 392 qp: -1
2025-11-19T14:00:31.165941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436093939279698:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:31.165949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:31.166184Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436095247529698:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.166197Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436095247529698:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.166525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436091685507977:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436095980475336:2108] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.168465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436095915737452:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29705
2025-11-19T14:00:31.169029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436095915737453:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29705
2025-11-19T14:00:31.168942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436097348265890:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57417
2025-11-19T14:00:31.169983Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436097348265866:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.171261Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436091685507978:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18912) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436095980475342:2111] held: no
2025-11-19T14:00:31.171301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436095915737454:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29705
2025-11-19T14:00:31.171855Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436095915737435:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.172111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436093939279700:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30250
2025-11-19T14:00:31.172016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436095980475364:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58424
2025-11-19T14:00:31.171151Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436097348265890:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.171187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436097348265871:2114] [node 6] ICP18 handshake (actor [2:7574436097348265890:2119]) is too old
2025-11-19T14:00:31.171534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436097348265859:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.171548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:31.171555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436097348265859:2108] [node 5] ICP22 created new session: [2:7574436097348265892:2108]
2025-11-19T14:00:31.171560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436097348265892:2108] [node 5] ICS09 handshake done sender: [2:7574436097348265866:2112] self: [2:7574436097348265860:0] peer: [5:7574436093711376704:0] socket: 383 qp: -1
2025-11-19T14:00:31.171566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436097348265892:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:31.171573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.173092Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436095247529698:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:31.173014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436095980475365:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58424
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.173216Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436093939279609:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18912) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436093939279695:2117] held: no
2025-11-19T14:00:31.173361Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436093939279695:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:31.173437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436095247529696:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.173457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:31.173462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436095247529696:2111] [node 6] ICP22 created new session: [1:7574436095247529721:2111]
2025-11-19T14:00:31.173469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436095247529721:2111] [node 6] ICS09 handshake done sender: [1:7574436095247529698:2112] self: [1:7574436095247529697:0] peer: [6:7574436093939279697:0] socket: 400 qp: -1
2025-11-19T14:00:31.173473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436095247529721:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:31.173503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:31.173782Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436093711376705:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.173490Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436095915737453:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.173639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436091620770080:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.173938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436091620770078:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.173950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:31.173956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436091620770078:2048] [node 1] ICP22 created new session: [4:7574436095915737456:2048]
2025-11-19T14:00:31.173962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436095915737456:2048] [node 1] ICS09 handshake done sender: [4:7574436095915737435:2108] self: [4:7574436095915737437:0] peer: [1:7574436095247529686:0] socket: 361 qp: -1
2025-11-19T14:00:31.173970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436095915737456:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:31.173991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:31.173816Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436095980475355:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.173825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436095980475355:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.174460Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436095980475342:2111] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:31.174648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436093939279609:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.174661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:31.174667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436093939279609:2049] [node 2] ICP22 created new session: [6:7574436093939279702:2049]
2025-11-19T14:00:31.174672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436093939279702:2049] [node 2] ICS09 handshake done sender: [6:7574436093939279695:2117] self: [6:7574436093939279696:0] peer: [2:7574436097348265872:0] socket: 409 qp: -1
2025-11-19T14:00:31.174678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436093939279702:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:31.174683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:31.175432Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436095980475336:2108] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:31.175584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436091685507977:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.175595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:31.175600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436091685507977:2048] [node 1] ICP22 created new session: [3:7574436095980475367:2048]
2025-11-19T14:00:31.175606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436095980475367:2048] [node 1] ICS09 handshake done sender: [3:7574436095980475336:2108] self: [3:7574436095980475344:0] peer: [1:7574436095247529689:0] socket: 363 qp: -1
2025-11-19T14:00:31.175611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436095980475367:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:31.175615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.175945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436091685507978:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.175959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:31.175964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436091685507978:2049] [node 2] ICP22 created new session: [3:7574436095980475370:2049]
2025-11-19T14:00:31.175968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436095980475370:2049] [node 2] ICS09 handshake done sender: [3:7574436095980475342:2111] self: [3:7574436095980475354:0] peer: [2:7574436097348265854:0] socket: 377 qp: -1
2025-11-19T14:00:31.175972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436095980475370:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:31.175987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: 2025-11-19T14:00:31.175992Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436095980475355:2116] [node 4] ICH04 handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.176196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436093711376707:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25887
2025-11-19T14:00:31.176244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436093711376708:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25887
2025-11-19T14:00:31.176304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436095980475334:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.176328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:31.176332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436095980475334:2107] [node 4] ICP22 created new session: [3:7574436095980475372:2107]
2025-11-19T14:00:31.176336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436095980475372:2107] [node 4] ICS09 handshake done sender: [3:7574436095980475355:2116] self: [3:7574436095980475343:0] peer: [4:7574436095915737455:0] socket: 412 qp: -1
2025-11-19T14:00:31.176340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436095980475372:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:31.176346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:31.176829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436093711376709:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25887
2025-11-19T14:00:31.176903Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436093711376696:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:31.176977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436089416409329:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.176989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:31.176997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436089416409329:2049] [node 2] ICP22 created new session: [5:7574436093711376710:2049]
2025-11-19T14:00:31.177005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436093711376710:2049] [node 2] ICS09 handshake done sender: [5:7574436093711376696:2113] self: [5:7574436093711376704:0] peer: [2:7574436097348265860:0] socket: 385 qp: -1
2025-11-19T14:00:31.177010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436093711376710:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:31.177016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:31.177651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436093939279701:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30250
2025-11-19T14:00:31.180839Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436091620770080:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14564) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436095915737453:2116] held: no
2025-11-19T14:00:31.181585Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436093939279608:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436093939279689:2114] held: no
2025-11-19T14:00:31.181798Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436093939279689:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:31.188986Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436093711376707:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.189103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436089416409328:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.190748Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436095247529696:2111] [node 6] ICP35 event processing took too much time PendingConnection:278(17229227ns) {HandleHandshakeStatus:329(17228562ns) {DropIncomingHandshake:441(6796ns) DropOutgoingHandshake:456(95ns) SwitchToState:178(108ns) ProcessPendingSessionEvents:495(13183194ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 13168574ns max# 13116261ns}}}
2025-11-19T14:00:31.193103Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436095980475334:2107] [node 4] ICP35 event processing took too much time PendingConnection:278(16733990ns) {HandleHandshakeStatus:329(16733717ns) {DropIncomingHandshake:441(3374ns) DropOutgoingHandshake:456(80ns) SwitchToState:178(75ns) ProcessPendingSessionEvents:495(15122476ns) {unordered ForwardSessionEventToSession:612 num# 60 sum# 15110601ns max# 15067105ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.190407Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436093711376700:2114] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25218423ns) {Configure:104(25217318ns) {ProcessConfigured:135(24864801ns) {StartInitialHandshake:157(24864125ns) {DropHandshakes:469(398ns) {DropIncomingHandshake:441(52ns) DropOutgoingHandshake:456(48ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:476(740ns) {SwitchToState:178(37ns)}}}}}
2025-11-19T14:00:31.190437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436093711376700:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.193475Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436095247529717:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.193488Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436095247529717:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.195155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436093939279608:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.195182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:31.195187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436093939279608:2048] [node 1] ICP22 created new session: [6:7574436093939279704:2048]
2025-11-19T14:00:31.195194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436093939279704:2048] [node 1] ICS09 handshake done sender: [6:7574436093939279689:2114] self: [6:7574436093939279697:0] peer: [1:7574436095247529697:0] socket: 401 qp: -1
2025-11-19T14:00:31.195199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436093939279704:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:31.195205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:31.192021Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436095915737450:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.192035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436095915737450:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.193261Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436095915737450:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:31.193398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436095915737462:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29705
2025-11-19T14:00:31.194606Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436095915737453:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:31.194669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436095915737448:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.194682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:31.194690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436095915737448:2113] [node 5] ICP22 created new session: [4:7574436095915737463:2113]
2025-11-19T14:00:31.194697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436095915737463:2113] [node 5] ICS09 handshake done sender: [4:7574436095915737450:2114] self: [4:7574436095915737449:0] peer: [5:7574436093711376714:0] socket: 395 qp: -1
2025-11-19T14:00:31.194704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436095915737463:2113] [node 5] ICS10 traffic start
2025-11-19T14:00:31.194713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:31.196412Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436095915737454:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.196428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436091620770080:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.196438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:31.196445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436091620770080:2050] [node 3] ICP22 created new session: [4:7574436095915737465:2050]
2025-11-19T14:00:31.196450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436095915737465:2050] [node 3] ICS09 handshake done sender: [4:7574436095915737453:2116] self: [4:7574436095915737455:0] peer: [3:7574436095980475343:0] socket: 380 qp: -1
2025-11-19T14:00:31.196455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436095915737465:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:31.196461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:31.196479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436091620770079:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.197099Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436093711376709:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.197133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436089416409330:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.197018Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436095247529717:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:31.197932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436095247529701:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.197950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:31.197956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436095247529701:2114] [node 5] ICP22 created new session: [1:7574436095247529724:2114]
2025-11-19T14:00:31.197962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436095247529724:2114] [node 5] ICS09 handshake done sender: [1:7574436095247529717:2118] self: [1:7574436095247529702:0] peer: [5:7574436093711376712:0] socket: 372 qp: -1
2025-11-19T14:00:31.197966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436095247529724:2114] [node 5] ICS10 traffic start
2025-11-19T14:00:31.197976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.212569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436097348265894:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57417
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.215982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436095915737468:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29705
2025-11-19T14:00:31.220332Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436097348265894:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.220786Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436093711376700:2114] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2420) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436093711376705:2116] held: no
2025-11-19T14:00:31.222397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436093711376718:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25887
2025-11-19T14:00:31.222538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436093711376717:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25887
2025-11-19T14:00:31.220279Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436095915737468:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.220316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574436095915737448:2113] [node 5] ICP18 handshake (actor [4:7574436095915737468:2120]) is too old
2025-11-19T14:00:31.224069Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436097348265880:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.224087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436097348265880:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.225514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436091620770080:2050] [node 3] ICP35 event processing took too much time PendingConnection:278(29046254ns) {HandleHandshakeStatus:329(29045366ns) {DropIncomingHandshake:441(115ns) DropOutgoingHandshake:456(120ns) SwitchToState:178(172ns) ProcessPendingSessionEvents:495(28980872ns) {ForwardSessionEventToSession:612(28966049ns) ForwardSessionEventToSession:612(2543ns) ForwardSessionEventToSession:612(937ns) ForwardSessionEventToSession:612(938ns) ForwardSessionEventToSession:612(724ns) ForwardSessionEventToSession:612(801ns) ForwardSessionEventToSession:612(2827ns) ForwardSessionEventToSession:612(730ns) ForwardSessionEventToSession:612(722ns) ForwardSessionEventToSession:612(608ns)}}}
2025-11-19T14:00:31.224229Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436095980475356:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.224245Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436095980475356:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.228165Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436095247529701:2114] [node 5] ICP35 event processing took too much time PendingConnection:278(30130905ns) {HandleHandshakeStatus:329(30129914ns) {DropIncomingHandshake:441(7819ns) DropOutgoingHandshake:456(180ns) SwitchToState:178(47ns) ProcessPendingSessionEvents:495(30049332ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 30022837ns max# 29943246ns}}}
2025-11-19T14:00:31.228771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436089416409328:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436093711376707:2117] held: no
2025-11-19T14:00:31.228939Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436093711376707:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:31.229057Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436093711376705:2116] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:31.229663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436093711376700:2114] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.229681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:31.229690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436093711376700:2114] [node 4] ICP22 created new session: [5:7574436093711376719:2114]
2025-11-19T14:00:31.229697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436093711376719:2114] [node 4] ICS09 handshake done sender: [5:7574436093711376705:2116] self: [5:7574436093711376714:0] peer: [4:7574436095915737449:0] socket: 396 qp: -1
2025-11-19T14:00:31.229704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436093711376719:2114] [node 4] ICS10 traffic start
2025-11-19T14:00:31.229718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:31.230086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436089416409328:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.230095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:31.230098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436089416409328:2048] [node 1] ICP22 created new session: [5:7574436093711376721:2048]
2025-11-19T14:00:31.230102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436093711376721:2048] [node 1] ICS09 handshake done sender: [5:7574436093711376707:2117] self: [5:7574436093711376712:0] peer: [1:7574436095247529702:0] socket: 413 qp: -1
2025-11-19T14:00:31.230106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436093711376721:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:31.230113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:31.237780Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436097348265880:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:31.237901Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436095980475356:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:31.237975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436097348265865:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.238009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:31.238019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436097348265865:2111] [node 4] ICP22 created new session: [2:7574436097348265897:2111]
2025-11-19T14:00:31.238026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436097348265897:2111] [node 4] ICS09 handshake done sender: [2:7574436097348265880:2118] self: [2:7574436097348265875:0] peer: [4:7574436095915737467:0] socket: 417 qp: -1
2025-11-19T14:00:31.238034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436097348265897:2111] [node 4] ICS10 traffic start
2025-11-19T14:00:31.238047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:31.238137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436095980475352:2115] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.238159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:31.238167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436095980475352:2115] [node 5] ICP22 created new session: [3:7574436095980475375:2115]
2025-11-19T14:00:31.238174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436095980475375:2115] [node 5] ICS09 handshake done sender: [3:7574436095980475356:2117] self: [3:7574436095980475353:0] peer: [5:7574436093711376715:0] socket: 399 qp: -1
2025-11-19T14:00:31.238180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436095980475375:2115] [node 5] ICS10 traffic start
2025-11-19T14:00:31.238189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.240683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436091620770079:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18912) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436095915737454:2117] held: no
2025-11-19T14:00:31.240877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436095915737469:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29705
2025-11-19T14:00:31.242246Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436095915737454:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:31.242852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436091620770079:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.242868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:31.242874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436091620770079:2049] [node 2] ICP22 created new session: [4:7574436095915737470:2049]
2025-11-19T14:00:31.242880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436095915737470:2049] [node 2] ICS09 handshake done sender: [4:7574436095915737454:2117] self: [4:7574436095915737467:0] peer: [2:7574436097348265875:0] socket: 387 qp: -1
2025-11-19T14:00:31.242886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436095915737470:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:31.242896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:31.241589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436089416409330:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14564) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436093711376709:2119] held: no
2025-11-19T14:00:31.241757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436093711376723:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25887
2025-11-19T14:00:31.242071Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436093711376709:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:31.242376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436089416409330:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.242388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:31.242395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436089416409330:2050] [node 3] ICP22 created new session: [5:7574436093711376724:2050]
2025-11-19T14:00:31.242400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436093711376724:2050] [node 3] ICS09 handshake done sender: [5:7574436093711376709:2119] self: [5:7574436093711376715:0] peer: [3:7574436095980475353:0] socket: 420 qp: -1
2025-11-19T14:00:31.242407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436093711376724:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:31.242421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:31.289690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:31.289788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436097348265878:2048] [node 1] ICS07 socket disconnect 370 reason# EndOfStream
2025-11-19T14:00:31.289795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436097348265878:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.289846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:31.289865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436097348265878:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:31.293289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436093053298498:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1944) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T14:00:31.293305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436097348265878:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:31.294071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436093053298498:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:31.294089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436097348265878:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:31.294096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436093053298498:2048] [node 1] ICP30 unregister session Session# [2:7574436097348265878:2048] VirtualId# [2:7574436097348265869:0]
2025-11-19T14:00:31.294125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:31.294130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574436093053298498:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.294133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.295904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:31.295994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574436095980475367:2048] [node 1] ICS07 socket disconnect 363 reason# EndOfStream
2025-11-19T14:00:31.296001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436095980475367:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.296055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:31.296078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436095980475367:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:31.296590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436091685507977:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:31.296605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436095980475367:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:31.296611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436091685507977:2048] [node 1] ICP30 unregister session Session# [3:7574436095980475367:2048] VirtualId# [3:7574436095980475344:0]
2025-11-19T14:00:31.296649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:31.296656Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436091685507977:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.296659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.298164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:31.298251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436095915737456:2048] [node 1] ICS01 socket: 361 reason# EndOfStream
2025-11-19T14:00:31.298259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436091620770078:2048] [node 1] ICP30 unregister session Session# [4:7574436095915737456:2048] VirtualId# [4:7574436095915737437:0]
2025-11-19T14:00:31.298265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436095915737456:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.298317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:31.298342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:31.300629Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:31.300731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436093939279704:2048] [node 1] ICS01 socket: 401 reason# EndOfStream
2025-11-19T14:00:31.300739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436093939279608:2048] [node 1] ICP30 unregister session Session# [6:7574436093939279704:2048] VirtualId# [6:7574436093939279697:0]
2025-11-19T14:00:31.300745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436093939279704:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.300802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:31.300826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:31.300861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:31.300934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436093711376721:2048] [node 1] ICS01 socket: 413 reason# EndOfStream
2025-11-19T14:00:31.300938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436089416409328:2048] [node 1] ICP30 unregister session Session# [5:7574436093711376721:2048] VirtualId# [5:7574436093711376712:0]
2025-11-19T14:00:31.300943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436093711376721:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.300969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:31.300991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:31.311815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574436091685507977:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:31.315904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:31.316010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574436095980475370:2049] [node 2] ICS07 socket disconnect 377 reason# EndOfStream
2025-11-19T14:00:31.316016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436095980475370:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.316058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:31.316079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436095980475370:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:31.316538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436091685507978:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18912) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:31.316551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436095980475370:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:31.316556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436091685507978:2049] [node 2] ICP30 unregister session Session# [3:7574436095980475370:2049] VirtualId# [3:7574436095980475354:0]
2025-11-19T14:00:31.316584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:31.316590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436091685507978:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.316593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.327287Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:31.327360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436095915737470:2049] [node 2] ICS01 socket: 387 reason# EndOfStream
2025-11-19T14:00:31.327369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436091620770079:2049] [node 2] ICP30 unregister session Session# [4:7574436095915737470:2049] VirtualId# [4:7574436095915737467:0]
2025-11-19T14:00:31.327375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436095915737470:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.327403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:31.327426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:31.326920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:31.327018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574436093711376710:2049] [node 2] ICS07 socket disconnect 385 reason# EndOfStream
2025-11-19T14:00:31.327024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436093711376710:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.327083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:31.327106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574436093711376710:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:31.327465Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:31.327525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436093939279702:2049] [node 2] ICS07 socket disconnect 409 reason# EndOfStream
2025-11-19T14:00:31.327528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436093939279702:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.327553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:31.327571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436093939279702:2049] [node 2] ICS15 start handshake
2025-11-19T14:00:31.336650Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436093939279609:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18912) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:31.336672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436093939279702:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:31.336679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436093939279609:2049] [node 2] ICP30 unregister session Session# [6:7574436093939279702:2049] VirtualId# [6:7574436093939279696:0]
2025-11-19T14:00:31.336726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:31.336734Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436093939279609:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.336737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.340126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436089416409329:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18912) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:31.340148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436093711376710:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:31.340155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436089416409329:2049] [node 2] ICP30 unregister session Session# [5:7574436093711376710:2049] VirtualId# [5:7574436093711376704:0]
2025-11-19T14:00:31.340201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:31.340207Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574436089416409329:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.340209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.343868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:31.343975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436093711376724:2050] [node 3] ICS01 socket: 420 reason# EndOfStream
2025-11-19T14:00:31.343872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T14:00:31.343972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436095915737465:2050] [node 3] ICS07 socket disconnect 380 reason# EndOfStream
2025-11-19T14:00:31.343979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436095915737465:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.344031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:31.344054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436095915737465:2050] [node 3] ICS15 start handshake
2025-11-19T14:00:31.344284Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:31.344329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436093939279609:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:31.344623Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436091620770080:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14564) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:31.344638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436095915737465:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:31.344643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436091620770080:2050] [node 3] ICP30 unregister session Session# [4:7574436095915737465:2050] VirtualId# [4:7574436095915737455:0]
2025-11-19T14:00:31.344656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:31.344662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436091620770080:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14564) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.344664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14564) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.343983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436089416409330:2050] [node 3] ICP30 unregister session Session# [5:7574436093711376724:2050] VirtualId# [5:7574436093711376715:0]
2025-11-19T14:00:31.343991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436093711376724:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.344036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:31.344057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:31.353018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574436089416409329:2049] [node 2] ICP33 wake up from error state
2025-11-19T14:00:31.351852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436093939279698:2050] [node 3] ICS07 socket disconnect 392 reason# EndOfStream
2025-11-19T14:00:31.351875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436093939279698:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.351950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:31.351976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436093939279698:2050] [node 3] ICS15 start handshake
2025-11-19T14:00:31.354749Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436093939279610:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14564) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:31.354762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436093939279698:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:31.354768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436093939279610:2050] [node 3] ICP30 unregister session Session# [6:7574436093939279698:2050] VirtualId# [6:7574436093939279690:0]
2025-11-19T14:00:31.354812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:31.354819Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436093939279610:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14564) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.354822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14564) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.367167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436093939279610:2050] [node 3] ICP33 wake up from error state
2025-11-19T14:00:31.391093Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:31.391196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436093939279683:2094] [node 4] ICS07 socket disconnect 368 reason# EndOfStream
2025-11-19T14:00:31.391204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436093939279683:2094] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.391273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:31.391296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436093939279683:2094] [node 4] ICS15 start handshake
2025-11-19T14:00:31.392185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436093939279666:2094] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2420) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:31.392199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436093939279683:2094] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:31.392204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436093939279666:2094] [node 4] ICP30 unregister session Session# [6:7574436093939279683:2094] VirtualId# [6:7574436093939279675:0]
2025-11-19T14:00:31.392222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:31.392227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436093939279666:2094] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2420) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.392229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2420) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:31.404050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436093939279666:2094] [node 4] ICP33 wake up from error state
2025-11-19T14:00:31.436037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T14:00:31.436151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436093939279681:2090] [node 5] ICS01 socket: 366 reason# EndOfStream
2025-11-19T14:00:31.436162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436093939279665:2090] [node 5] ICP30 unregister session Session# [6:7574436093939279681:2090] VirtualId# [6:7574436093939279672:0]
2025-11-19T14:00:31.436170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436093939279681:2090] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.436218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T14:00:31.436243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:31.539911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436097388065145:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:31.539931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436097388065146:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:31.572083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436093865843830:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:31.572099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436093865843831:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:31.620020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436094376707595:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:31.620036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436094376707596:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:31.695981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436097204664982:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:31.695998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436097204664983:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:31.696002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436097204664984:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:31.731903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436096608032819:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:31.731920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436096608032820:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:31.731925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436096608032821:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:31.775966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436096430580761:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:31.776015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436096430580762:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:31.776023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436096430580763:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.787087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436094376707655:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21819
2025-11-19T14:00:31.787289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436097204665035:2093] [node 5] ICP01 ready to work
2025-11-19T14:00:31.787406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436097204665037:2096] [node 6] ICP01 ready to work
2025-11-19T14:00:31.788089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436097388065207:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:31.788193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436097388065208:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:31.790303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436096608032861:2073] [node 4] ICP01 ready to work
2025-11-19T14:00:31.790441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436096608032862:2081] [node 6] ICP01 ready to work
2025-11-19T14:00:31.793343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436096430580809:2079] [node 4] ICP01 ready to work
2025-11-19T14:00:31.793456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436096430580810:2085] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.795969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436094376707656:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21819
2025-11-19T14:00:31.796044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436094376707657:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21819
2025-11-19T14:00:31.804483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436093865843897:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:31.806288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436097388065145:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(18038634ns) {Configure:104(18037893ns) {ProcessConfigured:135(17629099ns) {StartInitialHandshake:157(17628240ns) {DropHandshakes:469(530ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(78ns)} GenerateSessionVirtualId:988(74ns) PrepareNewSessionHandshake:476(1262ns) {SwitchToState:178(549ns)}}}}}
2025-11-19T14:00:31.807340Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436094376707656:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.832005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436097388065207:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25649277ns) {Configure:104(25647996ns) {ProcessConfigured:135(25244939ns) {StartInitialHandshake:157(25244009ns) {DropHandshakes:469(651ns) {DropIncomingHandshake:441(168ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:476(1999ns) {SwitchToState:178(874ns)}}}}}
2025-11-19T14:00:31.833032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436097388065221:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:31.833529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436097388065218:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48422
2025-11-19T14:00:31.833611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436097388065219:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48422
2025-11-19T14:00:31.833623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436097388065220:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48422
2025-11-19T14:00:31.833897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436097204664984:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(41923846ns) {Configure:104(41922472ns) {ProcessConfigured:135(41529125ns) {StartInitialHandshake:157(41528922ns) {DropHandshakes:469(313ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(176ns) PrepareNewSessionHandshake:476(1264ns) {SwitchToState:178(557ns)}}}}}
2025-11-19T14:00:31.834998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436097204665053:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10317
2025-11-19T14:00:31.835055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436097204665054:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10317
2025-11-19T14:00:31.835066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436097204665055:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10317
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.842944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436094376707596:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.843263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436094376707667:2111] [node 4] ICP01 ready to work
2025-11-19T14:00:31.843390Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436094376707655:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.843415Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436094376707657:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.843516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436094376707668:2112] [node 6] ICP01 ready to work
2025-11-19T14:00:31.843662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436094376707669:2113] [node 5] ICP01 ready to work
2025-11-19T14:00:31.846490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436094376707670:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21819
2025-11-19T14:00:31.846508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436094376707671:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21819
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.852080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436093865843897:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(46307049ns) {Configure:104(46305630ns) {ProcessConfigured:135(45877217ns) {StartInitialHandshake:157(45876417ns) {DropHandshakes:469(634ns) {DropIncomingHandshake:441(104ns) DropOutgoingHandshake:456(83ns)} GenerateSessionVirtualId:988(436ns) PrepareNewSessionHandshake:476(1581ns) {SwitchToState:178(800ns)}}}}}
2025-11-19T14:00:31.852295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436093865843903:2111] [node 6] ICP01 ready to work
2025-11-19T14:00:31.852472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436093865843904:2112] [node 5] ICP01 ready to work
2025-11-19T14:00:31.852775Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093865843889:2105] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.852782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436093865843889:2105] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.853205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436093865843898:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29544
2025-11-19T14:00:31.854046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436093865843905:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29544
2025-11-19T14:00:31.854058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436093865843906:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29544
2025-11-19T14:00:31.854357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436096608032897:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40298
2025-11-19T14:00:31.854384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436096608032898:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40298
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.857796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436096430580839:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.865117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436094376707682:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21819
2025-11-19T14:00:31.865337Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436097388065218:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.869938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436097388065219:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.868594Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436097204665053:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.871694Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436097204665055:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.873539Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436093865843889:2105] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:31.873866Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436097388065220:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.873984Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436096608032898:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.874013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436096608032821:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.874180Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093865843898:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.874241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436093865843831:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.874255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:31.874261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436093865843831:2049] [node 3] ICP22 created new session: [2:7574436093865843913:2049]
2025-11-19T14:00:31.874267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436093865843913:2049] [node 3] ICS09 handshake done sender: [2:7574436093865843889:2105] self: [2:7574436093865843888:0] peer: [3:7574436094376707666:0] socket: 368 qp: -1
2025-11-19T14:00:31.874275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436093865843913:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:31.874283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.876000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436096430580840:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-11-19T14:00:31.876187Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436094376707670:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.876228Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436094376707673:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.876230Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436094376707673:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.877921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436096608032900:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40298
2025-11-19T14:00:31.878075Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093865843905:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.878102Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093865843906:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.879392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436093865843830:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.879403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436093865843831:2049] [node 3] ICP18 handshake (actor [2:7574436093865843906:2115]) is too old
2025-11-19T14:00:31.876734Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436094376707656:2108] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:31.876896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436094376707596:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.876910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:31.876917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436094376707596:2049] [node 2] ICP22 created new session: [3:7574436094376707684:2049]
2025-11-19T14:00:31.876922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436094376707684:2049] [node 2] ICS09 handshake done sender: [3:7574436094376707656:2108] self: [3:7574436094376707666:0] peer: [2:7574436093865843888:0] socket: 373 qp: -1
2025-11-19T14:00:31.876928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436094376707684:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:31.876935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:31.877280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436094376707595:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.879705Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436097388065211:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.879718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436097388065211:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.880230Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436097388065211:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:31.880319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436097388065145:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.880342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:31.880347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436097388065145:2048] [node 2] ICP22 created new session: [1:7574436097388065231:2048]
2025-11-19T14:00:31.880352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436097388065231:2048] [node 2] ICS09 handshake done sender: [1:7574436097388065211:2110] self: [1:7574436097388065209:0] peer: [2:7574436093865843917:0] socket: 382 qp: -1
2025-11-19T14:00:31.880359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436097388065231:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:31.880366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.882437Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436096608032898:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:31.882520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436096608032821:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.882535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:31.882541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436096608032821:2050] [node 3] ICP22 created new session: [5:7574436096608032901:2050]
2025-11-19T14:00:31.882548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436096608032901:2050] [node 3] ICS09 handshake done sender: [5:7574436096608032898:2116] self: [5:7574436096608032899:0] peer: [3:7574436094376707672:0] socket: 400 qp: -1
2025-11-19T14:00:31.882553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436096608032901:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:31.882559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:31.880980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436096430580841:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.884623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436097204665063:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10317
2025-11-19T14:00:31.884751Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436097204665054:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.884835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436097204664982:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.882266Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436094376707673:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:31.885235Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436094376707671:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.885243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436094376707669:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.885255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:31.885260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436094376707669:2113] [node 5] ICP22 created new session: [3:7574436094376707687:2113]
2025-11-19T14:00:31.885264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436094376707687:2113] [node 5] ICS09 handshake done sender: [3:7574436094376707673:2117] self: [3:7574436094376707672:0] peer: [5:7574436096608032899:0] socket: 395 qp: -1
2025-11-19T14:00:31.885269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436094376707687:2113] [node 5] ICS10 traffic start
2025-11-19T14:00:31.885275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:31.885700Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436097204665052:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.885707Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436097204665052:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.886442Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436097388065215:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.886452Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436097388065215:2112] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.886878Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436097388065213:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.886880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436097388065213:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.887334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436097388065234:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48422
2025-11-19T14:00:31.884995Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436096430580839:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.885022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436096430580809:2079] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.886330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436096430580843:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-11-19T14:00:31.887491Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436096430580841:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.887516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436096430580761:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.888083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436096430580845:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-11-19T14:00:31.887979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436094376707691:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21819
2025-11-19T14:00:31.891898Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436097204665052:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:31.894319Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436096608032897:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.894673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436096608032861:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.892139Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436096430580839:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:31.892220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436096430580809:2079] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.892233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:31.892240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436096430580809:2079] [node 4] ICP22 created new session: [6:7574436096430580846:2079]
2025-11-19T14:00:31.892246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436096430580846:2079] [node 4] ICS09 handshake done sender: [6:7574436096430580839:2115] self: [6:7574436096430580842:0] peer: [4:7574436097204665051:0] socket: 401 qp: -1
2025-11-19T14:00:31.892253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436096430580846:2079] [node 4] ICS10 traffic start
2025-11-19T14:00:31.892262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:31.900049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436097204665040:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.900063Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436097204665040:2102] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.900352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436097204665037:2096] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.900369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:31.900376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436097204665037:2096] [node 6] ICP22 created new session: [4:7574436097204665070:2096]
2025-11-19T14:00:31.900383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436097204665070:2096] [node 6] ICS09 handshake done sender: [4:7574436097204665052:2112] self: [4:7574436097204665051:0] peer: [6:7574436096430580842:0] socket: 390 qp: -1
2025-11-19T14:00:31.900390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436097204665070:2096] [node 6] ICS10 traffic start
2025-11-19T14:00:31.900399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:31.904355Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436097388065215:2112] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:31.904562Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436097388065213:2111] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:31.908195Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436096430580840:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.908300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436096430580762:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.908572Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436097388065217:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.908584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436097388065217:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.909069Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436097388065234:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.909078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436097388065146:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.909096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:31.909105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436097388065146:2049] [node 3] ICP22 created new session: [1:7574436097388065237:2049]
2025-11-19T14:00:31.909111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436097388065237:2049] [node 3] ICS09 handshake done sender: [1:7574436097388065215:2112] self: [1:7574436097388065214:0] peer: [3:7574436094376707686:0] socket: 384 qp: -1
2025-11-19T14:00:31.909118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436097388065237:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:31.909129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:31.909389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436097388065207:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.909396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:31.909402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436097388065207:2107] [node 4] ICP22 created new session: [1:7574436097388065239:2107]
2025-11-19T14:00:31.909407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436097388065239:2107] [node 4] ICS09 handshake done sender: [1:7574436097388065213:2111] self: [1:7574436097388065212:0] peer: [4:7574436097204665065:0] socket: 383 qp: -1
2025-11-19T14:00:31.909413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436097388065239:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:31.909419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.910402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436097204665068:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10317
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.910595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436096608032905:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40298
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.910862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436093865843830:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9917) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436093865843905:2114] held: no
2025-11-19T14:00:31.910993Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093865843909:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.910999Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436093865843909:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.911638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436093865843918:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29544
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.911906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436097204665069:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10317
2025-11-19T14:00:31.912003Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436097204665063:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.912137Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436097204665055:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:31.912330Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436097204665040:2102] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:31.912533Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436096608032897:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:31.912625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436096608032861:2073] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.912643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:31.912650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436096608032861:2073] [node 4] ICP22 created new session: [5:7574436096608032906:2073]
2025-11-19T14:00:31.912656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436096608032906:2073] [node 4] ICS09 handshake done sender: [5:7574436096608032897:2115] self: [5:7574436096608032904:0] peer: [4:7574436097204665038:0] socket: 391 qp: -1
2025-11-19T14:00:31.912662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436096608032906:2073] [node 4] ICS10 traffic start
2025-11-19T14:00:31.912672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.913976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436096430580850:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-11-19T14:00:31.915233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436097204664982:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.915252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:31.915261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436097204664982:2048] [node 1] ICP22 created new session: [4:7574436097204665074:2048]
2025-11-19T14:00:31.915267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436097204665074:2048] [node 1] ICS09 handshake done sender: [4:7574436097204665055:2115] self: [4:7574436097204665065:0] peer: [1:7574436097388065212:0] socket: 387 qp: -1
2025-11-19T14:00:31.915273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436097204665074:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:31.915279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:31.915607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436097204664983:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.915616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436097204665035:2093] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.915626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:31.915630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436097204665035:2093] [node 5] ICP22 created new session: [4:7574436097204665077:2093]
2025-11-19T14:00:31.915634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436097204665077:2093] [node 5] ICS09 handshake done sender: [4:7574436097204665040:2102] self: [4:7574436097204665038:0] peer: [5:7574436096608032904:0] socket: 388 qp: -1
2025-11-19T14:00:31.915637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436097204665077:2093] [node 5] ICS10 traffic start
2025-11-19T14:00:31.915643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.916035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436093865843919:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29544
2025-11-19T14:00:31.916237Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436093865843905:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.916408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436096430580851:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.917033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436096430580852:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-11-19T14:00:31.917165Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436093865843909:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:31.917325Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436096430580840:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:31.917381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436096430580762:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.917394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:31.917399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436096430580762:2049] [node 2] ICP22 created new session: [6:7574436096430580853:2049]
2025-11-19T14:00:31.917404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436096430580853:2049] [node 2] ICS09 handshake done sender: [6:7574436096430580840:2116] self: [6:7574436096430580849:0] peer: [2:7574436093865843908:0] socket: 405 qp: -1
2025-11-19T14:00:31.917409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436096430580853:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:31.917415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.918180Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093865843919:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.918193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436093865843830:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.918203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:31.918208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436093865843830:2048] [node 1] ICP22 created new session: [2:7574436093865843923:2048]
2025-11-19T14:00:31.918213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436093865843923:2048] [node 1] ICS09 handshake done sender: [2:7574436093865843905:2114] self: [2:7574436093865843917:0] peer: [1:7574436097388065209:0] socket: 396 qp: -1
2025-11-19T14:00:31.918218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436093865843923:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:31.918223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:31.918376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436093865843903:2111] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.918381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:31.918385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436093865843903:2111] [node 6] ICP22 created new session: [2:7574436093865843925:2111]
2025-11-19T14:00:31.918390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436093865843925:2111] [node 6] ICS09 handshake done sender: [2:7574436093865843909:2116] self: [2:7574436093865843908:0] peer: [6:7574436096430580849:0] socket: 404 qp: -1
2025-11-19T14:00:31.918393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436093865843925:2111] [node 6] ICS10 traffic start
2025-11-19T14:00:31.918398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:31.918660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436093865843922:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29544
2025-11-19T14:00:31.918740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436093865843903:2111] [node 6] ICP18 handshake (actor [2:7574436093865843919:2119]) is too old
2025-11-19T14:00:31.920772Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436097388065217:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:31.920964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436097388065208:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.920979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:31.920993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436097388065208:2108] [node 6] ICP22 created new session: [1:7574436097388065244:2108]
2025-11-19T14:00:31.920998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436097388065244:2108] [node 6] ICS09 handshake done sender: [1:7574436097388065217:2113] self: [1:7574436097388065216:0] peer: [6:7574436096430580844:0] socket: 379 qp: -1
2025-11-19T14:00:31.921002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436097388065244:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:31.921009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:31.921351Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436096430580845:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.921779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436096608032909:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40298
2025-11-19T14:00:31.921796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436096608032910:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40298
2025-11-19T14:00:31.921805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436096608032911:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40298
2025-11-19T14:00:31.921990Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436096430580850:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.923104Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436096430580841:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:31.925332Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093865843902:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.925344Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436093865843902:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.927002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436094376707595:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9917) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436094376707670:2115] held: no
2025-11-19T14:00:31.927104Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436094376707670:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:31.932034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436094376707595:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.932059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:31.932070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436094376707595:2048] [node 1] ICP22 created new session: [3:7574436094376707693:2048]
2025-11-19T14:00:31.932077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436094376707693:2048] [node 1] ICS09 handshake done sender: [3:7574436094376707670:2115] self: [3:7574436094376707686:0] peer: [1:7574436097388065214:0] socket: 392 qp: -1
2025-11-19T14:00:31.932086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436094376707693:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:31.932096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.933285Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436097204665069:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.933395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436097204664984:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.933415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436097204665079:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10317
2025-11-19T14:00:31.940838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436096430580763:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.941451Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436094376707675:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.941464Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436094376707675:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.942377Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436096608032910:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.942408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436096608032819:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.942533Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436097388065223:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.942544Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436097388065223:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.943241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436096608032913:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40298
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.943345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436097204665081:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10317
2025-11-19T14:00:31.943465Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436093865843902:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:31.943655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436096430580761:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.943672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:31.943694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436096430580761:2048] [node 1] ICP22 created new session: [6:7574436096430580857:2048]
2025-11-19T14:00:31.943701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436096430580857:2048] [node 1] ICS09 handshake done sender: [6:7574436096430580841:2117] self: [6:7574436096430580844:0] peer: [1:7574436097388065216:0] socket: 409 qp: -1
2025-11-19T14:00:31.943708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436096430580857:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:31.943716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:31.943878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574436096430580761:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:31.944123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436096430580810:2085] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.944358Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436096608032911:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.944389Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436096608032872:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.944393Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436096608032872:2093] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.945060Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436096608032872:2093] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:31.945128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436096608032862:2081] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.945142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:31.945147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436096608032862:2081] [node 6] ICP22 created new session: [5:7574436096608032916:2081]
2025-11-19T14:00:31.945152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436096608032916:2081] [node 6] ICS09 handshake done sender: [5:7574436096608032872:2093] self: [5:7574436096608032871:0] peer: [6:7574436096430580859:0] socket: 371 qp: -1
2025-11-19T14:00:31.945157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436096608032916:2081] [node 6] ICS10 traffic start
2025-11-19T14:00:31.945165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:31.945805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436093865843897:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.945821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:31.945826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436093865843897:2109] [node 4] ICP22 created new session: [2:7574436093865843928:2109]
2025-11-19T14:00:31.945831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436093865843928:2109] [node 4] ICS09 handshake done sender: [2:7574436093865843902:2113] self: [2:7574436093865843900:0] peer: [4:7574436097204665076:0] socket: 399 qp: -1
2025-11-19T14:00:31.945836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436093865843928:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:31.945843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:31.946128Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436096608032909:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.946149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436096608032820:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:31.946262Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436094376707675:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:31.947886Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436094376707677:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.947894Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436094376707677:2119] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.948329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436094376707667:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.948342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:31.948347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436094376707667:2111] [node 4] ICP22 created new session: [3:7574436094376707696:2111]
2025-11-19T14:00:31.948352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436094376707696:2111] [node 4] ICS09 handshake done sender: [3:7574436094376707675:2118] self: [3:7574436094376707674:0] peer: [4:7574436097204665080:0] socket: 376 qp: -1
2025-11-19T14:00:31.948357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436094376707696:2111] [node 4] ICS10 traffic start
2025-11-19T14:00:31.948364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:31.948837Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436097204665063:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:31.948918Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436097204665069:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:31.948970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436097204664983:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.948983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:31.948988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436097204664983:2049] [node 2] ICP22 created new session: [4:7574436097204665082:2049]
2025-11-19T14:00:31.948993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436097204665082:2049] [node 2] ICS09 handshake done sender: [4:7574436097204665063:2118] self: [4:7574436097204665076:0] peer: [2:7574436093865843900:0] socket: 403 qp: -1
2025-11-19T14:00:31.948997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436097204665082:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:31.949004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:31.949326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436097204664984:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.949332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:31.949336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436097204664984:2050] [node 3] ICP22 created new session: [4:7574436097204665084:2050]
2025-11-19T14:00:31.949339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436097204665084:2050] [node 3] ICS09 handshake done sender: [4:7574436097204665069:2120] self: [4:7574436097204665080:0] peer: [3:7574436094376707674:0] socket: 418 qp: -1
2025-11-19T14:00:31.949343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436097204665084:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:31.949351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:31.950328Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093865843911:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.950339Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436093865843911:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:31.950766Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436093865843922:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:31.951108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436093865843897:2109] [node 4] ICP18 handshake (actor [2:7574436093865843922:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.951341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436096608032919:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40298
2025-11-19T14:00:31.951447Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436097388065223:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:31.951625Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436096608032910:2120] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:31.951695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436096608032819:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.951708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:31.951712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436096608032819:2048] [node 1] ICP22 created new session: [5:7574436096608032920:2048]
2025-11-19T14:00:31.951717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436096608032920:2048] [node 1] ICS09 handshake done sender: [5:7574436096608032910:2120] self: [5:7574436096608032912:0] peer: [1:7574436097388065222:0] socket: 429 qp: -1
2025-11-19T14:00:31.951722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436096608032920:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:31.951727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:31.952844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436097388065221:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.952862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:31.952867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436097388065221:2109] [node 5] ICP22 created new session: [1:7574436097388065247:2109]
2025-11-19T14:00:31.952872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436097388065247:2109] [node 5] ICS09 handshake done sender: [1:7574436097388065223:2117] self: [1:7574436097388065222:0] peer: [5:7574436096608032912:0] socket: 426 qp: -1
2025-11-19T14:00:31.952876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436097388065247:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:31.952883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:31.955272Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436093865843911:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.960027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436096430580810:2085] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27293) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436096430580850:2120] held: no
2025-11-19T14:00:31.960266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436096430580860:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-11-19T14:00:31.960403Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436094376707677:2119] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:31.960529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436093865843904:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.960552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:31.960557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436093865843904:2112] [node 5] ICP22 created new session: [2:7574436093865843931:2112]
2025-11-19T14:00:31.960563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436093865843931:2112] [node 5] ICS09 handshake done sender: [2:7574436093865843911:2117] self: [2:7574436093865843910:0] peer: [5:7574436096608032918:0] socket: 424 qp: -1
2025-11-19T14:00:31.960568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436093865843931:2112] [node 5] ICS10 traffic start
2025-11-19T14:00:31.960577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:31.961034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436094376707668:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:31.961050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:31.961054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436094376707668:2112] [node 6] ICP22 created new session: [3:7574436094376707699:2112]
2025-11-19T14:00:31.961059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436094376707699:2112] [node 6] ICS09 handshake done sender: [3:7574436094376707677:2119] self: [3:7574436094376707676:0] peer: [6:7574436096430580856:0] socket: 414 qp: -1
2025-11-19T14:00:31.961063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436094376707699:2112] [node 6] ICS10 traffic start
2025-11-19T14:00:31.961071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:31.961458Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436096608032909:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:31.962377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436096608032820:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.962394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:31.962399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436096608032820:2049] [node 2] ICP22 created new session: [5:7574436096608032922:2049]
2025-11-19T14:00:31.962404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436096608032922:2049] [node 2] ICS09 handshake done sender: [5:7574436096608032909:2119] self: [5:7574436096608032918:0] peer: [2:7574436093865843910:0] socket: 428 qp: -1
2025-11-19T14:00:31.962409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436096608032922:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:31.962416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:31.963383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436096430580861:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-11-19T14:00:31.963999Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436096430580850:2120] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:31.965813Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436096430580763:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15189) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436096430580845:2119] held: no
2025-11-19T14:00:31.965902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436096430580810:2085] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.965910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:31.965915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436096430580810:2085] [node 5] ICP22 created new session: [6:7574436096430580862:2085]
2025-11-19T14:00:31.965920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436096430580862:2085] [node 5] ICS09 handshake done sender: [6:7574436096430580850:2120] self: [6:7574436096430580859:0] peer: [5:7574436096608032871:0] socket: 421 qp: -1
2025-11-19T14:00:31.965925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436096430580862:2085] [node 5] ICS10 traffic start
2025-11-19T14:00:31.965933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:31.966105Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436096430580845:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:31.966408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436096430580763:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:31.966417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:31.966422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436096430580763:2050] [node 3] ICP22 created new session: [6:7574436096430580864:2050]
2025-11-19T14:00:31.966426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436096430580864:2050] [node 3] ICS09 handshake done sender: [6:7574436096430580845:2119] self: [6:7574436096430580856:0] peer: [3:7574436094376707676:0] socket: 416 qp: -1
2025-11-19T14:00:31.966430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436096430580864:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:31.966436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:31.983897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:31.983999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436093865843923:2048] [node 1] ICS01 socket: 396 reason# EndOfStream
2025-11-19T14:00:31.984010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436093865843830:2048] [node 1] ICP30 unregister session Session# [2:7574436093865843923:2048] VirtualId# [2:7574436093865843917:0]
2025-11-19T14:00:31.984020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436093865843923:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.984056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:31.984077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:31.984432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:31.984495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436094376707693:2048] [node 1] ICS01 socket: 392 reason# EndOfStream
2025-11-19T14:00:31.984500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436094376707595:2048] [node 1] ICP30 unregister session Session# [3:7574436094376707693:2048] VirtualId# [3:7574436094376707686:0]
2025-11-19T14:00:31.984504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436094376707693:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:31.984529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:31.984545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:32.006405Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:32.006511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436097204665074:2048] [node 1] ICS01 socket: 387 reason# EndOfStream
2025-11-19T14:00:32.006521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436097204664982:2048] [node 1] ICP30 unregister session Session# [4:7574436097204665074:2048] VirtualId# [4:7574436097204665065:0]
2025-11-19T14:00:32.006528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436097204665074:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.006587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:32.006615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:32.006663Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:32.006725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436096430580857:2048] [node 1] ICS01 socket: 409 reason# EndOfStream
2025-11-19T14:00:32.006730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436096430580761:2048] [node 1] ICP30 unregister session Session# [6:7574436096430580857:2048] VirtualId# [6:7574436096430580844:0]
2025-11-19T14:00:32.006735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436096430580857:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.006763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:32.006784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:32.006816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:32.006876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436096608032920:2048] [node 1] ICS01 socket: 429 reason# EndOfStream
2025-11-19T14:00:32.006881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436096608032819:2048] [node 1] ICP30 unregister session Session# [5:7574436096608032920:2048] VirtualId# [5:7574436096608032912:0]
2025-11-19T14:00:32.006885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436096608032920:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.006911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:32.006932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:32.012583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:32.012698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436094376707684:2049] [node 2] ICS01 socket: 373 reason# EndOfStream
2025-11-19T14:00:32.012707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436094376707596:2049] [node 2] ICP30 unregister session Session# [3:7574436094376707684:2049] VirtualId# [3:7574436094376707666:0]
2025-11-19T14:00:32.012717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436094376707684:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.012763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:32.012786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:32.044773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:32.044869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436097204665082:2049] [node 2] ICS01 socket: 403 reason# EndOfStream
2025-11-19T14:00:32.044878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436097204664983:2049] [node 2] ICP30 unregister session Session# [4:7574436097204665082:2049] VirtualId# [4:7574436097204665076:0]
2025-11-19T14:00:32.044885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436097204665082:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.044941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:32.044965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:32.045004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:32.045072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436096430580853:2049] [node 2] ICS01 socket: 405 reason# EndOfStream
2025-11-19T14:00:32.045076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436096430580762:2049] [node 2] ICP30 unregister session Session# [6:7574436096430580853:2049] VirtualId# [6:7574436096430580849:0]
2025-11-19T14:00:32.045080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436096430580853:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.045107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:32.045127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:32.045155Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:32.045214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436096608032922:2049] [node 2] ICS01 socket: 428 reason# EndOfStream
2025-11-19T14:00:32.045218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436096608032820:2049] [node 2] ICP30 unregister session Session# [5:7574436096608032922:2049] VirtualId# [5:7574436096608032918:0]
2025-11-19T14:00:32.045222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436096608032922:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.045244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:32.045264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:32.053335Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:32.053429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436096608032901:2050] [node 3] ICS01 socket: 400 reason# EndOfStream
2025-11-19T14:00:32.053440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436096608032821:2050] [node 3] ICP30 unregister session Session# [5:7574436096608032901:2050] VirtualId# [5:7574436096608032899:0]
2025-11-19T14:00:32.053447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436096608032901:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.053494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:32.053517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:32.053608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:32.053665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436096430580864:2050] [node 3] ICS01 socket: 416 reason# EndOfStream
2025-11-19T14:00:32.053670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436096430580763:2050] [node 3] ICP30 unregister session Session# [6:7574436096430580864:2050] VirtualId# [6:7574436096430580856:0]
2025-11-19T14:00:32.053675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436096430580864:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.053716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:32.053736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:32.067581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:32.067691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436096430580846:2079] [node 4] ICS01 socket: 401 reason# EndOfStream
2025-11-19T14:00:32.067700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436096430580809:2079] [node 4] ICP30 unregister session Session# [6:7574436096430580846:2079] VirtualId# [6:7574436096430580842:0]
2025-11-19T14:00:32.067707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436096430580846:2079] [node 4] ICS25 shutdown socket, reason# EndOfStream
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:32.502948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436099847780996:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:32.502964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436099847780997:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:32.528514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436100009706651:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:32.528530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436100009706652:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:32.543574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436101848039941:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:32.543589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436101848039942:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:32.560265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436101603562243:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:32.560281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436101603562244:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:32.560287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436101603562245:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:32.603940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436098523145744:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:32.603962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436098523145745:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:32.603967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436098523145746:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:32.631861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436099154018854:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:32.631880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436099154018855:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:32.631886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436099154018856:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:32.644150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436101603562300:2102] [node 5] ICP01 ready to work
2025-11-19T14:00:32.647256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436100009706716:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:32.647980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436099847781058:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:32.648099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436099847781059:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.652723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436099847781068:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:32.653025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436099847781062:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4158
2025-11-19T14:00:32.653038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436099847781063:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4158
2025-11-19T14:00:32.664010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436101848040005:2104] [node 4] ICP01 ready to work
2025-11-19T14:00:32.664111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436101848040006:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:32.663205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436100009706711:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.669326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436100009706721:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:32.668073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436098523145807:2097] [node 4] ICP01 ready to work
2025-11-19T14:00:32.668240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436098523145792:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3438
2025-11-19T14:00:32.668894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436098523145808:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3438
2025-11-19T14:00:32.672000Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436099847780996:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(23854513ns) {Configure:104(23853601ns) {ProcessConfigured:135(23474003ns) {StartInitialHandshake:157(23473537ns) {DropHandshakes:469(448ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(67ns)} GenerateSessionVirtualId:988(109ns) PrepareNewSessionHandshake:476(1528ns) {SwitchToState:178(903ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.687342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436099154018855:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(43312850ns) {Configure:104(43311865ns) {ProcessConfigured:135(42550546ns) {StartInitialHandshake:157(42550232ns) {DropHandshakes:469(208ns) {DropIncomingHandshake:441(24ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:476(775ns) {SwitchToState:178(225ns)}}}}}
2025-11-19T14:00:32.687481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436099154018920:2064] [node 4] ICP01 ready to work
2025-11-19T14:00:32.687612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436099154018921:2070] [node 5] ICP01 ready to work
2025-11-19T14:00:32.685217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436101848040014:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:32.686954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436101848040010:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61434
2025-11-19T14:00:32.686969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436101848040011:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61434
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.689420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436100009706719:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51745
2025-11-19T14:00:32.689468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436100009706720:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51745
2025-11-19T14:00:32.689480Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436100009706651:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(26143873ns) {Configure:104(26142195ns) {ProcessConfigured:135(25771944ns) {StartInitialHandshake:157(25771481ns) {DropHandshakes:469(1529ns) {DropIncomingHandshake:441(121ns) DropOutgoingHandshake:456(142ns)} GenerateSessionVirtualId:988(390ns) PrepareNewSessionHandshake:476(583ns) {SwitchToState:178(267ns)}}}}}
2025-11-19T14:00:32.689651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436100009706732:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51745
2025-11-19T14:00:32.689201Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436099847781062:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.690124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436101603562316:2112] [node 6] ICP01 ready to work
2025-11-19T14:00:32.689707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436101848040005:2104] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25538649ns) {Configure:104(25538367ns) {ProcessConfigured:135(25220554ns) {StartInitialHandshake:157(25220203ns) {DropHandshakes:469(720ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(113ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(662ns) {SwitchToState:178(36ns)}}}}}
2025-11-19T14:00:32.692228Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436100009706719:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.693187Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436101848040011:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.693226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436101848039942:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:32.692336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436098523145798:2095] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.699099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436099154018931:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29294
2025-11-19T14:00:32.699127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436099154018932:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29294
2025-11-19T14:00:32.699917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436099154018856:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(55185018ns) {Configure:104(55184110ns) {ProcessConfigured:135(54879983ns) {StartInitialHandshake:157(54878856ns) {DropHandshakes:469(337ns) {DropIncomingHandshake:441(90ns) DropOutgoingHandshake:456(54ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:476(1526ns) {SwitchToState:178(637ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.701360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436098523145822:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3438
2025-11-19T14:00:32.701419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436098523145823:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3438
2025-11-19T14:00:32.707859Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436098523145822:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.707907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436098523145746:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:32.708049Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436101848040013:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.708061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436101848040013:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.710258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436101603562317:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35696
2025-11-19T14:00:32.710277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436101603562318:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35696
2025-11-19T14:00:32.710316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436101603562319:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35696
2025-11-19T14:00:32.712035Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436101603562318:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.712123Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436098523145792:2093] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.712158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436098523145807:2097] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:32.712293Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436101603562310:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.712301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436101603562310:2108] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.716048Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436100009706713:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.716061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436100009706713:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.716875Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436100009706720:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.716922Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436100009706713:2107] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:32.723891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436099847780997:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(51594393ns) {Configure:104(51593728ns) {ProcessConfigured:135(51238935ns) {StartInitialHandshake:157(51238306ns) {DropHandshakes:469(468ns) {DropIncomingHandshake:441(60ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:476(963ns) {SwitchToState:178(311ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.727066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436099847781076:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4158
2025-11-19T14:00:32.727887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436100009706652:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.727920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:32.727927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436100009706652:2049] [node 3] ICP22 created new session: [2:7574436100009706735:2049]
2025-11-19T14:00:32.727936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436100009706735:2049] [node 3] ICS09 handshake done sender: [2:7574436100009706713:2107] self: [2:7574436100009706712:0] peer: [3:7574436101848040022:0] socket: 376 qp: -1
2025-11-19T14:00:32.727945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436100009706735:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:32.727954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:32.732687Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436099154018932:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.732755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436099154018855:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:32.733609Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436098523145823:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.733642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436098523145745:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:32.733715Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436101603562310:2108] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:32.733829Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436101848040013:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:32.736067Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436099847781063:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.736138Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436099847781076:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.739984Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436098523145808:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.740036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436098523145744:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:32.739907Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436100009706725:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.739923Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436100009706725:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.742272Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436099154018931:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.742309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436099154018854:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:32.744146Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436099847781067:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.744162Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436099847781067:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.744261Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436100009706723:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.744274Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436100009706723:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.744348Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436100009706725:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.748378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436098523145830:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3438
2025-11-19T14:00:32.748439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436098523145831:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3438
2025-11-19T14:00:32.748605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436098523145832:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3438
2025-11-19T14:00:32.748732Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436098523145792:2093] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:32.752129Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436098523145823:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.752257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436098523145807:2097] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.752277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:32.752285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436098523145807:2097] [node 4] ICP22 created new session: [5:7574436098523145833:2097]
2025-11-19T14:00:32.752293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436098523145833:2097] [node 4] ICS09 handshake done sender: [5:7574436098523145792:2093] self: [5:7574436098523145827:0] peer: [4:7574436101603562309:0] socket: 378 qp: -1
2025-11-19T14:00:32.752301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436098523145833:2097] [node 4] ICS10 traffic start
2025-11-19T14:00:32.752309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:32.752745Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436098523145822:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:32.751959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436101603562300:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.751989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:32.751998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436101603562300:2102] [node 5] ICP22 created new session: [4:7574436101603562327:2102]
2025-11-19T14:00:32.752005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436101603562327:2102] [node 5] ICS09 handshake done sender: [4:7574436101603562310:2108] self: [4:7574436101603562309:0] peer: [5:7574436098523145827:0] socket: 377 qp: -1
2025-11-19T14:00:32.752013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436101603562327:2102] [node 5] ICS10 traffic start
2025-11-19T14:00:32.752024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:32.752308Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436101603562317:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.752605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436101603562326:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35696
2025-11-19T14:00:32.752728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436101603562243:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.753780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436098523145745:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.753797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:32.753804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436098523145745:2049] [node 2] ICP22 created new session: [5:7574436098523145835:2049]
2025-11-19T14:00:32.753811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436098523145835:2049] [node 2] ICS09 handshake done sender: [5:7574436098523145823:2117] self: [5:7574436098523145828:0] peer: [2:7574436100009706724:0] socket: 404 qp: -1
2025-11-19T14:00:32.753818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436098523145835:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:32.753825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:32.754045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436098523145746:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.754052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:32.754056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436098523145746:2050] [node 3] ICP22 created new session: [5:7574436098523145837:2050]
2025-11-19T14:00:32.754060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436098523145837:2050] [node 3] ICS09 handshake done sender: [5:7574436098523145822:2116] self: [5:7574436098523145826:0] peer: [3:7574436101848040012:0] socket: 403 qp: -1
2025-11-19T14:00:32.754065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436098523145837:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:32.754070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:32.752947Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436101848040010:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.752967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436101848040006:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.752978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:32.752984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436101848040006:2107] [node 5] ICP22 created new session: [3:7574436101848040025:2107]
2025-11-19T14:00:32.752991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436101848040025:2107] [node 5] ICS09 handshake done sender: [3:7574436101848040013:2113] self: [3:7574436101848040012:0] peer: [5:7574436098523145826:0] socket: 392 qp: -1
2025-11-19T14:00:32.752996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436101848040025:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:32.753004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:32.753214Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436101848039942:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8650) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436101848040011:2112] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.760059Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436099847781070:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.760074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436099847781070:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.760539Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436099847781065:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.760542Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436099847781065:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.761121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436099847781081:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4158
2025-11-19T14:00:32.763957Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436099847781070:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:32.764120Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436099847781065:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:32.764323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436099847781068:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.764337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:32.764344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436099847781068:2109] [node 6] ICP22 created new session: [1:7574436099847781084:2109]
2025-11-19T14:00:32.764351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436099847781084:2109] [node 6] ICS09 handshake done sender: [1:7574436099847781070:2114] self: [1:7574436099847781069:0] peer: [6:7574436099154018935:0] socket: 384 qp: -1
2025-11-19T14:00:32.764359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436099847781084:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:32.764384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:32.764618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436099847781058:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.764628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:32.764634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436099847781058:2107] [node 4] ICP22 created new session: [1:7574436099847781086:2107]
2025-11-19T14:00:32.764638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436099847781086:2107] [node 4] ICS09 handshake done sender: [1:7574436099847781065:2112] self: [1:7574436099847781064:0] peer: [4:7574436101603562330:0] socket: 382 qp: -1
2025-11-19T14:00:32.764641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436099847781086:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:32.764646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.767184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436100009706711:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.767215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:32.767224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436100009706711:2106] [node 5] ICP22 created new session: [2:7574436100009706740:2106]
2025-11-19T14:00:32.767231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436100009706740:2106] [node 5] ICS09 handshake done sender: [2:7574436100009706725:2113] self: [2:7574436100009706724:0] peer: [5:7574436098523145828:0] socket: 400 qp: -1
2025-11-19T14:00:32.767237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436100009706740:2106] [node 5] ICS10 traffic start
2025-11-19T14:00:32.767249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:32.768227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436100009706742:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51745
2025-11-19T14:00:32.769328Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436100009706723:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:32.769809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436100009706716:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.769822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:32.769830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436100009706716:2108] [node 6] ICP22 created new session: [2:7574436100009706745:2108]
2025-11-19T14:00:32.769836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436100009706745:2108] [node 6] ICS09 handshake done sender: [2:7574436100009706723:2112] self: [2:7574436100009706722:0] peer: [6:7574436099154018934:0] socket: 399 qp: -1
2025-11-19T14:00:32.769843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436100009706745:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:32.769850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:32.772395Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436100009706732:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.772422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436100009706716:2108] [node 6] ICP18 handshake (actor [2:7574436100009706732:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.772412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436101848040024:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61434
2025-11-19T14:00:32.772727Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436099847781081:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.773057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436099847781091:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4158
2025-11-19T14:00:32.773089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436099847781068:2109] [node 6] ICP18 handshake (actor [1:7574436099847781081:2118]) is too old
2025-11-19T14:00:32.775429Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436101848040011:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.773609Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436101603562326:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.775839Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436101603562243:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15888) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436101603562317:2113] held: no
2025-11-19T14:00:32.776056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436101603562331:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35696
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.776546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436098523145840:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3438
2025-11-19T14:00:32.779956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436101603562332:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35696
2025-11-19T14:00:32.780068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436101603562244:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:32.780131Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436101603562319:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.780286Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436101603562317:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.780526Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436099847781067:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.780875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436101848039942:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.780903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:32.780913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436101848039942:2049] [node 2] ICP22 created new session: [3:7574436101848040031:2049]
2025-11-19T14:00:32.780922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436101848040031:2049] [node 2] ICS09 handshake done sender: [3:7574436101848040011:2112] self: [3:7574436101848040022:0] peer: [2:7574436100009706712:0] socket: 391 qp: -1
2025-11-19T14:00:32.780931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436101848040031:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:32.780941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:32.781254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436101848040030:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61434
2025-11-19T14:00:32.781620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436099154018854:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15888) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436099154018931:2114] held: no
2025-11-19T14:00:32.781853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436099154018936:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29294
2025-11-19T14:00:32.781911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436099154018937:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29294
2025-11-19T14:00:32.782407Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436100009706727:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.782420Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436100009706727:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.783393Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436099847781091:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.783412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436099847781059:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.783428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:32.783434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436099847781059:2108] [node 5] ICP22 created new session: [1:7574436099847781092:2108]
2025-11-19T14:00:32.783439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436099847781092:2108] [node 5] ICS09 handshake done sender: [1:7574436099847781067:2113] self: [1:7574436099847781066:0] peer: [5:7574436098523145829:0] socket: 383 qp: -1
2025-11-19T14:00:32.783445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436099847781092:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:32.783453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.784273Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436100009706727:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:32.784270Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436101603562332:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.784392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436101603562243:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.784403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:32.784410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436101603562243:2048] [node 1] ICP22 created new session: [4:7574436101603562336:2048]
2025-11-19T14:00:32.784415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436101603562336:2048] [node 1] ICS09 handshake done sender: [4:7574436101603562317:2113] self: [4:7574436101603562330:0] peer: [1:7574436099847781064:0] socket: 386 qp: -1
2025-11-19T14:00:32.784420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436101603562336:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:32.784427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:32.785249Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436100009706742:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.785259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436100009706721:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.785269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:32.785274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436100009706721:2109] [node 4] ICP22 created new session: [2:7574436100009706749:2109]
2025-11-19T14:00:32.785279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436100009706749:2109] [node 4] ICS09 handshake done sender: [2:7574436100009706727:2114] self: [2:7574436100009706726:0] peer: [4:7574436101603562333:0] socket: 380 qp: -1
2025-11-19T14:00:32.785284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436100009706749:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:32.785289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:32.784528Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436098523145808:2107] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.786544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436101603562335:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35696
2025-11-19T14:00:32.786686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436099154018938:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29294
2025-11-19T14:00:32.786753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436099154018939:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29294
2025-11-19T14:00:32.786928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436098523145744:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15888) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436098523145808:2107] held: no
2025-11-19T14:00:32.787029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436098523145744:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.787039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:32.787045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436098523145744:2048] [node 1] ICP22 created new session: [5:7574436098523145841:2048]
2025-11-19T14:00:32.787050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436098523145841:2048] [node 1] ICS09 handshake done sender: [5:7574436098523145808:2107] self: [5:7574436098523145829:0] peer: [1:7574436099847781066:0] socket: 387 qp: -1
2025-11-19T14:00:32.787057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436098523145841:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:32.787066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.788426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436099154018940:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29294
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.789174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436100009706748:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51745
2025-11-19T14:00:32.792083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436100009706721:2109] [node 4] ICP18 handshake (actor [2:7574436100009706742:2118]) is too old
2025-11-19T14:00:32.793333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436101603562245:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:32.793729Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436101603562326:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:32.794310Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436099154018936:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.794337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436099154018855:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8650) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436099154018932:2115] held: no
2025-11-19T14:00:32.794450Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436099154018938:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.794473Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436099154018931:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:32.794746Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436099154018932:2115] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:32.794876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436099154018854:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.794892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:32.794898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436099154018854:2048] [node 1] ICP22 created new session: [6:7574436099154018941:2048]
2025-11-19T14:00:32.794903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436099154018941:2048] [node 1] ICS09 handshake done sender: [6:7574436099154018931:2114] self: [6:7574436099154018935:0] peer: [1:7574436099847781069:0] socket: 401 qp: -1
2025-11-19T14:00:32.794910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436099154018941:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:32.794918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:32.795127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436099154018855:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.795134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:32.795138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436099154018855:2049] [node 2] ICP22 created new session: [6:7574436099154018943:2049]
2025-11-19T14:00:32.795143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436099154018943:2049] [node 2] ICS09 handshake done sender: [6:7574436099154018932:2115] self: [6:7574436099154018934:0] peer: [2:7574436100009706722:0] socket: 402 qp: -1
2025-11-19T14:00:32.795147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436099154018943:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:32.795152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:32.795421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436099154018921:2070] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:32.795433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436099154018856:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.800102Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436101603562244:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8650) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436101603562326:2118] held: no
2025-11-19T14:00:32.800213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436101603562244:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.800228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:32.800234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436101603562244:2049] [node 2] ICP22 created new session: [4:7574436101603562340:2049]
2025-11-19T14:00:32.800240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436101603562340:2049] [node 2] ICS09 handshake done sender: [4:7574436101603562326:2118] self: [4:7574436101603562333:0] peer: [2:7574436100009706726:0] socket: 413 qp: -1
2025-11-19T14:00:32.800247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436101603562340:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:32.800260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:32.800136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436098523145843:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3438
2025-11-19T14:00:32.800520Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436099847781077:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.800534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436099847781077:2117] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.801657Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436099847781077:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:32.798636Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436101848040030:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.798685Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436101848040021:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.798689Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436101848040021:2116] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.799119Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436101848040018:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.799122Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436101848040018:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.799716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436101848039941:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:32.799741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436101848040033:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61434
2025-11-19T14:00:32.801814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436099847780997:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.801832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:32.801837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436099847780997:2049] [node 3] ICP22 created new session: [1:7574436099847781096:2049]
2025-11-19T14:00:32.801843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436099847781096:2049] [node 3] ICS09 handshake done sender: [1:7574436099847781077:2117] self: [1:7574436099847781075:0] peer: [3:7574436101848040036:0] socket: 424 qp: -1
2025-11-19T14:00:32.801848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436099847781096:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:32.801855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:32.802309Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436100009706748:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.802348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436100009706651:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:32.802424Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436101848040021:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.802561Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436098523145825:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.802579Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436098523145825:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.803814Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436098523145825:2118] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:32.803888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436098523145798:2095] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.803901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:32.803909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436098523145798:2095] [node 6] ICP22 created new session: [5:7574436098523145845:2095]
2025-11-19T14:00:32.803915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436098523145845:2095] [node 6] ICS09 handshake done sender: [5:7574436098523145825:2118] self: [5:7574436098523145824:0] peer: [6:7574436099154018947:0] socket: 388 qp: -1
2025-11-19T14:00:32.803921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436098523145845:2095] [node 6] ICS10 traffic start
2025-11-19T14:00:32.803929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:32.804105Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436101848040018:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:32.807789Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436099847781074:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.807812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436099847781074:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.809601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436099154018949:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29294
2025-11-19T14:00:32.812316Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436098523145843:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.812609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7574436098523145798:2095] [node 6] ICP18 handshake (actor [5:7574436098523145843:2123]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.812915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436101848040005:2104] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.812944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:32.812951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436101848040005:2104] [node 4] ICP22 created new session: [3:7574436101848040038:2104]
2025-11-19T14:00:32.812959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436101848040038:2104] [node 4] ICS09 handshake done sender: [3:7574436101848040021:2116] self: [3:7574436101848040007:0] peer: [4:7574436101603562339:0] socket: 419 qp: -1
2025-11-19T14:00:32.812967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436101848040038:2104] [node 4] ICS10 traffic start
2025-11-19T14:00:32.812977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:32.813226Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436101848040033:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.813235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436101848040014:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.813242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:32.813246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436101848040014:2109] [node 6] ICP22 created new session: [3:7574436101848040040:2109]
2025-11-19T14:00:32.813250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436101848040040:2109] [node 6] ICS09 handshake done sender: [3:7574436101848040018:2115] self: [3:7574436101848040017:0] peer: [6:7574436099154018948:0] socket: 418 qp: -1
2025-11-19T14:00:32.813254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436101848040040:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:32.813260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:32.813450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436101848039941:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15888) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436101848040030:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.815883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436101848040037:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61434
2025-11-19T14:00:32.816269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574436101848040014:2109] [node 6] ICP18 handshake (actor [3:7574436101848040033:2119]) is too old
2025-11-19T14:00:32.813742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436101603562342:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35696
2025-11-19T14:00:32.814080Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436101603562332:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:32.814198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436101603562245:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.814211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:32.814226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436101603562245:2050] [node 3] ICP22 created new session: [4:7574436101603562343:2050]
2025-11-19T14:00:32.814231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436101603562343:2050] [node 3] ICS09 handshake done sender: [4:7574436101603562332:2120] self: [4:7574436101603562339:0] peer: [3:7574436101848040007:0] socket: 426 qp: -1
2025-11-19T14:00:32.814236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436101603562343:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:32.814244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:32.816027Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436099154018940:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.816059Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436099154018938:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:32.816222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436099154018856:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.816234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:32.816239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436099154018856:2050] [node 3] ICP22 created new session: [6:7574436099154018950:2050]
2025-11-19T14:00:32.816244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436099154018950:2050] [node 3] ICS09 handshake done sender: [6:7574436099154018938:2119] self: [6:7574436099154018948:0] peer: [3:7574436101848040017:0] socket: 430 qp: -1
2025-11-19T14:00:32.816250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436099154018950:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:32.816257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:32.816588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436099154018920:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.817304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436100009706752:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51745
2025-11-19T14:00:32.817270Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436101848040030:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:32.817609Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436099847781074:2115] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:32.817436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436101848039941:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.817452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:32.817458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436101848039941:2048] [node 1] ICP22 created new session: [3:7574436101848040042:2048]
2025-11-19T14:00:32.817463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436101848040042:2048] [node 1] ICS09 handshake done sender: [3:7574436101848040030:2118] self: [3:7574436101848040036:0] peer: [1:7574436099847781075:0] socket: 427 qp: -1
2025-11-19T14:00:32.817469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436101848040042:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:32.817476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:32.817802Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436100009706748:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:32.817910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436100009706651:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.817934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:32.817940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436100009706651:2048] [node 1] ICP22 created new session: [2:7574436100009706753:2048]
2025-11-19T14:00:32.817948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436100009706753:2048] [node 1] ICS09 handshake done sender: [2:7574436100009706748:2119] self: [2:7574436100009706751:0] peer: [1:7574436099847781060:0] socket: 428 qp: -1
2025-11-19T14:00:32.817953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436100009706753:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:32.817962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:32.818105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436099847780996:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.818127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:32.818132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436099847780996:2048] [node 2] ICP22 created new session: [1:7574436099847781099:2048]
2025-11-19T14:00:32.818137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436099847781099:2048] [node 2] ICS09 handshake done sender: [1:7574436099847781074:2115] self: [1:7574436099847781060:0] peer: [2:7574436100009706751:0] socket: 423 qp: -1
2025-11-19T14:00:32.818142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436099847781099:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:32.818148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.824253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436099154018921:2070] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28173) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436099154018936:2117] held: no
2025-11-19T14:00:32.824538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436099154018953:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29294
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:32.824760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436099154018954:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29294
2025-11-19T14:00:32.824774Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436099154018936:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:32.825005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436099154018921:2070] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.825021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:32.825027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436099154018921:2070] [node 5] ICP22 created new session: [6:7574436099154018955:2070]
2025-11-19T14:00:32.825034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436099154018955:2070] [node 5] ICS09 handshake done sender: [6:7574436099154018936:2117] self: [6:7574436099154018947:0] peer: [5:7574436098523145824:0] socket: 389 qp: -1
2025-11-19T14:00:32.825041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436099154018955:2070] [node 5] ICS10 traffic start
2025-11-19T14:00:32.825048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:32.823135Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436101603562323:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:32.823158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436101603562323:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:32.828574Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436099154018940:2121] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:32.828256Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436101603562323:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:32.828742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436101603562316:2112] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:32.828765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:32.828773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436101603562316:2112] [node 6] ICP22 created new session: [4:7574436101603562346:2112]
2025-11-19T14:00:32.828779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436101603562346:2112] [node 6] ICS09 handshake done sender: [4:7574436101603562323:2117] self: [4:7574436101603562322:0] peer: [6:7574436099154018952:0] socket: 408 qp: -1
2025-11-19T14:00:32.828786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436101603562346:2112] [node 6] ICS10 traffic start
2025-11-19T14:00:32.828797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:32.829313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436099154018920:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:32.829328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:32.829334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436099154018920:2064] [node 4] ICP22 created new session: [6:7574436099154018957:2064]
2025-11-19T14:00:32.829339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436099154018957:2064] [node 4] ICS09 handshake done sender: [6:7574436099154018940:2121] self: [6:7574436099154018952:0] peer: [4:7574436101603562322:0] socket: 432 qp: -1
2025-11-19T14:00:32.829345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436099154018957:2064] [node 4] ICS10 traffic start
2025-11-19T14:00:32.829352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:32.841458Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:32.841580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436100009706753:2048] [node 1] ICS01 socket: 428 reason# EndOfStream
2025-11-19T14:00:32.841591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436100009706651:2048] [node 1] ICP30 unregister session Session# [2:7574436100009706753:2048] VirtualId# [2:7574436100009706751:0]
2025-11-19T14:00:32.841599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436100009706753:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.841654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:32.841682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:32.844178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:32.844267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436101848040042:2048] [node 1] ICS01 socket: 427 reason# EndOfStream
2025-11-19T14:00:32.844275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436101848039941:2048] [node 1] ICP30 unregister session Session# [3:7574436101848040042:2048] VirtualId# [3:7574436101848040036:0]
2025-11-19T14:00:32.844281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436101848040042:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.844313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:32.844331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:32.854554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:32.854655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436098523145841:2048] [node 1] ICS01 socket: 387 reason# EndOfStream
2025-11-19T14:00:32.854664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436098523145744:2048] [node 1] ICP30 unregister session Session# [5:7574436098523145841:2048] VirtualId# [5:7574436098523145829:0]
2025-11-19T14:00:32.854671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436098523145841:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.854730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:32.854756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:32.854796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:32.854859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436099154018941:2048] [node 1] ICS01 socket: 401 reason# EndOfStream
2025-11-19T14:00:32.854863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436099154018854:2048] [node 1] ICP30 unregister session Session# [6:7574436099154018941:2048] VirtualId# [6:7574436099154018935:0]
2025-11-19T14:00:32.854868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436099154018941:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.854892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:32.854912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:32.856088Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:32.856159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436101603562336:2048] [node 1] ICS01 socket: 386 reason# EndOfStream
2025-11-19T14:00:32.856165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436101603562243:2048] [node 1] ICP30 unregister session Session# [4:7574436101603562336:2048] VirtualId# [4:7574436101603562330:0]
2025-11-19T14:00:32.856171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436101603562336:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.856215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:32.856236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:32.857477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:32.857553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436101848040031:2049] [node 2] ICS01 socket: 391 reason# EndOfStream
2025-11-19T14:00:32.857558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436101848039942:2049] [node 2] ICP30 unregister session Session# [3:7574436101848040031:2049] VirtualId# [3:7574436101848040022:0]
2025-11-19T14:00:32.857563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436101848040031:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.857591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:32.857607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:32.867520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:32.867618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436099154018943:2049] [node 2] ICS01 socket: 402 reason# EndOfStream
2025-11-19T14:00:32.867626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436099154018855:2049] [node 2] ICP30 unregister session Session# [6:7574436099154018943:2049] VirtualId# [6:7574436099154018934:0]
2025-11-19T14:00:32.867632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436099154018943:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.867700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:32.867723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:32.867776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:32.867894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436101603562340:2049] [node 2] ICS01 socket: 413 reason# EndOfStream
2025-11-19T14:00:32.867900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436101603562244:2049] [node 2] ICP30 unregister session Session# [4:7574436101603562340:2049] VirtualId# [4:7574436101603562333:0]
2025-11-19T14:00:32.867905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436101603562340:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.867936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:32.867957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:32.868248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:32.868311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436098523145835:2049] [node 2] ICS01 socket: 404 reason# EndOfStream
2025-11-19T14:00:32.868316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436098523145745:2049] [node 2] ICP30 unregister session Session# [5:7574436098523145835:2049] VirtualId# [5:7574436098523145828:0]
2025-11-19T14:00:32.868319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436098523145835:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.868351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:32.868372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:32.880942Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:32.881039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436098523145837:2050] [node 3] ICS01 socket: 403 reason# EndOfStream
2025-11-19T14:00:32.881048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436098523145746:2050] [node 3] ICP30 unregister session Session# [5:7574436098523145837:2050] VirtualId# [5:7574436098523145826:0]
2025-11-19T14:00:32.881057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436098523145837:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.881118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:32.881143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:32.881183Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:32.881240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436099154018950:2050] [node 3] ICS01 socket: 430 reason# EndOfStream
2025-11-19T14:00:32.881245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436099154018856:2050] [node 3] ICP30 unregister session Session# [6:7574436099154018950:2050] VirtualId# [6:7574436099154018948:0]
2025-11-19T14:00:32.881249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436099154018950:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.881274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:32.881296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:32.894446Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:32.894535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436099154018957:2064] [node 4] ICS01 socket: 432 reason# EndOfStream
2025-11-19T14:00:32.894544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436099154018920:2064] [node 4] ICP30 unregister session Session# [6:7574436099154018957:2064] VirtualId# [6:7574436099154018952:0]
2025-11-19T14:00:32.894551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436099154018957:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:32.894613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:32.894635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:33.096048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436103358404684:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:33.096067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436103358404685:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:33.128205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436104137559036:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:33.128226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436104137559037:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:33.162386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436103899498606:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:33.162405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436103899498607:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:33.216045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436103087368039:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:33.216065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436103087368040:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:33.216071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436103087368041:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:33.271939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436104258168652:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:33.271957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436104258168653:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:33.271962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436104258168654:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:33.337463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436103412434105:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:33.337484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436103412434106:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:33.337489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436103412434107:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:33.392395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436103358404745:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:33.392163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436103087368098:2103] [node 6] ICP01 ready to work
2025-11-19T14:00:33.392610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436103412434123:2064] [node 5] ICP01 ready to work
2025-11-19T14:00:33.400389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436103899498668:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:33.400495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436103899498669:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:33.405411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436103087368107:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:33.408800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436103358404684:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(16433572ns) {Configure:104(16432665ns) {ProcessConfigured:135(15000381ns) {StartInitialHandshake:157(15000116ns) {DropHandshakes:469(912ns) {DropIncomingHandshake:441(147ns) DropOutgoingHandshake:456(84ns)} GenerateSessionVirtualId:988(259ns) PrepareNewSessionHandshake:476(975ns) {SwitchToState:178(487ns)}}}}}
2025-11-19T14:00:33.409034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436103358404752:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.411128Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436103412434107:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18969857ns) {Configure:104(18968811ns) {ProcessConfigured:135(17004948ns) {StartInitialHandshake:157(17004615ns) {DropHandshakes:469(488ns) {DropIncomingHandshake:441(79ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(272ns) PrepareNewSessionHandshake:476(739ns) {SwitchToState:178(22ns)}}}}}
2025-11-19T14:00:33.411357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436103412434154:2075] [node 4] ICP01 ready to work
2025-11-19T14:00:33.412030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436103412434152:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14582
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.412155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436103412434123:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(19495320ns) {Configure:104(19494934ns) {ProcessConfigured:135(17906478ns) {StartInitialHandshake:157(17906380ns) {DropHandshakes:469(215ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(24ns)} GenerateSessionVirtualId:988(102ns) PrepareNewSessionHandshake:476(538ns) {SwitchToState:178(247ns)}}}}}
2025-11-19T14:00:33.410981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436103358404745:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(18532002ns) {Configure:104(18531362ns) {ProcessConfigured:135(17115676ns) {StartInitialHandshake:157(17115054ns) {DropHandshakes:469(323ns) {DropIncomingHandshake:441(20ns) DropOutgoingHandshake:456(17ns)} GenerateSessionVirtualId:988(94ns) PrepareNewSessionHandshake:476(1072ns) {SwitchToState:178(725ns)}}}}}
2025-11-19T14:00:33.412986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436103358404747:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:33.415388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436103412434153:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14582
2025-11-19T14:00:33.415872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436103358404761:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28656
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.420985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436103087368039:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(20631724ns) {Configure:104(20630280ns) {ProcessConfigured:135(20250426ns) {StartInitialHandshake:157(20249494ns) {DropHandshakes:469(1106ns) {DropIncomingHandshake:441(189ns) DropOutgoingHandshake:456(322ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:476(1252ns) {SwitchToState:178(905ns)}}}}}
2025-11-19T14:00:33.422217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436104258168686:2066] [node 6] ICP01 ready to work
2025-11-19T14:00:33.422710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436104258168687:2069] [node 4] ICP01 ready to work
2025-11-19T14:00:33.424023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436103087368117:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13332
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.426339Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436103899498669:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(22486125ns) {Configure:104(10488680ns) {ProcessConfigured:135(10163773ns) {StartInitialHandshake:157(10163470ns) {DropHandshakes:469(690ns) {DropIncomingHandshake:441(72ns) DropOutgoingHandshake:456(91ns)} GenerateSessionVirtualId:988(345ns) PrepareNewSessionHandshake:476(802ns) {SwitchToState:178(350ns)}}}}}
2025-11-19T14:00:33.427472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436103899498675:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34132
2025-11-19T14:00:33.427490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436103899498676:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34132
2025-11-19T14:00:33.427560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436103899498677:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34132
2025-11-19T14:00:33.427577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436103899498678:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34132
2025-11-19T14:00:33.429412Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436103358404761:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.431952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436104137559100:2107] [node 6] ICP01 ready to work
2025-11-19T14:00:33.432020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436104137559101:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:33.432223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436104137559102:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:33.431784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436103899498673:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:33.431920Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436103899498676:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.435265Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436103412434105:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(43076887ns) {Configure:104(43076175ns) {ProcessConfigured:135(40944543ns) {StartInitialHandshake:157(40943945ns) {DropHandshakes:469(238ns) {DropIncomingHandshake:441(16ns) DropOutgoingHandshake:456(19ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:476(1563ns) {SwitchToState:178(897ns)}}}}}
2025-11-19T14:00:33.435628Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436103899498668:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(35089691ns) {Configure:104(35088732ns) {ProcessConfigured:135(34701398ns) {StartInitialHandshake:157(34701258ns) {DropHandshakes:469(613ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(56ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:476(771ns) {SwitchToState:178(579ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.445203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436103087368119:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13332
2025-11-19T14:00:33.447543Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436104137559037:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(42958464ns) {Configure:104(42957139ns) {ProcessConfigured:135(42502136ns) {StartInitialHandshake:157(42501151ns) {DropHandshakes:469(1244ns) {DropIncomingHandshake:441(86ns) DropOutgoingHandshake:456(20ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:476(2232ns) {SwitchToState:178(1135ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.455947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436103358404765:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28656
2025-11-19T14:00:33.456960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436104137559106:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36721
2025-11-19T14:00:33.456981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436104137559107:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36721
2025-11-19T14:00:33.457056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436104137559108:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36721
2025-11-19T14:00:33.457084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436104137559109:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36721
2025-11-19T14:00:33.457095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436104137559110:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36721
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.457533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436103899498673:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(25487809ns) {Configure:104(25487062ns) {ProcessConfigured:135(25173531ns) {StartInitialHandshake:157(25173101ns) {DropHandshakes:469(682ns) {DropIncomingHandshake:441(77ns) DropOutgoingHandshake:456(202ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:476(1401ns) {SwitchToState:178(928ns)}}}}}
2025-11-19T14:00:33.458483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436103899498690:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34132
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.459903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436103087368120:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13332
2025-11-19T14:00:33.457702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436104137559036:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(25308318ns) {Configure:104(25307829ns) {ProcessConfigured:135(24976123ns) {StartInitialHandshake:157(24975841ns) {DropHandshakes:469(700ns) {DropIncomingHandshake:441(207ns) DropOutgoingHandshake:456(91ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:476(676ns) {SwitchToState:178(164ns)}}}}}
2025-11-19T14:00:33.462157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436104258168730:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32101
2025-11-19T14:00:33.461564Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436104137559107:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.461612Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436104137559108:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.461671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436104137559036:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.464605Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436103899498677:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.464667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436103899498606:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.464872Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436103358404754:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.464889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436103358404754:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.467653Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436103358404756:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.467663Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436103358404756:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.468393Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436103358404756:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:33.467536Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436103412434153:2091] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.467574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436103412434105:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.472428Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436103358404751:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.472444Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436103358404751:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.472877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436103358404752:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.472895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:33.472903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436103358404752:2109] [node 6] ICP22 created new session: [1:7574436103358404767:2109]
2025-11-19T14:00:33.472911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436103358404767:2109] [node 6] ICS09 handshake done sender: [1:7574436103358404756:2112] self: [1:7574436103358404755:0] peer: [6:7574436103412434185:0] socket: 386 qp: -1
2025-11-19T14:00:33.472921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436103358404767:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:33.472944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:33.473594Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436103358404751:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:33.473719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436103358404684:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.473732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:33.473740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436103358404684:2048] [node 2] ICP22 created new session: [1:7574436103358404771:2048]
2025-11-19T14:00:33.473745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436103358404771:2048] [node 2] ICS09 handshake done sender: [1:7574436103358404751:2110] self: [1:7574436103358404748:0] peer: [2:7574436104137559120:0] socket: 384 qp: -1
2025-11-19T14:00:33.473750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436103358404771:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:33.473757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.475458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436104137559121:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36721
2025-11-19T14:00:33.475337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436104258168731:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32101
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.475806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436103087368121:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13332
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.479922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436103412434186:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14582
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.480227Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436103358404754:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:33.473363Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436103899498678:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.480083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436103899498693:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34132
2025-11-19T14:00:33.475659Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436104137559108:2113] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:33.475736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436104137559036:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.475769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:33.475776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436104137559036:2048] [node 1] ICP22 created new session: [2:7574436104137559122:2048]
2025-11-19T14:00:33.475782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436104137559122:2048] [node 1] ICS09 handshake done sender: [2:7574436104137559108:2113] self: [2:7574436104137559120:0] peer: [1:7574436103358404748:0] socket: 405 qp: -1
2025-11-19T14:00:33.475791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436104137559122:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:33.475811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:33.476430Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436104137559109:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.480596Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436104137559100:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(32905592ns) {Configure:104(32880620ns) {ProcessConfigured:135(32486118ns) {StartInitialHandshake:157(32484877ns) {DropHandshakes:469(1620ns) {DropIncomingHandshake:441(230ns) DropOutgoingHandshake:456(180ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:476(2221ns) {SwitchToState:178(1123ns)}}}}}
2025-11-19T14:00:33.492077Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436103087368121:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.492188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436103087368039:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.492885Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436104258168730:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.492927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436104258168653:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.493169Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436103412434153:2091] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:33.493242Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436103899498690:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.493259Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436103899498677:2113] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:33.493367Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436103087368117:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.493419Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436103087368120:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.493435Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436103087368119:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.493463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436103087368040:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.493471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436103087368041:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.493583Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436104137559106:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.500137Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436103358404684:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(26292133ns) {HandleHandshakeStatus:329(26291182ns) {DropIncomingHandshake:441(23ns) DropOutgoingHandshake:456(201ns) SwitchToState:178(217ns) ProcessPendingSessionEvents:495(26219921ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26174894ns max# 26064990ns}}}
2025-11-19T14:00:33.500234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436103358404685:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.500251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:33.500259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436103358404685:2049] [node 3] ICP22 created new session: [1:7574436103358404775:2049]
2025-11-19T14:00:33.500266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436103358404775:2049] [node 3] ICS09 handshake done sender: [1:7574436103358404754:2111] self: [1:7574436103358404753:0] peer: [3:7574436103899498691:0] socket: 385 qp: -1
2025-11-19T14:00:33.500275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436103358404775:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:33.500287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:33.500534Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436103358404760:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.500538Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436103358404760:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.502115Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436103358404760:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:33.502220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436103358404745:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.502237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:33.502244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436103358404745:2107] [node 4] ICP22 created new session: [1:7574436103358404778:2107]
2025-11-19T14:00:33.502251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436103358404778:2107] [node 4] ICS09 handshake done sender: [1:7574436103358404760:2113] self: [1:7574436103358404749:0] peer: [4:7574436103087368122:0] socket: 417 qp: -1
2025-11-19T14:00:33.502258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436103358404778:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:33.502267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:33.502352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436103899498606:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.502373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:33.502378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436103899498606:2048] [node 1] ICP22 created new session: [3:7574436103899498694:2048]
2025-11-19T14:00:33.502386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436103899498694:2048] [node 1] ICS09 handshake done sender: [3:7574436103899498677:2113] self: [3:7574436103899498691:0] peer: [1:7574436103358404753:0] socket: 396 qp: -1
2025-11-19T14:00:33.502391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436103899498694:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:33.502411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:33.503285Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436104137559112:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.503301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436104137559112:2116] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.503404Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436104137559114:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.503409Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436104137559114:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.504740Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436103899498674:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.504752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436103899498674:2110] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.502712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436103412434105:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.502739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:33.502746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436103412434105:2048] [node 1] ICP22 created new session: [6:7574436103412434188:2048]
2025-11-19T14:00:33.502753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436103412434188:2048] [node 1] ICS09 handshake done sender: [6:7574436103412434153:2091] self: [6:7574436103412434185:0] peer: [1:7574436103358404755:0] socket: 388 qp: -1
2025-11-19T14:00:33.502760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436103412434188:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:33.502768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:33.503131Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436103412434152:2090] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.504402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436103412434187:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14582
2025-11-19T14:00:33.504461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436103412434154:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.504583Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436104258168731:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.504623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436104258168652:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.506166Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436103087368100:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.506178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436103087368100:2106] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.507010Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436103358404763:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.507022Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436103358404763:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.507092Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436103087368100:2106] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:33.507163Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436104137559114:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:33.507423Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436103358404765:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.507459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436103358404685:2049] [node 3] ICP18 handshake (actor [1:7574436103358404765:2116]) is too old
2025-11-19T14:00:33.507882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436103899498607:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.508755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436104137559102:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.508777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:33.508784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436104137559102:2109] [node 4] ICP22 created new session: [2:7574436104137559129:2109]
2025-11-19T14:00:33.508792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436104137559129:2109] [node 4] ICS09 handshake done sender: [2:7574436104137559114:2117] self: [2:7574436104137559113:0] peer: [4:7574436103087368124:0] socket: 410 qp: -1
2025-11-19T14:00:33.508799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436104137559129:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:33.508808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:33.508966Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436103899498674:2110] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:33.507422Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436103412434187:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.507474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436103412434106:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.509919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436103899498669:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.509937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:33.509946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436103899498669:2108] [node 4] ICP22 created new session: [3:7574436103899498699:2108]
2025-11-19T14:00:33.509952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436103899498699:2108] [node 4] ICS09 handshake done sender: [3:7574436103899498674:2110] self: [3:7574436103899498672:0] peer: [4:7574436103087368125:0] socket: 398 qp: -1
2025-11-19T14:00:33.509958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436103899498699:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:33.509966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:33.510253Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436103899498675:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.515393Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436104137559124:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.515407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436104137559124:2120] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.516218Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436104137559110:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.516255Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436104137559104:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.516259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436104137559104:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.517108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436103087368127:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13332
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.517914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436104258168735:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32101
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.517922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574436103899498669:2108] [node 4] ICP18 handshake (actor [3:7574436103899498675:2111]) is too old
2025-11-19T14:00:33.518237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436103087368098:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.518238Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436103087368121:2119] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:33.518262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:33.518268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436103087368098:2103] [node 6] ICP22 created new session: [4:7574436103087368131:2103]
2025-11-19T14:00:33.518276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436103087368131:2103] [node 6] ICS09 handshake done sender: [4:7574436103087368100:2106] self: [4:7574436103087368099:0] peer: [6:7574436103412434190:0] socket: 378 qp: -1
2025-11-19T14:00:33.518283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436103087368131:2103] [node 6] ICS10 traffic start
2025-11-19T14:00:33.518308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:33.518338Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436104258168735:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.518367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436104258168654:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.518386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436104258168734:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32101
2025-11-19T14:00:33.518502Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436103358404763:2115] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:33.518546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436103087368128:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13332
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.520184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436103358404747:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.520208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:33.520214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436103358404747:2108] [node 5] ICP22 created new session: [1:7574436103358404781:2108]
2025-11-19T14:00:33.520219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436103358404781:2108] [node 5] ICS09 handshake done sender: [1:7574436103358404763:2115] self: [1:7574436103358404762:0] peer: [5:7574436104258168733:0] socket: 419 qp: -1
2025-11-19T14:00:33.520225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436103358404781:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:33.520234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:33.520712Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436104137559112:2116] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:33.519447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436103412434193:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14582
2025-11-19T14:00:33.519492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436103412434194:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14582
2025-11-19T14:00:33.519503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436103412434195:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14582
2025-11-19T14:00:33.521809Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436104137559124:2120] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:33.521935Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436104137559104:2110] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:33.522087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436104137559101:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.522106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:33.522113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436104137559101:2108] [node 5] ICP22 created new session: [2:7574436104137559135:2108]
2025-11-19T14:00:33.522118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436104137559135:2108] [node 5] ICS09 handshake done sender: [2:7574436104137559112:2116] self: [2:7574436104137559111:0] peer: [5:7574436104258168732:0] socket: 409 qp: -1
2025-11-19T14:00:33.522123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436104137559135:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:33.522130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:33.522331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436104137559100:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.522339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:33.522342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436104137559100:2107] [node 6] ICP22 created new session: [2:7574436104137559137:2107]
2025-11-19T14:00:33.522346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436104137559137:2107] [node 6] ICS09 handshake done sender: [2:7574436104137559124:2120] self: [2:7574436104137559105:0] peer: [6:7574436103412434192:0] socket: 424 qp: -1
2025-11-19T14:00:33.522349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436104137559137:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:33.522354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:33.522522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436104137559037:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.522530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:33.522534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436104137559037:2049] [node 3] ICP22 created new session: [2:7574436104137559139:2049]
2025-11-19T14:00:33.522537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436104137559139:2049] [node 3] ICS09 handshake done sender: [2:7574436104137559104:2110] self: [2:7574436104137559097:0] peer: [3:7574436103899498697:0] socket: 408 qp: -1
2025-11-19T14:00:33.522540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436104137559139:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:33.522545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:33.523346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436103087368039:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.523365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:33.523370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436103087368039:2048] [node 1] ICP22 created new session: [4:7574436103087368133:2048]
2025-11-19T14:00:33.523375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436103087368133:2048] [node 1] ICS09 handshake done sender: [4:7574436103087368121:2119] self: [4:7574436103087368122:0] peer: [1:7574436103358404749:0] socket: 422 qp: -1
2025-11-19T14:00:33.523379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436103087368133:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:33.523384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:33.523430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436103087368130:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13332
2025-11-19T14:00:33.523621Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436103087368041:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21637) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436103087368119:2117] held: no
2025-11-19T14:00:33.518625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436103899498702:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34132
2025-11-19T14:00:33.518657Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436103899498687:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.518660Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436103899498687:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.519812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436104258168737:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32101
2025-11-19T14:00:33.519856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436104258168738:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32101
2025-11-19T14:00:33.519867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436104258168739:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32101
2025-11-19T14:00:33.519878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436104258168740:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32101
2025-11-19T14:00:33.525264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436104258168652:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61551) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436104258168731:2116] held: no
2025-11-19T14:00:33.525370Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436104258168730:2115] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:33.525574Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436103087368120:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:33.525854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436104258168653:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.525869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:33.525875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436104258168653:2049] [node 2] ICP22 created new session: [5:7574436104258168741:2049]
2025-11-19T14:00:33.525880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436104258168741:2049] [node 2] ICS09 handshake done sender: [5:7574436104258168730:2115] self: [5:7574436104258168732:0] peer: [2:7574436104137559111:0] socket: 412 qp: -1
2025-11-19T14:00:33.525885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436104258168741:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:33.525890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:33.526093Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436103087368119:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:33.526285Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436103899498687:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:33.526945Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436103412434154:2075] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5172) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436103412434152:2090] held: no
2025-11-19T14:00:33.527100Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436103412434187:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:33.527159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436103899498668:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.527177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:33.527182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436103899498668:2107] [node 5] ICP22 created new session: [3:7574436103899498704:2107]
2025-11-19T14:00:33.527188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436103899498704:2107] [node 5] ICS09 handshake done sender: [3:7574436103899498687:2117] self: [3:7574436103899498670:0] peer: [5:7574436104258168736:0] socket: 433 qp: -1
2025-11-19T14:00:33.527194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436103899498704:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:33.527201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:33.526906Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436104258168731:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:33.528182Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436104258168734:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.528194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436104258168652:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.528203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:33.528208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436104258168652:2048] [node 1] ICP22 created new session: [5:7574436104258168743:2048]
2025-11-19T14:00:33.528212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436104258168743:2048] [node 1] ICS09 handshake done sender: [5:7574436104258168731:2116] self: [5:7574436104258168733:0] peer: [1:7574436103358404762:0] socket: 420 qp: -1
2025-11-19T14:00:33.528217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436104258168743:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:33.528238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:33.528483Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436104258168735:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:33.528857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436104258168654:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.528870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:33.528874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436104258168654:2050] [node 3] ICP22 created new session: [5:7574436104258168748:2050]
2025-11-19T14:00:33.528879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436104258168748:2050] [node 3] ICS09 handshake done sender: [5:7574436104258168735:2118] self: [5:7574436104258168736:0] peer: [3:7574436103899498670:0] socket: 440 qp: -1
2025-11-19T14:00:33.528883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436104258168748:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:33.528887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:33.532101Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436103412434152:2090] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:33.532891Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436103899498690:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:33.533487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436103087368040:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.533507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:33.533514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436103087368040:2049] [node 2] ICP22 created new session: [4:7574436103087368135:2049]
2025-11-19T14:00:33.533519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436103087368135:2049] [node 2] ICS09 handshake done sender: [4:7574436103087368120:2118] self: [4:7574436103087368124:0] peer: [2:7574436104137559113:0] socket: 413 qp: -1
2025-11-19T14:00:33.533524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436103087368135:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:33.533531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:33.533802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436103087368041:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.533808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:33.533812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436103087368041:2050] [node 3] ICP22 created new session: [4:7574436103087368137:2050]
2025-11-19T14:00:33.533816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436103087368137:2050] [node 3] ICS09 handshake done sender: [4:7574436103087368119:2117] self: [4:7574436103087368125:0] peer: [3:7574436103899498672:0] socket: 401 qp: -1
2025-11-19T14:00:33.533821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436103087368137:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:33.533825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:33.535893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436103412434106:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.535913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:33.535921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436103412434106:2049] [node 2] ICP22 created new session: [6:7574436103412434197:2049]
2025-11-19T14:00:33.535926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436103412434197:2049] [node 2] ICS09 handshake done sender: [6:7574436103412434187:2118] self: [6:7574436103412434192:0] peer: [2:7574436104137559105:0] socket: 425 qp: -1
2025-11-19T14:00:33.535932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436103412434197:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:33.535961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:33.536268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436103412434154:2075] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.536275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:33.536278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436103412434154:2075] [node 4] ICP22 created new session: [6:7574436103412434199:2075]
2025-11-19T14:00:33.536282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436103412434199:2075] [node 4] ICS09 handshake done sender: [6:7574436103412434152:2090] self: [6:7574436103412434190:0] peer: [4:7574436103087368099:0] socket: 387 qp: -1
2025-11-19T14:00:33.536286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436103412434199:2075] [node 4] ICS10 traffic start
2025-11-19T14:00:33.536291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.538041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574436103087368040:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:33.538284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436103087368139:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13332
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.538948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574436103412434106:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:33.539021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436103412434196:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14582
2025-11-19T14:00:33.540018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436103899498607:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.540043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:33.540050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436103899498607:2049] [node 2] ICP22 created new session: [3:7574436103899498706:2049]
2025-11-19T14:00:33.540057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436103899498706:2049] [node 2] ICS09 handshake done sender: [3:7574436103899498690:2119] self: [3:7574436103899498697:0] peer: [2:7574436104137559097:0] socket: 411 qp: -1
2025-11-19T14:00:33.540064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436103899498706:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:33.540073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:33.540234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574436103899498607:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:33.546382Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436103087368139:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.547006Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436103412434196:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.547079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436103412434123:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.547421Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436104258168739:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.547467Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436104258168728:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.547471Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436104258168728:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.547949Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436103087368113:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.547956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436103087368113:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.547833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436104258168687:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.548056Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436103412434194:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.548097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436103412434107:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:33.548633Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436104258168728:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:33.548709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436104258168686:2066] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.548723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:33.548731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436104258168686:2066] [node 6] ICP22 created new session: [5:7574436104258168752:2066]
2025-11-19T14:00:33.548737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436104258168752:2066] [node 6] ICS09 handshake done sender: [5:7574436104258168728:2113] self: [5:7574436104258168725:0] peer: [6:7574436103412434201:0] socket: 380 qp: -1
2025-11-19T14:00:33.548742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436104258168752:2066] [node 6] ICS10 traffic start
2025-11-19T14:00:33.548751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:33.548856Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436103899498689:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:33.548870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436103899498689:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:33.550183Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436103087368113:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:33.550269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436103087368107:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.550284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:33.550290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436103087368107:2108] [node 5] ICP22 created new session: [4:7574436103087368142:2108]
2025-11-19T14:00:33.550295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436103087368142:2108] [node 5] ICS09 handshake done sender: [4:7574436103087368113:2113] self: [4:7574436103087368112:0] peer: [5:7574436104258168750:0] socket: 443 qp: -1
2025-11-19T14:00:33.550301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436103087368142:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:33.550308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:33.551960Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436103899498689:2118] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:33.552113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436103412434123:2064] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25981) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436103412434196:2122] held: no
2025-11-19T14:00:33.552337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436103899498673:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:33.552353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:33.552361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436103899498673:2109] [node 6] ICP22 created new session: [3:7574436103899498709:2109]
2025-11-19T14:00:33.552368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436103899498709:2109] [node 6] ICS09 handshake done sender: [3:7574436103899498689:2118] self: [3:7574436103899498686:0] peer: [6:7574436103412434202:0] socket: 399 qp: -1
2025-11-19T14:00:33.552373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436103899498709:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:33.552382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.556057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436104258168754:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32101
2025-11-19T14:00:33.556425Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436104258168687:2069] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5172) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436104258168739:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.576932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436103412434107:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21637) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436103412434194:2120] held: no
2025-11-19T14:00:33.577101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436103412434203:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14582
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:33.577413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436103412434204:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14582
2025-11-19T14:00:33.577631Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436103412434194:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:33.578188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436103412434107:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.578201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:33.578206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436103412434107:2050] [node 3] ICP22 created new session: [6:7574436103412434205:2050]
2025-11-19T14:00:33.578211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436103412434205:2050] [node 3] ICS09 handshake done sender: [6:7574436103412434194:2120] self: [6:7574436103412434202:0] peer: [3:7574436103899498686:0] socket: 444 qp: -1
2025-11-19T14:00:33.578215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436103412434205:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:33.578222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:33.578603Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436103412434196:2122] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:33.578667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436103412434123:2064] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.578679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:33.578683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436103412434123:2064] [node 5] ICP22 created new session: [6:7574436103412434207:2064]
2025-11-19T14:00:33.578689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436103412434207:2064] [node 5] ICS09 handshake done sender: [6:7574436103412434196:2122] self: [6:7574436103412434201:0] peer: [5:7574436104258168725:0] socket: 382 qp: -1
2025-11-19T14:00:33.578694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436103412434207:2064] [node 5] ICS10 traffic start
2025-11-19T14:00:33.578699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:33.577267Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436104258168739:2121] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:33.577705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436104258168687:2069] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:33.577719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:33.577727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436104258168687:2069] [node 4] ICP22 created new session: [5:7574436104258168755:2069]
2025-11-19T14:00:33.577734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436104258168755:2069] [node 4] ICS09 handshake done sender: [5:7574436104258168739:2121] self: [5:7574436104258168750:0] peer: [4:7574436103087368112:0] socket: 447 qp: -1
2025-11-19T14:00:33.577744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436104258168755:2069] [node 4] ICS10 traffic start
2025-11-19T14:00:33.577757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:33.594627Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:33.594738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436104137559122:2048] [node 1] ICS01 socket: 405 reason# EndOfStream
2025-11-19T14:00:33.594751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436104137559036:2048] [node 1] ICP30 unregister session Session# [2:7574436104137559122:2048] VirtualId# [2:7574436104137559120:0]
2025-11-19T14:00:33.594759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436104137559122:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:33.594799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:33.594821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:33.599941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:33.600067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574436103899498694:2048] [node 1] ICS07 socket disconnect 396 reason# EndOfStream
2025-11-19T14:00:33.600075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436103899498694:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:33.600140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:33.600166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436103899498694:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:33.600640Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436103899498606:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61551) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:33.600654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436103899498694:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:33.600659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436103899498606:2048] [node 1] ICP30 unregister session Session# [3:7574436103899498694:2048] VirtualId# [3:7574436103899498691:0]
2025-11-19T14:00:33.600715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:33.600721Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436103899498606:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:33.600725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61551) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:33.609529Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:33.609626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436103087368133:2048] [node 1] ICS01 socket: 422 reason# EndOfStream
2025-11-19T14:00:33.609636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436103087368039:2048] [node 1] ICP30 unregister session Session# [4:7574436103087368133:2048] VirtualId# [4:7574436103087368122:0]
2025-11-19T14:00:33.609644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436103087368133:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:33.609692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:33.609716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:33.609750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:33.609810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436104258168743:2048] [node 1] ICS01 socket: 420 reason# EndOfStream
2025-11-19T14:00:33.609814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436104258168652:2048] [node 1] ICP30 unregister session Session# [5:7574436104258168743:2048] VirtualId# [5:7574436104258168733:0]
2025-11-19T14:00:33.609818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436104258168743:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:33.609841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:33.609858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:33.609884Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:33.609944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436103412434188:2048] [node 1] ICS01 socket: 388 reason# EndOfStream
2025-11-19T14:00:33.609949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436103412434105:2048] [node 1] ICP30 unregister session Session# [6:7574436103412434188:2048] VirtualId# [6:7574436103412434185:0]
2025-11-19T14:00:33.609954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436103412434188:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:33.609975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:33.609992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:33.611888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574436103899498606:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:33.619224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:33.619331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436103899498706:2049] [node 2] ICS01 socket: 411 reason# EndOfStream
2025-11-19T14:00:33.619342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436103899498607:2049] [node 2] ICP30 unregister session Session# [3:7574436103899498706:2049] VirtualId# [3:7574436103899498697:0]
2025-11-19T14:00:33.619350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436103899498706:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:33.619390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:33.619412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:33.623991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:33.624110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436103412434197:2049] [node 2] ICS01 socket: 425 reason# EndOfStream
2025-11-19T14:00:33.624122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436103412434106:2049] [node 2] ICP30 unregister session Session# [6:7574436103412434197:2049] VirtualId# [6:7574436103412434192:0]
2025-11-19T14:00:33.624133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436103412434197:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:33.624181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:33.624206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:33.624874Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:33.624954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436104258168741:2049] [node 2] ICS01 socket: 412 reason# EndOfStream
2025-11-19T14:00:33.624960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436104258168653:2049] [node 2] ICP30 unregister session Session# [5:7574436104258168741:2049] VirtualId# [5:7574436104258168732:0]
2025-11-19T14:00:33.624967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436104258168741:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:33.625013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:33.625034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:33.625066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:33.625128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436103087368135:2049] [node 2] ICS01 socket: 413 reason# EndOfStream
2025-11-19T14:00:33.625134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436103087368040:2049] [node 2] ICP30 unregister session Session# [4:7574436103087368135:2049] VirtualId# [4:7574436103087368124:0]
2025-11-19T14:00:33.625138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436103087368135:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:33.625161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:33.625178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:33.654502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:33.654571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436104258168748:2050] [node 3] ICS01 socket: 440 reason# EndOfStream
2025-11-19T14:00:33.654576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436104258168654:2050] [node 3] ICP30 unregister session Session# [5:7574436104258168748:2050] VirtualId# [5:7574436104258168736:0]
2025-11-19T14:00:33.654580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436104258168748:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:33.654607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:33.654626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:33.654035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:33.654151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436103412434205:2050] [node 3] ICS01 socket: 444 reason# EndOfStream
2025-11-19T14:00:33.654163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436103412434107:2050] [node 3] ICP30 unregister session Session# [6:7574436103412434205:2050] VirtualId# [6:7574436103412434202:0]
2025-11-19T14:00:33.654172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436103412434205:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:33.654231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:33.654257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:33.676079Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:33.676192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436103412434199:2075] [node 4] ICS01 socket: 387 reason# EndOfStream
2025-11-19T14:00:33.676205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436103412434154:2075] [node 4] ICP30 unregister session Session# [6:7574436103412434199:2075] VirtualId# [6:7574436103412434190:0]
2025-11-19T14:00:33.676215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436103412434199:2075] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:33.676261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:33.676285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:33.828869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436105993520548:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:33.828890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436105993520549:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:33.897266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436106016722488:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:33.897286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436106016722489:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:33.923894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436105077469575:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:33.923916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436105077469576:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:33.963915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436102865912297:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:33.963937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436102865912298:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:33.963943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436102865912299:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:33.996522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436106334247352:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:33.996548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436106334247353:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:33.996553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436106334247354:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:34.023838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436108651401840:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:34.023859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436108651401841:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:34.023865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436108651401842:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:34.094493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436110288487905:2107] [node 6] ICP01 ready to work
2025-11-19T14:00:34.094112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436108651401857:2061] [node 5] ICP01 ready to work
2025-11-19T14:00:34.094212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436108651401858:2063] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.095078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436110629214683:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42600
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.094864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436110288487909:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:34.095661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436107160879654:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:103
2025-11-19T14:00:34.095699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436109372436931:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32020
2025-11-19T14:00:34.095934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436107160879655:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:103
2025-11-19T14:00:34.103090Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436109372436931:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.104712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436110288487918:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.110353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436109372436934:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32020
2025-11-19T14:00:34.112186Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436109372436934:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.112386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436109372436936:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:34.112726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436109372436936:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.112549Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436107160879655:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.112707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436107160879657:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:34.113113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436107160879657:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.115088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436107160879659:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:103
2025-11-19T14:00:34.115543Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436107160879655:2109] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:34.115630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436107160879657:2110] [node 6] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.115642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:34.115649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436107160879657:2110] [node 6] ICP22 created new session: [4:7574436107160879660:2110]
2025-11-19T14:00:34.115656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436107160879660:2110] [node 6] ICS09 handshake done sender: [4:7574436107160879655:2109] self: [4:7574436107160879658:0] peer: [6:7574436108651401862:0] socket: 392 qp: -1
2025-11-19T14:00:34.115663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436107160879660:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:34.115688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:34.113522Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436108651401899:2097] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.113537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574436108651401899:2097] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:34.114212Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436108651401863:2065] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.114215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [6:7574436108651401863:2065] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:34.115242Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436108651401899:2097] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:34.115359Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436108651401863:2065] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:34.115819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574436108651401842:2050] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.115831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:34.115836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436108651401842:2050] [node 3] ICP22 created new session: [6:7574436108651401906:2050]
2025-11-19T14:00:34.115841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436108651401906:2050] [node 3] ICS09 handshake done sender: [6:7574436108651401899:2097] self: [6:7574436108651401898:0] peer: [3:7574436109372436937:0] socket: 393 qp: -1
2025-11-19T14:00:34.115847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436108651401906:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:34.115853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:34.115797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436106016722489:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17748610ns) {Configure:104(17747498ns) {ProcessConfigured:135(17336727ns) {StartInitialHandshake:157(17336280ns) {DropHandshakes:469(836ns) {DropIncomingHandshake:441(198ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:476(16766ns) {SwitchToState:178(1076ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.116665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7574436108651401858:2063] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.116681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:34.116686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436108651401858:2063] [node 4] ICP22 created new session: [6:7574436108651401908:2063]
2025-11-19T14:00:34.116692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436108651401908:2063] [node 4] ICS09 handshake done sender: [6:7574436108651401863:2065] self: [6:7574436108651401862:0] peer: [4:7574436107160879658:0] socket: 389 qp: -1
2025-11-19T14:00:34.116697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436108651401908:2063] [node 4] ICS10 traffic start
2025-11-19T14:00:34.116704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:34.116789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436108651401841:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(18494183ns) {Configure:104(18492849ns) {ProcessConfigured:135(18168079ns) {StartInitialHandshake:157(18167745ns) {DropHandshakes:469(467ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(40ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:476(620ns) {SwitchToState:178(273ns)}}}}}
2025-11-19T14:00:34.116481Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436105077469575:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18362502ns) {Configure:104(18361491ns) {ProcessConfigured:135(17957816ns) {StartInitialHandshake:157(17957442ns) {DropHandshakes:469(667ns) {DropIncomingHandshake:441(140ns) DropOutgoingHandshake:456(79ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(1062ns) {SwitchToState:178(580ns)}}}}}
2025-11-19T14:00:34.116529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436105077469575:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.116913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436109372436941:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32020
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.117127Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436109372436934:2108] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:34.118850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436109372436936:2109] [node 6] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.118862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:34.118868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436109372436936:2109] [node 6] ICP22 created new session: [3:7574436109372436942:2109]
2025-11-19T14:00:34.118874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436109372436942:2109] [node 6] ICS09 handshake done sender: [3:7574436109372436934:2108] self: [3:7574436109372436937:0] peer: [6:7574436108651401898:0] socket: 394 qp: -1
2025-11-19T14:00:34.118879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436109372436942:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:34.118911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:34.119844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436110629214695:2086] [node 4] ICP01 ready to work
2025-11-19T14:00:34.119922Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436110629214683:2082] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.119218Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436110288487908:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.119234Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436110288487908:2109] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:34.122311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436108651401840:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18283746ns) {Configure:104(18283332ns) {ProcessConfigured:135(17986392ns) {StartInitialHandshake:157(17986057ns) {DropHandshakes:469(421ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(1066ns) {SwitchToState:178(727ns)}}}}}
2025-11-19T14:00:34.122686Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436107160879654:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.122458Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436106334247354:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(17819625ns) {Configure:104(17818588ns) {ProcessConfigured:135(17428218ns) {StartInitialHandshake:157(17427935ns) {DropHandshakes:469(523ns) {DropIncomingHandshake:441(76ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:476(808ns) {SwitchToState:178(398ns)}}}}}
2025-11-19T14:00:34.123847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436110629214707:2090] [node 6] ICP01 ready to work
2025-11-19T14:00:34.122655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436110311689849:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:34.122818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436110311689848:2107] [node 6] ICP01 ready to work
2025-11-19T14:00:34.126356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436110311689850:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:34.122766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436110288487922:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41503
2025-11-19T14:00:34.123671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436109372436946:2112] [node 5] ICP01 ready to work
2025-11-19T14:00:34.123792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436109372436947:2113] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.131236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436110311689852:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15979
2025-11-19T14:00:34.132055Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436110311689852:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.133302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436106016722488:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.136122Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436110288487922:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.160060Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436110288487908:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.169734Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436105077469576:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(71005167ns) {Configure:104(71004435ns) {ProcessConfigured:135(70772536ns) {StartInitialHandshake:157(70771696ns) {DropHandshakes:469(346ns) {DropIncomingHandshake:441(46ns) DropOutgoingHandshake:456(43ns)} GenerateSessionVirtualId:988(142ns) PrepareNewSessionHandshake:476(1942ns) {SwitchToState:178(977ns)}}}}}
2025-11-19T14:00:34.169540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436110311689863:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15979
2025-11-19T14:00:34.169590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436110311689864:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15979
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.174972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436109372436955:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32020
2025-11-19T14:00:34.174991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436109372436956:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32020
2025-11-19T14:00:34.175046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436109372436957:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32020
2025-11-19T14:00:34.175352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436105993520549:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.175392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:34.175399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436105993520549:2049] [node 3] ICP22 created new session: [1:7574436110288487928:2049]
2025-11-19T14:00:34.175407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436110288487928:2049] [node 3] ICS09 handshake done sender: [1:7574436110288487908:2109] self: [1:7574436110288487907:0] peer: [3:7574436109372436940:0] socket: 386 qp: -1
2025-11-19T14:00:34.175415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436110288487928:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:34.175427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:34.175758Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436110288487912:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.175766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436110288487912:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.176545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436110629214707:2090] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(52545317ns) {Configure:104(52544569ns) {ProcessConfigured:135(52272803ns) {StartInitialHandshake:157(52255512ns) {DropHandshakes:469(504ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(333ns) PrepareNewSessionHandshake:476(1333ns) {SwitchToState:178(988ns)}} IncomingHandshake:261(8180ns) {DropIncomingHandshake:441(253ns) PrepareNewSessionHandshake:476(191ns) {SwitchToState:178(150ns)} GenerateSessionVirtualId:988(715ns) IssueIncomingHandshakeReply:193(667ns)}}}}
2025-11-19T14:00:34.177887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436106334247353:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(53936095ns) {Configure:104(53935168ns) {ProcessConfigured:135(53627909ns) {StartInitialHandshake:157(53627034ns) {DropHandshakes:469(512ns) {DropIncomingHandshake:441(84ns) DropOutgoingHandshake:456(74ns)} GenerateSessionVirtualId:988(307ns) PrepareNewSessionHandshake:476(1096ns) {SwitchToState:178(468ns)}}}}}
2025-11-19T14:00:34.178013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436110629214723:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42600
2025-11-19T14:00:34.178077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436110629214724:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42600
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.179065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436110311689848:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(52565521ns) {Configure:104(52565067ns) {ProcessConfigured:135(52239580ns) {StartInitialHandshake:157(52239321ns) {DropHandshakes:469(544ns) {DropIncomingHandshake:441(33ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(242ns) PrepareNewSessionHandshake:476(718ns) {SwitchToState:178(262ns)}}}}}
2025-11-19T14:00:34.179126Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436110311689863:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.178985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436108651401926:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25442
2025-11-19T14:00:34.178793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436106334247352:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(53277149ns) {Configure:104(53275957ns) {ProcessConfigured:135(53004194ns) {StartInitialHandshake:157(53003365ns) {DropHandshakes:469(421ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:476(669ns) {SwitchToState:178(327ns)}}}}}
2025-11-19T14:00:34.179466Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436110629214723:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.179489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436106334247354:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.179592Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436109372436949:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.179601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436109372436949:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:34.180261Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436110629214724:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.180294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436106334247353:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.180337Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436109372436956:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.180379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436105077469576:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.180530Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436109372436931:2107] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:34.180868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436105077469575:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.180890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:34.180896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436105077469575:2048] [node 1] ICP22 created new session: [3:7574436109372436959:2048]
2025-11-19T14:00:34.180904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436109372436959:2048] [node 1] ICS09 handshake done sender: [3:7574436109372436931:2107] self: [3:7574436109372436940:0] peer: [1:7574436110288487907:0] socket: 390 qp: -1
2025-11-19T14:00:34.180913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436109372436959:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:34.180923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:34.188855Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436109372436949:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:34.189016Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436110288487915:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.189060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436110288487915:2111] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:34.191816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436102865912297:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.191880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436102865912297:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(69104802ns) {IncomingHandshake:261(69103966ns) {DropIncomingHandshake:441(101ns) PrepareNewSessionHandshake:476(378ns) {SwitchToState:178(122ns)} GenerateSessionVirtualId:988(219ns) IssueIncomingHandshakeReply:193(69098666ns) {SwitchToState:178(1091ns)}}}
2025-11-19T14:00:34.192198Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436110311689858:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.192217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436110311689858:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.197219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436107160879671:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:103
2025-11-19T14:00:34.195782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436110288487918:2112] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(90950086ns) {Configure:104(90949012ns) {ProcessConfigured:135(90642650ns) {StartInitialHandshake:157(90641403ns) {DropHandshakes:469(450ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(69ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:476(1021ns) {SwitchToState:178(629ns)}}}}}
2025-11-19T14:00:34.196288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436110288487925:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41503
2025-11-19T14:00:34.196305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436110288487926:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41503
2025-11-19T14:00:34.196351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436110288487927:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41503
2025-11-19T14:00:34.197569Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436110288487912:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:34.197660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436105993520548:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.197673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:34.197680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436105993520548:2048] [node 2] ICP22 created new session: [1:7574436110288487933:2048]
2025-11-19T14:00:34.197687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436110288487933:2048] [node 2] ICS09 handshake done sender: [1:7574436110288487912:2110] self: [1:7574436110288487911:0] peer: [2:7574436110311689862:0] socket: 387 qp: -1
2025-11-19T14:00:34.197692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436110288487933:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:34.197700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:34.208440Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436110288487926:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.208637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436110311689867:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15979
2025-11-19T14:00:34.208689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436110311689868:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15979
2025-11-19T14:00:34.208777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436109372436946:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.208804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:34.208814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436109372436946:2112] [node 5] ICP22 created new session: [3:7574436109372436962:2112]
2025-11-19T14:00:34.208826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436109372436962:2112] [node 5] ICS09 handshake done sender: [3:7574436109372436949:2114] self: [3:7574436109372436948:0] peer: [5:7574436110629214729:0] socket: 410 qp: -1
2025-11-19T14:00:34.208835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436109372436962:2112] [node 5] ICS10 traffic start
2025-11-19T14:00:34.208874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:34.209395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436109372436964:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32020
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.209976Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436110288487927:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.210002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436105993520548:2048] [node 2] ICP18 handshake (actor [1:7574436110288487927:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.210363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436110629214731:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42600
2025-11-19T14:00:34.210464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436110629214732:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42600
2025-11-19T14:00:34.210506Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436110629214695:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(90473079ns) {Configure:104(90471995ns) {ProcessConfigured:135(90163982ns) {StartInitialHandshake:157(90162410ns) {DropHandshakes:469(423ns) {DropIncomingHandshake:441(65ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:476(1136ns) {SwitchToState:178(817ns)}}}}}
2025-11-19T14:00:34.210860Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436110311689851:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.210875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436110311689851:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:34.210975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436107160879670:2114] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.212096Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436107160879671:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.212190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436102865912299:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.212209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436107160879676:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:103
2025-11-19T14:00:34.212357Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436109372436951:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.212366Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436109372436951:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.214261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436107160879678:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:103
2025-11-19T14:00:34.217148Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436110629214723:2113] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:34.218902Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436107160879671:2115] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:34.217240Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436109372436951:2115] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:34.219346Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436110288487915:2111] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:34.219547Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436107160879654:2108] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:34.219292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436106334247354:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.219326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:34.219336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436106334247354:2050] [node 3] ICP22 created new session: [5:7574436110629214734:2050]
2025-11-19T14:00:34.219343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436110629214734:2050] [node 3] ICS09 handshake done sender: [5:7574436110629214723:2113] self: [5:7574436110629214729:0] peer: [3:7574436109372436948:0] socket: 422 qp: -1
2025-11-19T14:00:34.219352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436110629214734:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:34.219377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.219867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436109372436947:2113] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.219883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:34.219890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436109372436947:2113] [node 4] ICP22 created new session: [3:7574436109372436967:2113]
2025-11-19T14:00:34.219896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436109372436967:2113] [node 4] ICS09 handshake done sender: [3:7574436109372436951:2115] self: [3:7574436109372436950:0] peer: [4:7574436107160879677:0] socket: 411 qp: -1
2025-11-19T14:00:34.219903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436109372436967:2113] [node 4] ICS10 traffic start
2025-11-19T14:00:34.219912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:34.219895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436110311689849:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(96954355ns) {Configure:104(96953582ns) {ProcessConfigured:135(96642431ns) {StartInitialHandshake:157(96641059ns) {DropHandshakes:469(552ns) {DropIncomingHandshake:441(46ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:476(1626ns) {SwitchToState:178(1114ns)}}}}}
2025-11-19T14:00:34.219927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436106016722488:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8098) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436110311689852:2111] held: no
2025-11-19T14:00:34.220061Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436110311689852:2111] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:34.220375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436102865912297:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.220392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:34.220399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436102865912297:2048] [node 1] ICP22 created new session: [4:7574436107160879679:2048]
2025-11-19T14:00:34.220406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436107160879679:2048] [node 1] ICS09 handshake done sender: [4:7574436107160879654:2108] self: [4:7574436107160879668:0] peer: [1:7574436110288487913:0] socket: 391 qp: -1
2025-11-19T14:00:34.220413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436107160879679:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:34.220420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:34.220675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436102865912299:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.220689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:34.220695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436102865912299:2050] [node 3] ICP22 created new session: [4:7574436107160879681:2050]
2025-11-19T14:00:34.220699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436107160879681:2050] [node 3] ICS09 handshake done sender: [4:7574436107160879671:2115] self: [4:7574436107160879677:0] peer: [3:7574436109372436950:0] socket: 416 qp: -1
2025-11-19T14:00:34.220708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436107160879681:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:34.220715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.220958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436110288487909:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.220975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:34.220980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436110288487909:2108] [node 4] ICP22 created new session: [1:7574436110288487939:2108]
2025-11-19T14:00:34.220984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436110288487939:2108] [node 4] ICS09 handshake done sender: [1:7574436110288487915:2111] self: [1:7574436110288487913:0] peer: [4:7574436107160879668:0] socket: 388 qp: -1
2025-11-19T14:00:34.220989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436110288487939:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:34.220995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.221061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436108651401927:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25442
2025-11-19T14:00:34.222164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436110288487938:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41503
2025-11-19T14:00:34.221875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436110311689869:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15979
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.222342Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436110629214732:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.222442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436106334247352:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.222462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436110629214737:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42600
2025-11-19T14:00:34.222625Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436108651401927:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.222661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436108651401840:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.222957Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436110311689858:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:34.223159Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436110629214724:2114] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:34.223278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436106334247353:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.223294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:34.223299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436106334247353:2049] [node 2] ICP22 created new session: [5:7574436110629214739:2049]
2025-11-19T14:00:34.223304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436110629214739:2049] [node 2] ICS09 handshake done sender: [5:7574436110629214724:2114] self: [5:7574436110629214730:0] peer: [2:7574436110311689857:0] socket: 423 qp: -1
2025-11-19T14:00:34.223309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436110629214739:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:34.223315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:34.223668Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436110311689851:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.223985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436110629214742:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42600
2025-11-19T14:00:34.224110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436106016722488:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.224123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:34.224129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436106016722488:2048] [node 1] ICP22 created new session: [2:7574436110311689873:2048]
2025-11-19T14:00:34.224134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436110311689873:2048] [node 1] ICS09 handshake done sender: [2:7574436110311689852:2111] self: [2:7574436110311689862:0] peer: [1:7574436110288487911:0] socket: 398 qp: -1
2025-11-19T14:00:34.224139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436110311689873:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:34.224156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:34.224312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436110311689850:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.224319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:34.224323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436110311689850:2108] [node 5] ICP22 created new session: [2:7574436110311689875:2108]
2025-11-19T14:00:34.224327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436110311689875:2108] [node 5] ICS09 handshake done sender: [2:7574436110311689858:2113] self: [2:7574436110311689857:0] peer: [5:7574436110629214730:0] socket: 414 qp: -1
2025-11-19T14:00:34.224331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436110311689875:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:34.224335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.225187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436108651401929:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25442
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.226722Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436110288487930:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.226733Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436110288487930:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:34.227180Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436110288487920:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.227183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436110288487920:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:34.228377Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436110288487930:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:34.228521Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436110288487920:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:34.228664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436110288487918:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.228680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:34.228686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436110288487918:2112] [node 5] ICP22 created new session: [1:7574436110288487943:2112]
2025-11-19T14:00:34.228693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436110288487943:2112] [node 5] ICS09 handshake done sender: [1:7574436110288487930:2118] self: [1:7574436110288487921:0] peer: [5:7574436110629214738:0] socket: 432 qp: -1
2025-11-19T14:00:34.228699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436110288487943:2112] [node 5] ICS10 traffic start
2025-11-19T14:00:34.228709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.225698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436107160879685:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:103
2025-11-19T14:00:34.225715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436107160879686:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:103
2025-11-19T14:00:34.229179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436108651401930:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25442
2025-11-19T14:00:34.229383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436110288487905:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.229402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:34.229407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436110288487905:2107] [node 6] ICP22 created new session: [1:7574436110288487945:2107]
2025-11-19T14:00:34.229413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436110288487945:2107] [node 6] ICS09 handshake done sender: [1:7574436110288487920:2113] self: [1:7574436110288487919:0] peer: [6:7574436108651401928:0] socket: 430 qp: -1
2025-11-19T14:00:34.229419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436110288487945:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:34.229425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:34.229872Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436108651401927:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:34.229961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436108651401840:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.229979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:34.229986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436108651401840:2048] [node 1] ICP22 created new session: [6:7574436108651401931:2048]
2025-11-19T14:00:34.229991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436108651401931:2048] [node 1] ICS09 handshake done sender: [6:7574436108651401927:2116] self: [6:7574436108651401928:0] peer: [1:7574436110288487919:0] socket: 433 qp: -1
2025-11-19T14:00:34.229996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436108651401931:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:34.230002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.233503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436109372436969:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32020
2025-11-19T14:00:34.229976Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436110311689869:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.233940Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436109372436956:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:34.234050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436105077469576:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.234070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:34.234075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436105077469576:2049] [node 2] ICP22 created new session: [3:7574436109372436970:2049]
2025-11-19T14:00:34.234082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436109372436970:2049] [node 2] ICS09 handshake done sender: [3:7574436109372436956:2118] self: [3:7574436109372436958:0] peer: [2:7574436110311689845:0] socket: 418 qp: -1
2025-11-19T14:00:34.234088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436109372436970:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:34.234097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.235135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436110629214743:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42600
2025-11-19T14:00:34.235812Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436110629214742:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.235856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436110629214695:2086] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.236118Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436107160879683:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.236135Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436107160879683:2119] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.240446Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436108651401926:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.240499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436108651401857:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.240555Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436107160879683:2119] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:34.240346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436110629214745:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42600
2025-11-19T14:00:34.241626Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436108651401929:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.241652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436108651401841:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.241931Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436107160879685:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.241952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436107160879670:2114] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.241964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:34.241971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436107160879670:2114] [node 5] ICP22 created new session: [4:7574436107160879688:2114]
2025-11-19T14:00:34.241977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436107160879688:2114] [node 5] ICS09 handshake done sender: [4:7574436107160879683:2119] self: [4:7574436107160879675:0] peer: [5:7574436110629214744:0] socket: 431 qp: -1
2025-11-19T14:00:34.241982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436107160879688:2114] [node 5] ICS10 traffic start
2025-11-19T14:00:34.242003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:34.243930Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436110288487938:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.244253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436110288487918:2112] [node 5] ICP18 handshake (actor [1:7574436110288487938:2119]) is too old
2025-11-19T14:00:34.243778Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436110629214721:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.243792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436110629214721:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:34.244227Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436110629214742:2121] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:34.244511Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436110629214732:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:34.244953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436106016722489:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.244976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:34.244982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436106016722489:2049] [node 3] ICP22 created new session: [2:7574436110311689880:2049]
2025-11-19T14:00:34.244988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436110311689880:2049] [node 3] ICS09 handshake done sender: [2:7574436110311689851:2110] self: [2:7574436110311689845:0] peer: [3:7574436109372436958:0] socket: 412 qp: -1
2025-11-19T14:00:34.244993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436110311689880:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:34.245024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:34.245778Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436110311689865:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.245786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436110311689865:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:34.250929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7574436107160879670:2114] [node 5] ICP18 handshake (actor [4:7574436107160879685:2120]) is too old
2025-11-19T14:00:34.251081Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436107160879686:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.251245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436102865912298:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:34.256011Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436110311689865:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:34.256068Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436110311689864:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.256234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436110311689848:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.256250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:34.256258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436110311689848:2107] [node 6] ICP22 created new session: [2:7574436110311689884:2107]
2025-11-19T14:00:34.256264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436110311689884:2107] [node 6] ICS09 handshake done sender: [2:7574436110311689865:2116] self: [2:7574436110311689855:0] peer: [6:7574436108651401934:0] socket: 444 qp: -1
2025-11-19T14:00:34.256273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436110311689884:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:34.256282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:34.254865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436110629214695:2086] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.254891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:34.254900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436110629214695:2086] [node 4] ICP22 created new session: [5:7574436110629214747:2086]
2025-11-19T14:00:34.254907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436110629214747:2086] [node 4] ICS09 handshake done sender: [5:7574436110629214742:2121] self: [5:7574436110629214744:0] peer: [4:7574436107160879675:0] socket: 413 qp: -1
2025-11-19T14:00:34.254914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436110629214747:2086] [node 4] ICS10 traffic start
2025-11-19T14:00:34.254939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:34.255355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436106334247352:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8098) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436110629214732:2118] held: no
2025-11-19T14:00:34.255434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436106334247352:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.255445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:34.255451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436106334247352:2048] [node 1] ICP22 created new session: [5:7574436110629214749:2048]
2025-11-19T14:00:34.255457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436110629214749:2048] [node 1] ICS09 handshake done sender: [5:7574436110629214732:2118] self: [5:7574436110629214738:0] peer: [1:7574436110288487921:0] socket: 437 qp: -1
2025-11-19T14:00:34.255462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436110629214749:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:34.255467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:34.256824Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574436110629214695:2086] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:34.259994Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436110629214721:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:34.263853Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436110311689870:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:34.263872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436110311689870:2120] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.268150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436108651401935:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25442
2025-11-19T14:00:34.268291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436108651401936:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25442
2025-11-19T14:00:34.268371Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436108651401929:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:34.268536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436108651401841:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.268563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:34.268571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436108651401841:2049] [node 2] ICP22 created new session: [6:7574436108651401937:2049]
2025-11-19T14:00:34.268577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436108651401937:2049] [node 2] ICS09 handshake done sender: [6:7574436108651401929:2117] self: [6:7574436108651401934:0] peer: [2:7574436110311689855:0] socket: 446 qp: -1
2025-11-19T14:00:34.268585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436108651401937:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:34.268594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:34.270235Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436108651401926:2115] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:34.270344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436108651401857:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.270362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:34.270368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436108651401857:2061] [node 5] ICP22 created new session: [6:7574436108651401939:2061]
2025-11-19T14:00:34.270373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436108651401939:2061] [node 5] ICS09 handshake done sender: [6:7574436108651401926:2115] self: [6:7574436108651401933:0] peer: [5:7574436110629214716:0] socket: 426 qp: -1
2025-11-19T14:00:34.270379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436108651401939:2061] [node 5] ICS10 traffic start
2025-11-19T14:00:34.270386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:34.271835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436110629214707:2090] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.271864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:34.271871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436110629214707:2090] [node 6] ICP22 created new session: [5:7574436110629214751:2090]
2025-11-19T14:00:34.271877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436110629214751:2090] [node 6] ICS09 handshake done sender: [5:7574436110629214721:2112] self: [5:7574436110629214716:0] peer: [6:7574436108651401933:0] socket: 424 qp: -1
2025-11-19T14:00:34.271884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436110629214751:2090] [node 6] ICS10 traffic start
2025-11-19T14:00:34.271891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:34.278660Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436110311689870:2120] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:34.278678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436107160879691:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:103
2025-11-19T14:00:34.279007Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436107160879686:2121] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:34.278828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436110311689849:2109] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:34.278852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:34.278859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436110311689849:2109] [node 4] ICP22 created new session: [2:7574436110311689887:2109]
2025-11-19T14:00:34.278865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436110311689887:2109] [node 4] ICS09 handshake done sender: [2:7574436110311689870:2120] self: [2:7574436110311689854:0] peer: [4:7574436107160879690:0] socket: 445 qp: -1
2025-11-19T14:00:34.278871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436110311689887:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:34.278882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:34.279448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436102865912298:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:34.279476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:34.279482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436102865912298:2049] [node 2] ICP22 created new session: [4:7574436107160879692:2049]
2025-11-19T14:00:34.279490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436107160879692:2049] [node 2] ICS09 handshake done sender: [4:7574436107160879686:2121] self: [4:7574436107160879690:0] peer: [2:7574436110311689854:0] socket: 448 qp: -1
2025-11-19T14:00:34.279496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436107160879692:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:34.279507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:34.288924Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:34.289042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436110311689873:2048] [node 1] ICS01 socket: 398 reason# EndOfStream
2025-11-19T14:00:34.289053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436106016722488:2048] [node 1] ICP30 unregister session Session# [2:7574436110311689873:2048] VirtualId# [2:7574436110311689862:0]
2025-11-19T14:00:34.289070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436110311689873:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.289107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:34.289126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:34.292226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:34.292335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436109372436959:2048] [node 1] ICS01 socket: 390 reason# EndOfStream
2025-11-19T14:00:34.292347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436105077469575:2048] [node 1] ICP30 unregister session Session# [3:7574436109372436959:2048] VirtualId# [3:7574436109372436940:0]
2025-11-19T14:00:34.292356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436109372436959:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.292409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:34.292435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:34.302585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:34.302699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436107160879679:2048] [node 1] ICS01 socket: 391 reason# EndOfStream
2025-11-19T14:00:34.302710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436102865912297:2048] [node 1] ICP30 unregister session Session# [4:7574436107160879679:2048] VirtualId# [4:7574436107160879668:0]
2025-11-19T14:00:34.302734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436107160879679:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.302781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:34.302806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:34.302685Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:34.302764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436108651401931:2048] [node 1] ICS07 socket disconnect 433 reason# EndOfStream
2025-11-19T14:00:34.302770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436108651401931:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.302812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:34.302832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436108651401931:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:34.303538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436108651401840:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8098) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:34.303551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436108651401931:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:34.303555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436108651401840:2048] [node 1] ICP30 unregister session Session# [6:7574436108651401931:2048] VirtualId# [6:7574436108651401928:0]
2025-11-19T14:00:34.303604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:34.303611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436108651401840:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8098) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:34.303615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8098) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:34.306301Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:34.306401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436110629214749:2048] [node 1] ICS01 socket: 437 reason# EndOfStream
2025-11-19T14:00:34.306410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436106334247352:2048] [node 1] ICP30 unregister session Session# [5:7574436110629214749:2048] VirtualId# [5:7574436110629214738:0]
2025-11-19T14:00:34.306418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436110629214749:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.306469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:34.306501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:34.311882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436108651401840:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:34.311888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:34.311997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436109372436970:2049] [node 2] ICS01 socket: 418 reason# EndOfStream
2025-11-19T14:00:34.312007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436105077469576:2049] [node 2] ICP30 unregister session Session# [3:7574436109372436970:2049] VirtualId# [3:7574436109372436958:0]
2025-11-19T14:00:34.312015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436109372436970:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.312065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:34.312090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:34.316734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:34.316830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436110629214739:2049] [node 2] ICS01 socket: 423 reason# EndOfStream
2025-11-19T14:00:34.316840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436106334247353:2049] [node 2] ICP30 unregister session Session# [5:7574436110629214739:2049] VirtualId# [5:7574436110629214730:0]
2025-11-19T14:00:34.316848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436110629214739:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.316900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:34.316925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:34.319850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:34.319946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436108651401937:2049] [node 2] ICS01 socket: 446 reason# EndOfStream
2025-11-19T14:00:34.319960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436108651401841:2049] [node 2] ICP30 unregister session Session# [6:7574436108651401937:2049] VirtualId# [6:7574436108651401934:0]
2025-11-19T14:00:34.319970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436108651401937:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.320043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:34.320077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:34.326869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:34.326981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436107160879692:2049] [node 2] ICS01 socket: 448 reason# EndOfStream
2025-11-19T14:00:34.326994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436102865912298:2049] [node 2] ICP30 unregister session Session# [4:7574436107160879692:2049] VirtualId# [4:7574436107160879690:0]
2025-11-19T14:00:34.327003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436107160879692:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.327065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:34.327094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:34.373590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-11-19T14:00:34.373707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [4:7574436107160879681:2050] [node 3] ICS07 socket disconnect 416 reason# EndOfStream
2025-11-19T14:00:34.373714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436107160879681:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.373771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:34.373794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [4:7574436107160879681:2050] [node 3] ICS15 start handshake
2025-11-19T14:00:34.374483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436102865912299:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5245) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:34.374500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436107160879681:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:34.374507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436102865912299:2050] [node 3] ICP30 unregister session Session# [4:7574436107160879681:2050] VirtualId# [4:7574436107160879677:0]
2025-11-19T14:00:34.374568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:34.374577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7574436102865912299:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5245) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:34.374580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5245) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:34.376535Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:34.376616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574436110629214734:2050] [node 3] ICS07 socket disconnect 422 reason# EndOfStream
2025-11-19T14:00:34.376623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436110629214734:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.376675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:34.376698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574436110629214734:2050] [node 3] ICS15 start handshake
2025-11-19T14:00:34.377408Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:34.377487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436108651401906:2050] [node 3] ICS01 socket: 393 reason# EndOfStream
2025-11-19T14:00:34.377509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436108651401842:2050] [node 3] ICP30 unregister session Session# [6:7574436108651401906:2050] VirtualId# [6:7574436108651401898:0]
2025-11-19T14:00:34.377518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436108651401906:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.377569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:34.377606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:34.377842Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436106334247354:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5245) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:34.377858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436110629214734:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:34.377862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436106334247354:2050] [node 3] ICP30 unregister session Session# [5:7574436110629214734:2050] VirtualId# [5:7574436110629214729:0]
2025-11-19T14:00:34.377916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:34.377924Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574436106334247354:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5245) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:34.377928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5245) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:34.390303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574436106334247354:2050] [node 3] ICP33 wake up from error state
2025-11-19T14:00:34.404923Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-11-19T14:00:34.405037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574436110629214747:2086] [node 4] ICS07 socket disconnect 413 reason# EndOfStream
2025-11-19T14:00:34.405044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436110629214747:2086] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.405092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:34.405114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574436110629214747:2086] [node 4] ICS15 start handshake
2025-11-19T14:00:34.408069Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:34.408171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436108651401908:2063] [node 4] ICS01 socket: 389 reason# EndOfStream
2025-11-19T14:00:34.408183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436108651401858:2063] [node 4] ICP30 unregister session Session# [6:7574436108651401908:2063] VirtualId# [6:7574436108651401862:0]
2025-11-19T14:00:34.408192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436108651401908:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:34.408248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:34.408281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:34.604261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436108355405938:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:34.604278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436108355405939:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:34.651846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436107359981667:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:34.651845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436107359981666:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:34.695864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436108945593521:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:34.695880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436108945593522:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:34.795899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436108236894495:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:34.795917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436108236894496:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:34.795922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436108236894497:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:34.879125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436109549212049:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:34.879141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436109549212050:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:34.879146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436109549212051:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:35.004182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436109954399440:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:35.004257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436109954399441:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:35.004322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436109954399439:2048] [node 1] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.013663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436112650373298:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:35.013827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436112650373301:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:35.014489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436112650373299:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31497
2025-11-19T14:00:35.014501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436112650373300:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31497
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.013996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436111654949025:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32521
2025-11-19T14:00:35.014019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436111654949026:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32521
2025-11-19T14:00:35.015200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436111654949031:2110] [node 4] ICP01 ready to work
2025-11-19T14:00:35.015284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436111654949032:2112] [node 5] ICP01 ready to work
2025-11-19T14:00:35.018361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436113240560883:2110] [node 5] ICP01 ready to work
2025-11-19T14:00:35.018841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436113844179381:2073] [node 6] ICP01 ready to work
2025-11-19T14:00:35.020184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436114249366765:2068] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.020698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436111654949041:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32521
2025-11-19T14:00:35.019686Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436112650373299:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.021363Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436111654949041:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.021391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436107359981666:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.021493Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436112650373310:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.021502Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436112650373310:2115] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.022367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436112650373308:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31497
2025-11-19T14:00:35.022027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112531861820:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58184
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.033925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113844179401:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56305
2025-11-19T14:00:35.033950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113844179402:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56305
2025-11-19T14:00:35.035290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436113844179366:2066] [node 4] ICP01 ready to work
2025-11-19T14:00:35.035741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436112650373296:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:35.035959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436112531861829:2076] [node 5] ICP01 ready to work
2025-11-19T14:00:35.036151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436111654949040:2115] [node 6] ICP01 ready to work
2025-11-19T14:00:35.036884Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436111654949026:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.036730Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436112650373310:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.043881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436109549212050:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(27413574ns) {Configure:104(27407663ns) {ProcessConfigured:135(26669090ns) {StartInitialHandshake:157(26668129ns) {DropHandshakes:469(488ns) {DropIncomingHandshake:441(33ns) DropOutgoingHandshake:456(14ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:476(1511ns) {SwitchToState:178(852ns)}}}}}
2025-11-19T14:00:35.044425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436113240560890:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
2025-11-19T14:00:35.044498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436113240560891:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
2025-11-19T14:00:35.044511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436113240560892:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
2025-11-19T14:00:35.044545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436113240560893:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.047968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112531861860:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58184
2025-11-19T14:00:35.048302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436113844179381:2073] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27770065ns) {Configure:104(27767800ns) {ProcessConfigured:135(27379574ns) {StartInitialHandshake:157(27378273ns) {DropHandshakes:469(534ns) {DropIncomingHandshake:441(85ns) DropOutgoingHandshake:456(79ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:476(1474ns) {SwitchToState:178(651ns)}}}}}
2025-11-19T14:00:35.048862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436109549212049:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(26148146ns) {Configure:104(26147195ns) {ProcessConfigured:135(25884516ns) {StartInitialHandshake:157(25884022ns) {DropHandshakes:469(641ns) {DropIncomingHandshake:441(229ns) DropOutgoingHandshake:456(68ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:476(1194ns) {SwitchToState:178(444ns)}}}}}
2025-11-19T14:00:35.049899Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436113240560890:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.049938Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436113240560892:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.049962Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436113240560893:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.050013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436108945593521:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.051716Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436112531861860:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.051783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436108236894496:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.052867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436108945593522:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(26743436ns) {Configure:104(26742518ns) {ProcessConfigured:135(26369285ns) {StartInitialHandshake:157(26368469ns) {DropHandshakes:469(891ns) {DropIncomingHandshake:441(50ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(276ns) PrepareNewSessionHandshake:476(1008ns) {SwitchToState:178(787ns)}}}}}
2025-11-19T14:00:35.052934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436108945593522:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.053246Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436112650373300:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.056374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436114249366780:2078] [node 4] ICP01 ready to work
2025-11-19T14:00:35.053681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436113240560889:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:35.054416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436113240560888:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:35.056626Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436111654949034:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.056649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436111654949034:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.057767Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436112531861820:2071] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.060183Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436112650373312:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.060208Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436112650373312:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.063739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436109954399439:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(50674643ns) {Configure:104(50673688ns) {ProcessConfigured:135(50381634ns) {StartInitialHandshake:157(50380869ns) {DropHandshakes:469(410ns) {DropIncomingHandshake:441(69ns) DropOutgoingHandshake:456(53ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:476(2396ns) {SwitchToState:178(1077ns)}}}}}
2025-11-19T14:00:35.063984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436113844179366:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(28589927ns) {Configure:104(28588453ns) {ProcessConfigured:135(28247535ns) {StartInitialHandshake:157(28246956ns) {DropHandshakes:469(1820ns) {DropIncomingHandshake:441(217ns) DropOutgoingHandshake:456(202ns)} GenerateSessionVirtualId:988(171ns) PrepareNewSessionHandshake:476(954ns) {SwitchToState:178(230ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.064173Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436112650373296:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(28287525ns) {Configure:104(28287281ns) {ProcessConfigured:135(27928804ns) {StartInitialHandshake:157(27928528ns) {DropHandshakes:469(412ns) {DropIncomingHandshake:441(64ns) DropOutgoingHandshake:456(50ns)} GenerateSessionVirtualId:988(377ns) PrepareNewSessionHandshake:476(658ns) {SwitchToState:178(334ns)}}}}}
2025-11-19T14:00:35.064658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436112531861829:2076] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(28521546ns) {Configure:104(28520976ns) {ProcessConfigured:135(28154855ns) {StartInitialHandshake:157(28154283ns) {DropHandshakes:469(667ns) {DropIncomingHandshake:441(86ns) DropOutgoingHandshake:456(52ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:476(589ns) {SwitchToState:178(28ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.064850Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436111654949040:2115] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(28566139ns) {Configure:104(28565747ns) {ProcessConfigured:135(28211310ns) {StartInitialHandshake:157(28210382ns) {DropHandshakes:469(446ns) {DropIncomingHandshake:441(61ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:476(954ns) {SwitchToState:178(143ns)}}}}}
2025-11-19T14:00:35.065047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436111654949046:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32521
2025-11-19T14:00:35.065097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436111654949047:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32521
2025-11-19T14:00:35.065145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436111654949048:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32521
2025-11-19T14:00:35.064456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114249366812:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11304
2025-11-19T14:00:35.064522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114249366813:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11304
2025-11-19T14:00:35.067092Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436114249366812:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.067133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436109954399439:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.067051Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436111654949041:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:35.067151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436107359981666:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.067172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:35.067184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436107359981666:2048] [node 1] ICP22 created new session: [2:7574436111654949049:2048]
2025-11-19T14:00:35.067191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436111654949049:2048] [node 1] ICS09 handshake done sender: [2:7574436111654949041:2116] self: [2:7574436111654949042:0] peer: [1:7574436112650373309:0] socket: 411 qp: -1
2025-11-19T14:00:35.067201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436111654949049:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:35.067226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:35.067417Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436112650373305:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.067431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436112650373305:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.067540Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436111654949047:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.067647Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436114249366813:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.067667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436114249366765:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.072107Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436111654949029:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.072132Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436111654949029:2111] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.071998Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436112650373305:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:35.072186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436112531861848:2083] [node 6] ICP01 ready to work
2025-11-19T14:00:35.072322Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436113844179419:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.072336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436113844179419:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.072445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436112650373301:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.072464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:35.072472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436112650373301:2109] [node 6] ICP22 created new session: [1:7574436112650373321:2109]
2025-11-19T14:00:35.072479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436112650373321:2109] [node 6] ICS09 handshake done sender: [1:7574436112650373305:2113] self: [1:7574436112650373304:0] peer: [6:7574436114249366814:0] socket: 399 qp: -1
2025-11-19T14:00:35.072486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436112650373321:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:35.072514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:35.072589Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436112650373303:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.072597Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436112650373303:2112] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.072859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436108355405938:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.072891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:35.072900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436108355405938:2048] [node 2] ICP22 created new session: [1:7574436112650373323:2048]
2025-11-19T14:00:35.072908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436112650373323:2048] [node 2] ICS09 handshake done sender: [1:7574436112650373310:2115] self: [1:7574436112650373309:0] peer: [2:7574436111654949042:0] socket: 408 qp: -1
2025-11-19T14:00:35.072915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436112650373323:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:35.072936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:35.079936Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436113240560889:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(26098208ns) {Configure:104(26097331ns) {ProcessConfigured:135(25770546ns) {StartInitialHandshake:157(25769611ns) {DropHandshakes:469(1059ns) {DropIncomingHandshake:441(51ns) DropOutgoingHandshake:456(646ns)} GenerateSessionVirtualId:988(260ns) PrepareNewSessionHandshake:476(1904ns) {SwitchToState:178(298ns)}}}}}
2025-11-19T14:00:35.080093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436113240560888:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(25568769ns) {Configure:104(25568582ns) {ProcessConfigured:135(25360964ns) {StartInitialHandshake:157(25360685ns) {DropHandshakes:469(279ns) {DropIncomingHandshake:441(19ns) DropOutgoingHandshake:456(24ns)} GenerateSessionVirtualId:988(187ns) PrepareNewSessionHandshake:476(832ns) {SwitchToState:178(278ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.082644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436108236894495:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.082684Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436108236894495:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(24861607ns) {IncomingHandshake:261(24861022ns) {DropIncomingHandshake:441(59ns) PrepareNewSessionHandshake:476(680ns) {SwitchToState:178(445ns)} GenerateSessionVirtualId:988(210ns) IssueIncomingHandshakeReply:193(24854265ns) {SwitchToState:178(1354ns)}}}
2025-11-19T14:00:35.081493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436113240560903:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
2025-11-19T14:00:35.083338Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436111654949034:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:35.083452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436111654949031:2110] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.083473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:35.083483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436111654949031:2110] [node 4] ICP22 created new session: [2:7574436111654949053:2110]
2025-11-19T14:00:35.083490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436111654949053:2110] [node 4] ICS09 handshake done sender: [2:7574436111654949034:2113] self: [2:7574436111654949033:0] peer: [4:7574436112531861861:0] socket: 402 qp: -1
2025-11-19T14:00:35.083497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436111654949053:2110] [node 4] ICS10 traffic start
2025-11-19T14:00:35.083510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:35.081244Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436114249366780:2078] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(24700271ns) {Configure:104(24699635ns) {ProcessConfigured:135(24311650ns) {StartInitialHandshake:157(24311157ns) {DropHandshakes:469(803ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(71ns)} GenerateSessionVirtualId:988(311ns) PrepareNewSessionHandshake:476(958ns) {SwitchToState:178(597ns)}}}}}
2025-11-19T14:00:35.083972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436108236894497:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(37238843ns) {Configure:104(37236911ns) {ProcessConfigured:135(36918888ns) {StartInitialHandshake:157(36918751ns) {DropHandshakes:469(453ns) {DropIncomingHandshake:441(48ns) DropOutgoingHandshake:456(60ns)} GenerateSessionVirtualId:988(129ns) PrepareNewSessionHandshake:476(705ns) {SwitchToState:178(282ns)}}}}}
2025-11-19T14:00:35.084382Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436113240560903:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.084881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436108355405938:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(47323356ns) {HandleHandshakeStatus:329(47321319ns) {DropIncomingHandshake:441(9691ns) DropOutgoingHandshake:456(291ns) SwitchToState:178(268ns) ProcessPendingSessionEvents:495(147946ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 128707ns max# 39896ns}}}
2025-11-19T14:00:35.096491Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436112650373312:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:35.096586Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436113844179401:2096] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.096651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436109549212051:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.096942Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436113240560891:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.105681Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436113844179419:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:35.105933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436113844179381:2073] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.105948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:35.105956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436113844179381:2073] [node 6] ICP22 created new session: [5:7574436113844179427:2073]
2025-11-19T14:00:35.105962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436113844179427:2073] [node 6] ICS09 handshake done sender: [5:7574436113844179419:2112] self: [5:7574436113844179382:0] peer: [6:7574436114249366815:0] socket: 423 qp: -1
2025-11-19T14:00:35.105969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436113844179427:2073] [node 6] ICS10 traffic start
2025-11-19T14:00:35.105978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:35.105750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436108355405939:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.105776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:35.105786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436108355405939:2049] [node 3] ICP22 created new session: [1:7574436112650373327:2049]
2025-11-19T14:00:35.105795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436112650373327:2049] [node 3] ICS09 handshake done sender: [1:7574436112650373312:2116] self: [1:7574436112650373311:0] peer: [3:7574436113240560895:0] socket: 418 qp: -1
2025-11-19T14:00:35.105803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436112650373327:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:35.105813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:35.106098Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436112650373308:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.110004Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436108945593521:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436113240560893:2115] held: no
2025-11-19T14:00:35.111522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436108236894495:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436112531861820:2071] held: no
2025-11-19T14:00:35.108592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436108355405939:2049] [node 3] ICP18 handshake (actor [1:7574436112650373308:2114]) is too old
2025-11-19T14:00:35.110361Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436112650373303:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:35.110427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436112650373298:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.110447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:35.110455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436112650373298:2108] [node 4] ICP22 created new session: [1:7574436112650373330:2108]
2025-11-19T14:00:35.110462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436112650373330:2108] [node 4] ICS09 handshake done sender: [1:7574436112650373303:2112] self: [1:7574436112650373302:0] peer: [4:7574436112531861869:0] socket: 398 qp: -1
2025-11-19T14:00:35.110469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436112650373330:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:35.110482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:35.113075Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436113240560885:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.113090Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436113240560885:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.111915Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436112531861848:2083] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(39557892ns) {Configure:104(39555887ns) {ProcessConfigured:135(39157456ns) {StartInitialHandshake:157(39156482ns) {DropHandshakes:469(853ns) {DropIncomingHandshake:441(120ns) DropOutgoingHandshake:456(80ns)} GenerateSessionVirtualId:988(333ns) PrepareNewSessionHandshake:476(878ns) {SwitchToState:178(176ns)}}}}}
2025-11-19T14:00:35.113439Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436111654949029:2111] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:35.112420Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436112650373301:2109] [node 6] ICP35 event processing took too much time PendingConnection:278(39878941ns) {HandleHandshakeStatus:329(39877838ns) {DropIncomingHandshake:441(401ns) DropOutgoingHandshake:456(303ns) SwitchToState:178(344ns) ProcessPendingSessionEvents:495(29512908ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29494231ns max# 29418154ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.115873Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436111654949025:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.115892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436107359981667:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.115906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:35.115917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436107359981667:2049] [node 3] ICP22 created new session: [2:7574436111654949058:2049]
2025-11-19T14:00:35.115924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436111654949058:2049] [node 3] ICS09 handshake done sender: [2:7574436111654949029:2111] self: [2:7574436111654949028:0] peer: [3:7574436113240560898:0] socket: 401 qp: -1
2025-11-19T14:00:35.115930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436111654949058:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:35.115940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.116191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114249366817:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11304
2025-11-19T14:00:35.116608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114249366818:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11304
2025-11-19T14:00:35.116643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114249366819:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11304
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.117161Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436114249366813:2115] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:35.117201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114249366820:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11304
2025-11-19T14:00:35.117382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436114249366765:2068] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.117395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:35.117400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436114249366765:2068] [node 5] ICP22 created new session: [6:7574436114249366821:2068]
2025-11-19T14:00:35.117406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436114249366821:2068] [node 5] ICS09 handshake done sender: [6:7574436114249366813:2115] self: [6:7574436114249366815:0] peer: [5:7574436113844179382:0] socket: 427 qp: -1
2025-11-19T14:00:35.117411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436114249366821:2068] [node 5] ICS10 traffic start
2025-11-19T14:00:35.117418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.118492Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436108236894496:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2431) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436112531861860:2108] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.120588Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436112650373319:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.120597Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436112650373319:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.121520Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436112531861870:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.121528Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436112531861870:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.122327Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436112650373319:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.118298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113844179430:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56305
2025-11-19T14:00:35.118358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113844179431:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56305
2025-11-19T14:00:35.120372Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436113844179430:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.120426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113844179432:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56305
2025-11-19T14:00:35.120461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436109549212049:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.121378Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436113844179432:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.121412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113844179434:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56305
2025-11-19T14:00:35.121435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436113844179366:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.122195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113844179436:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56305
2025-11-19T14:00:35.122497Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436113844179430:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:35.122570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436109549212049:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.122605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:35.122610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436109549212049:2048] [node 1] ICP22 created new session: [5:7574436113844179437:2048]
2025-11-19T14:00:35.122616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436113844179437:2048] [node 1] ICS09 handshake done sender: [5:7574436113844179430:2117] self: [5:7574436113844179433:0] peer: [1:7574436112650373316:0] socket: 435 qp: -1
2025-11-19T14:00:35.122622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436113844179437:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:35.122639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.123975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436113240560907:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
2025-11-19T14:00:35.124499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436113240560908:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
2025-11-19T14:00:35.124860Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436112531861870:2115] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:35.125032Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436113844179432:2119] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:35.125232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436112650373296:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.125247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:35.125253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436112650373296:2107] [node 5] ICP22 created new session: [1:7574436112650373333:2107]
2025-11-19T14:00:35.125259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436112650373333:2107] [node 5] ICS09 handshake done sender: [1:7574436112650373319:2117] self: [1:7574436112650373316:0] peer: [5:7574436113844179433:0] socket: 434 qp: -1
2025-11-19T14:00:35.125265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436112650373333:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:35.125278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:35.125610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436111654949031:2110] [node 4] ICP35 event processing took too much time PendingConnection:278(42054474ns) {HandleHandshakeStatus:329(42052691ns) {DropIncomingHandshake:441(4724ns) DropOutgoingHandshake:456(361ns) SwitchToState:178(68ns) ProcessPendingSessionEvents:495(41949523ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 41928659ns max# 41856582ns}}}
2025-11-19T14:00:35.125690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112531861874:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58184
2025-11-19T14:00:35.125722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436113844179366:2066] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.125738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:35.125745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436113844179366:2066] [node 4] ICP22 created new session: [5:7574436113844179440:2066]
2025-11-19T14:00:35.125752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436113844179440:2066] [node 4] ICS09 handshake done sender: [5:7574436113844179432:2119] self: [5:7574436113844179435:0] peer: [4:7574436112531861859:0] socket: 446 qp: -1
2025-11-19T14:00:35.125758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436113844179440:2066] [node 4] ICS10 traffic start
2025-11-19T14:00:35.125767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:35.125908Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436113240560893:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:35.126295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436108945593521:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.126314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:35.126319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436108945593521:2048] [node 1] ICP22 created new session: [3:7574436113240560909:2048]
2025-11-19T14:00:35.126325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436113240560909:2048] [node 1] ICS09 handshake done sender: [3:7574436113240560893:2115] self: [3:7574436113240560895:0] peer: [1:7574436112650373311:0] socket: 422 qp: -1
2025-11-19T14:00:35.126330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436113240560909:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:35.126345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:35.126677Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436114249366812:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:35.126623Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436113240560890:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:35.126957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436108945593522:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.126971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:35.126976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436108945593522:2049] [node 2] ICP22 created new session: [3:7574436113240560912:2049]
2025-11-19T14:00:35.126980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436113240560912:2049] [node 2] ICS09 handshake done sender: [3:7574436113240560890:2112] self: [3:7574436113240560898:0] peer: [2:7574436111654949028:0] socket: 412 qp: -1
2025-11-19T14:00:35.126986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436113240560912:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:35.127147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436109954399439:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.127162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:35.127167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436109954399439:2048] [node 1] ICP22 created new session: [6:7574436114249366824:2048]
2025-11-19T14:00:35.127173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436114249366824:2048] [node 1] ICS09 handshake done sender: [6:7574436114249366812:2114] self: [6:7574436114249366814:0] peer: [1:7574436112650373304:0] socket: 416 qp: -1
2025-11-19T14:00:35.127177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436114249366824:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:35.127193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:35.125848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112531861873:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58184
2025-11-19T14:00:35.129956Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436112531861820:2071] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:35.130032Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436112531861860:2108] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.130153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436108236894495:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.130164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:35.130170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436108236894495:2048] [node 1] ICP22 created new session: [4:7574436112531861879:2048]
2025-11-19T14:00:35.130175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436112531861879:2048] [node 1] ICS09 handshake done sender: [4:7574436112531861820:2071] self: [4:7574436112531861869:0] peer: [1:7574436112650373302:0] socket: 405 qp: -1
2025-11-19T14:00:35.130180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436112531861879:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:35.130185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:35.130367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436108236894496:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.130374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:35.130377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436108236894496:2049] [node 2] ICP22 created new session: [4:7574436112531861881:2049]
2025-11-19T14:00:35.130381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436112531861881:2049] [node 2] ICS09 handshake done sender: [4:7574436112531861860:2108] self: [4:7574436112531861861:0] peer: [2:7574436111654949033:0] socket: 406 qp: -1
2025-11-19T14:00:35.130385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436112531861881:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:35.130389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:35.130541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112531861877:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58184
2025-11-19T14:00:35.129273Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436111654949048:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.132765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114249366826:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11304
2025-11-19T14:00:35.126991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:35.133851Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436114249366819:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.133899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436109954399440:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.134185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436112531861829:2076] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.134200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:35.134205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436112531861829:2076] [node 5] ICP22 created new session: [4:7574436112531861883:2076]
2025-11-19T14:00:35.134210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436112531861883:2076] [node 5] ICS09 handshake done sender: [4:7574436112531861870:2115] self: [4:7574436112531861859:0] peer: [5:7574436113844179435:0] socket: 445 qp: -1
2025-11-19T14:00:35.134215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436112531861883:2076] [node 5] ICS10 traffic start
2025-11-19T14:00:35.134232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:35.134297Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436114249366826:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.140271Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436113240560885:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:35.140273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436114249366780:2078] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.140789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436107359981667:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(24787219ns) {HandleHandshakeStatus:329(24786042ns) {DropIncomingHandshake:441(255ns) DropOutgoingHandshake:456(406ns) SwitchToState:178(219ns) ProcessPendingSessionEvents:495(24696765ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16047677ns max# 15969997ns}}}
2025-11-19T14:00:35.140963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436107359981667:2049] [node 3] ICP18 handshake (actor [2:7574436111654949048:2120]) is too old
2025-11-19T14:00:35.141273Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436114249366820:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.141308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436109954399441:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.141770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574436113844179381:2073] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:35.141825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113844179442:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56305
2025-11-19T14:00:35.142057Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436113844179366:2066] [node 4] ICP35 event processing took too much time PendingConnection:278(16258959ns) {HandleHandshakeStatus:329(16258172ns) {DropIncomingHandshake:441(500ns) DropOutgoingHandshake:456(1062ns) SwitchToState:178(100ns) ProcessPendingSessionEvents:495(16159251ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16132309ns max# 16054134ns}}}
2025-11-19T14:00:35.143315Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436113844179431:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.143510Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436113844179401:2096] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:35.142310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436113240560883:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.142337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:35.142344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436113240560883:2110] [node 5] ICP22 created new session: [3:7574436113240560914:2110]
2025-11-19T14:00:35.142350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436113240560914:2110] [node 5] ICS09 handshake done sender: [3:7574436113240560885:2111] self: [3:7574436113240560884:0] peer: [5:7574436113844179425:0] socket: 409 qp: -1
2025-11-19T14:00:35.142357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436113240560914:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:35.142380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:35.142702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436112650373296:2107] [node 5] ICP35 event processing took too much time PendingConnection:278(17381345ns) {HandleHandshakeStatus:329(17380765ns) {DropIncomingHandshake:441(71ns) DropOutgoingHandshake:456(185ns) SwitchToState:178(62ns) ProcessPendingSessionEvents:495(17298518ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17267388ns max# 17191103ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.148716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436112650373335:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31497
2025-11-19T14:00:35.148908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436112650373336:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31497
2025-11-19T14:00:35.149041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574436112650373296:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:35.153565Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436113240560901:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.153580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436113240560901:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.154838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112531861885:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58184
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.154621Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436113240560901:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:35.155389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436113240560889:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.155416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:35.155425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436113240560889:2109] [node 6] ICP22 created new session: [3:7574436113240560918:2109]
2025-11-19T14:00:35.155432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436113240560918:2109] [node 6] ICS09 handshake done sender: [3:7574436113240560901:2117] self: [3:7574436113240560899:0] peer: [6:7574436114249366830:0] socket: 442 qp: -1
2025-11-19T14:00:35.155439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436113240560918:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:35.155449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:35.156089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436109549212051:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.156113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:35.156122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436109549212051:2050] [node 3] ICP22 created new session: [5:7574436113844179443:2050]
2025-11-19T14:00:35.156129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436113844179443:2050] [node 3] ICS09 handshake done sender: [5:7574436113844179401:2096] self: [5:7574436113844179425:0] peer: [3:7574436113240560884:0] socket: 410 qp: -1
2025-11-19T14:00:35.156135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436113844179443:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:35.156142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:35.157252Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436111654949045:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.157272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436111654949045:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.157778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436109954399441:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436114249366820:2120] held: no
2025-11-19T14:00:35.158055Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436112531861885:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.158198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112531861886:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58184
2025-11-19T14:00:35.161037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436109549212050:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.161192Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7574436109549212051:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:35.160082Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436112531861872:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.160099Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436112531861872:2116] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.163912Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436112531861872:2116] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:35.176604Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436112531861886:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.176701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436108236894497:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.177083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436112531861848:2083] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.177124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:35.177133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436112531861848:2083] [node 6] ICP22 created new session: [4:7574436112531861890:2083]
2025-11-19T14:00:35.177142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436112531861890:2083] [node 6] ICS09 handshake done sender: [4:7574436112531861872:2116] self: [4:7574436112531861871:0] peer: [6:7574436114249366829:0] socket: 453 qp: -1
2025-11-19T14:00:35.177151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436112531861890:2083] [node 6] ICS10 traffic start
2025-11-19T14:00:35.177164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.179109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114249366831:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11304
2025-11-19T14:00:35.179175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114249366832:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11304
2025-11-19T14:00:35.179445Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436114249366826:2121] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:35.179834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436113240560883:2110] [node 5] ICP35 event processing took too much time PendingConnection:278(37382750ns) {HandleHandshakeStatus:329(37381804ns) {DropIncomingHandshake:441(6647ns) DropOutgoingHandshake:456(491ns) SwitchToState:178(74ns) ProcessPendingSessionEvents:495(19712471ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19686746ns max# 19595307ns}}}
2025-11-19T14:00:35.180770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436114249366780:2078] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.180798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:35.180807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436114249366780:2078] [node 4] ICP22 created new session: [6:7574436114249366833:2078]
2025-11-19T14:00:35.180813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436114249366833:2078] [node 4] ICS09 handshake done sender: [6:7574436114249366826:2121] self: [6:7574436114249366829:0] peer: [4:7574436112531861871:0] socket: 454 qp: -1
2025-11-19T14:00:35.180821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436114249366833:2078] [node 4] ICS10 traffic start
2025-11-19T14:00:35.180848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:35.182576Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436111654949036:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.182598Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436111654949036:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.182846Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436113240560902:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.182860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436113240560902:2118] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.184087Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436114249366820:2120] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:35.184335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436109954399441:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.184358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:35.184368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436109954399441:2050] [node 3] ICP22 created new session: [6:7574436114249366835:2050]
2025-11-19T14:00:35.184376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436114249366835:2050] [node 3] ICS09 handshake done sender: [6:7574436114249366820:2120] self: [6:7574436114249366830:0] peer: [3:7574436113240560899:0] socket: 443 qp: -1
2025-11-19T14:00:35.184386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436114249366835:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:35.184394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:35.186062Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436111654949045:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.186470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114249366837:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11304
2025-11-19T14:00:35.186753Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436111654949036:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:35.186610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113844179446:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56305
2025-11-19T14:00:35.187849Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436113240560902:2118] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:35.188217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436113240560888:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.188239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:35.188248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436113240560888:2108] [node 4] ICP22 created new session: [3:7574436113240560921:2108]
2025-11-19T14:00:35.188254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436113240560921:2108] [node 4] ICS09 handshake done sender: [3:7574436113240560902:2118] self: [3:7574436113240560900:0] peer: [4:7574436112531861889:0] socket: 439 qp: -1
2025-11-19T14:00:35.188261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436113240560921:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:35.188269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:35.188779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436111654949032:2112] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.188800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:35.188805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436111654949032:2112] [node 5] ICP22 created new session: [2:7574436111654949063:2112]
2025-11-19T14:00:35.188810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436111654949063:2112] [node 5] ICS09 handshake done sender: [2:7574436111654949036:2114] self: [2:7574436111654949035:0] peer: [5:7574436113844179445:0] socket: 400 qp: -1
2025-11-19T14:00:35.188815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436111654949063:2112] [node 5] ICS10 traffic start
2025-11-19T14:00:35.188824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.189375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574436112531861829:2076] [node 5] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:35.189419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112531861892:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58184
2025-11-19T14:00:35.189578Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436114249366819:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:35.189721Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436113844179431:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:35.189991Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436112531861886:2121] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:35.190060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436109954399440:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.190079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:35.190086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436109954399440:2049] [node 2] ICP22 created new session: [6:7574436114249366838:2049]
2025-11-19T14:00:35.190092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436114249366838:2049] [node 2] ICS09 handshake done sender: [6:7574436114249366819:2119] self: [6:7574436114249366828:0] peer: [2:7574436111654949043:0] socket: 441 qp: -1
2025-11-19T14:00:35.190098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436114249366838:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:35.190105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:35.191303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436109549212050:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.191324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:35.191330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436109549212050:2049] [node 2] ICP22 created new session: [5:7574436113844179447:2049]
2025-11-19T14:00:35.191335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436113844179447:2049] [node 2] ICS09 handshake done sender: [5:7574436113844179431:2118] self: [5:7574436113844179445:0] peer: [2:7574436111654949035:0] socket: 444 qp: -1
2025-11-19T14:00:35.191341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436113844179447:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:35.191349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:35.193694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436108236894497:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.193722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:35.193728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436108236894497:2050] [node 3] ICP22 created new session: [4:7574436112531861893:2050]
2025-11-19T14:00:35.193734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436112531861893:2050] [node 3] ICS09 handshake done sender: [4:7574436112531861886:2121] self: [4:7574436112531861889:0] peer: [3:7574436113240560900:0] socket: 456 qp: -1
2025-11-19T14:00:35.193741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436112531861893:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:35.193749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:35.196201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436111654949040:2115] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.196230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:35.196236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436111654949040:2115] [node 6] ICP22 created new session: [2:7574436111654949065:2115]
2025-11-19T14:00:35.196242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436111654949065:2115] [node 6] ICS09 handshake done sender: [2:7574436111654949045:2117] self: [2:7574436111654949043:0] peer: [6:7574436114249366828:0] socket: 404 qp: -1
2025-11-19T14:00:35.196247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436111654949065:2115] [node 6] ICS10 traffic start
2025-11-19T14:00:35.196258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:35.216337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:35.216449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436111654949049:2048] [node 1] ICS01 socket: 411 reason# EndOfStream
2025-11-19T14:00:35.216460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436107359981666:2048] [node 1] ICP30 unregister session Session# [2:7574436111654949049:2048] VirtualId# [2:7574436111654949042:0]
2025-11-19T14:00:35.216469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436111654949049:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.216506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:35.216529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:35.216956Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:35.217025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436113240560909:2048] [node 1] ICS01 socket: 422 reason# EndOfStream
2025-11-19T14:00:35.217036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436108945593521:2048] [node 1] ICP30 unregister session Session# [3:7574436113240560909:2048] VirtualId# [3:7574436113240560895:0]
2025-11-19T14:00:35.217041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436113240560909:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.217064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:35.217083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:35.248867Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:35.248979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436114249366824:2048] [node 1] ICS07 socket disconnect 416 reason# EndOfStream
2025-11-19T14:00:35.248989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436114249366824:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.249039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:35.249061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436114249366824:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:35.249461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:35.249525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436112531861879:2048] [node 1] ICS01 socket: 405 reason# EndOfStream
2025-11-19T14:00:35.249531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436108236894495:2048] [node 1] ICP30 unregister session Session# [4:7574436112531861879:2048] VirtualId# [4:7574436112531861869:0]
2025-11-19T14:00:35.249537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436112531861879:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.249566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:35.249590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:35.250896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436109954399439:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2427) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:35.250915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436114249366824:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:35.250920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436109954399439:2048] [node 1] ICP30 unregister session Session# [6:7574436114249366824:2048] VirtualId# [6:7574436114249366814:0]
2025-11-19T14:00:35.250984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:35.250990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436109954399439:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2427) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:35.250994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2427) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:35.255514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:35.255614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436113844179437:2048] [node 1] ICS01 socket: 435 reason# EndOfStream
2025-11-19T14:00:35.255622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436109549212049:2048] [node 1] ICP30 unregister session Session# [5:7574436113844179437:2048] VirtualId# [5:7574436113844179433:0]
2025-11-19T14:00:35.255629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436113844179437:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.255683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:35.255708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:35.259812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436109954399439:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:35.277338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:35.277456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436113240560912:2049] [node 2] ICS01 socket: 412 reason# EndOfStream
2025-11-19T14:00:35.277469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436108945593522:2049] [node 2] ICP30 unregister session Session# [3:7574436113240560912:2049] VirtualId# [3:7574436113240560898:0]
2025-11-19T14:00:35.277478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436113240560912:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.277542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:35.277567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:35.315337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:35.315408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436113844179447:2049] [node 2] ICS01 socket: 444 reason# EndOfStream
2025-11-19T14:00:35.315414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436109549212050:2049] [node 2] ICP30 unregister session Session# [5:7574436113844179447:2049] VirtualId# [5:7574436113844179445:0]
2025-11-19T14:00:35.315419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436113844179447:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.315447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:35.315475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:35.315052Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:35.315183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436112531861881:2049] [node 2] ICS01 socket: 406 reason# EndOfStream
2025-11-19T14:00:35.315195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436108236894496:2049] [node 2] ICP30 unregister session Session# [4:7574436112531861881:2049] VirtualId# [4:7574436112531861861:0]
2025-11-19T14:00:35.315205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436112531861881:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.315262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:35.315291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:35.315510Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:35.315577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436114249366838:2049] [node 2] ICS01 socket: 441 reason# EndOfStream
2025-11-19T14:00:35.315582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436109954399440:2049] [node 2] ICP30 unregister session Session# [6:7574436114249366838:2049] VirtualId# [6:7574436114249366828:0]
2025-11-19T14:00:35.315587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436114249366838:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.315614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:35.315638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:35.352277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:35.352391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436114249366835:2050] [node 3] ICS01 socket: 443 reason# EndOfStream
2025-11-19T14:00:35.352402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436109954399441:2050] [node 3] ICP30 unregister session Session# [6:7574436114249366835:2050] VirtualId# [6:7574436114249366830:0]
2025-11-19T14:00:35.352410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436114249366835:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.352471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:35.352497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:35.352588Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:35.352648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436113844179443:2050] [node 3] ICS01 socket: 410 reason# EndOfStream
2025-11-19T14:00:35.352653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436109549212051:2050] [node 3] ICP30 unregister session Session# [5:7574436113844179443:2050] VirtualId# [5:7574436113844179425:0]
2025-11-19T14:00:35.352658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436113844179443:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.352684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:35.352704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:35.383844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:35.383978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436114249366833:2078] [node 4] ICS01 socket: 454 reason# EndOfStream
2025-11-19T14:00:35.383992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436114249366780:2078] [node 4] ICP30 unregister session Session# [6:7574436114249366833:2078] VirtualId# [6:7574436114249366829:0]
2025-11-19T14:00:35.384001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436114249366833:2078] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.384074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:35.384122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:35.517100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436114704500645:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:35.517126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436114704500646:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:35.538121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436112803453624:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:35.538150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436112803453625:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:35.563275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436114506237158:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:35.563301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436114506237159:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:35.594513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436112254688775:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:35.594533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436112254688776:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:35.594539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436112254688777:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:35.603908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436113386890933:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:35.603924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436113386890934:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:35.603930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436113386890935:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:35.619068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436114571057936:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:35.619083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436114571057937:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:35.619089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436114571057938:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.630457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436114506237177:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44058
2025-11-19T14:00:35.630558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436114506237178:2067] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44058
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.630951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114571057947:2058] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35873
2025-11-19T14:00:35.631526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436112254688791:2062] [node 6] ICP01 ready to work
2025-11-19T14:00:35.631578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436113386890951:2062] [node 4] ICP01 ready to work
2025-11-19T14:00:35.631664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436112803453656:2075] [node 4] ICP01 ready to work
2025-11-19T14:00:35.629693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436114704500683:2078] [node 5] ICP01 ready to work
2025-11-19T14:00:35.629795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436114704500684:2082] [node 6] ICP01 ready to work
2025-11-19T14:00:35.632888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436114506237184:2068] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.633539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113386890954:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58388
2025-11-19T14:00:35.633774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436114704500698:2090] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.655924Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436113386890933:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(20958717ns) {Configure:104(20957236ns) {ProcessConfigured:135(20670423ns) {StartInitialHandshake:157(20668580ns) {DropHandshakes:469(361ns) {DropIncomingHandshake:441(81ns) DropOutgoingHandshake:456(35ns)} GenerateSessionVirtualId:988(277ns) PrepareNewSessionHandshake:476(2733ns) {SwitchToState:178(1296ns)}}}}}
2025-11-19T14:00:35.656191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436113386890983:2069] [node 6] ICP01 ready to work
2025-11-19T14:00:35.659029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436112254688791:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(27432882ns) {Configure:104(27431917ns) {ProcessConfigured:135(25770384ns) {StartInitialHandshake:157(25769946ns) {DropHandshakes:469(710ns) {DropIncomingHandshake:441(247ns) DropOutgoingHandshake:456(73ns)} GenerateSessionVirtualId:988(233ns) PrepareNewSessionHandshake:476(1616ns) {SwitchToState:178(951ns)}}}}}
2025-11-19T14:00:35.659201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436112254688826:2064] [node 5] ICP01 ready to work
2025-11-19T14:00:35.660913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112254688827:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22293
2025-11-19T14:00:35.661385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436114506237184:2068] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(28401011ns) {Configure:104(28400701ns) {ProcessConfigured:135(28008885ns) {StartInitialHandshake:157(28008634ns) {DropHandshakes:469(413ns) {DropIncomingHandshake:441(68ns) DropOutgoingHandshake:456(59ns)} GenerateSessionVirtualId:988(259ns) PrepareNewSessionHandshake:476(880ns) {SwitchToState:178(403ns)}}}}}
2025-11-19T14:00:35.661969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436114506237199:2073] [node 5] ICP01 ready to work
2025-11-19T14:00:35.662137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436114506237200:2080] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.677034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436114704500721:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40271
2025-11-19T14:00:35.677060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436114704500722:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40271
2025-11-19T14:00:35.677072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436114704500723:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40271
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.682053Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436113386890954:2064] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.684751Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436114506237178:2067] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.685060Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436114704500723:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.684808Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436114506237177:2066] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.684915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436114506237158:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.688053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113386891010:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58388
2025-11-19T14:00:35.692323Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436114571057937:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(56940400ns) {Configure:104(56938669ns) {ProcessConfigured:135(56569477ns) {StartInitialHandshake:157(56568452ns) {DropHandshakes:469(548ns) {DropIncomingHandshake:441(83ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:476(2191ns) {SwitchToState:178(1403ns)}}}}}
2025-11-19T14:00:35.692531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436114571058002:2061] [node 5] ICP01 ready to work
2025-11-19T14:00:35.692752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436114571058003:2064] [node 4] ICP01 ready to work
2025-11-19T14:00:35.693018Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436114704500682:2083] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.693039Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436114704500682:2083] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.693121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113386891011:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58388
2025-11-19T14:00:35.693187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436113386890933:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.694167Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436113386891011:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.694201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436113386890935:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.699958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436112803453656:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(59876352ns) {Configure:104(59875037ns) {ProcessConfigured:135(59493264ns) {StartInitialHandshake:157(59492278ns) {DropHandshakes:469(885ns) {DropIncomingHandshake:441(87ns) DropOutgoingHandshake:456(47ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:476(2101ns) {SwitchToState:178(1074ns)}}}}}
2025-11-19T14:00:35.700175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436112803453694:2079] [node 6] ICP01 ready to work
2025-11-19T14:00:35.700402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436112803453695:2085] [node 5] ICP01 ready to work
2025-11-19T14:00:35.701140Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436114506237202:2085] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.701156Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436114506237202:2085] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.703237Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436114704500721:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.703279Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436114704500687:2085] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.703283Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436114704500687:2085] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.702429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114571058004:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35873
2025-11-19T14:00:35.702450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114571058005:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35873
2025-11-19T14:00:35.704033Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436114704500682:2083] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:35.705684Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436112254688827:2096] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.710458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436112803453701:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.711195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436114506237239:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44058
2025-11-19T14:00:35.711216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436114506237240:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44058
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.711508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436114506237241:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44058
2025-11-19T14:00:35.711567Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436114506237202:2085] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.711681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436114506237199:2073] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.711704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:35.711710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436114506237199:2073] [node 5] ICP22 created new session: [3:7574436114506237243:2073]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.711715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436114506237243:2073] [node 5] ICS09 handshake done sender: [3:7574436114506237202:2085] self: [3:7574436114506237201:0] peer: [5:7574436113386891015:0] socket: 412 qp: -1
2025-11-19T14:00:35.711725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436114506237243:2073] [node 5] ICS10 traffic start
2025-11-19T14:00:35.711764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.02025-11-19T14:00:35.712037Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436114506237178:2067] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:35.712081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113386891016:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58388
2025-11-19T14:00:35.712104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113386891017:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58388
2025-11-19T14:00:35.712135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113386891018:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58388
2025-11-19T14:00:35.712341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436114704500646:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.712365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:35.712370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436114704500646:2049] [node 3] ICP22 created new session: [1:7574436114704500732:2049]
2025-11-19T14:00:35.712375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436114704500732:2049] [node 3] ICS09 handshake done sender: [1:7574436114704500682:2083] self: [1:7574436114704500681:0] peer: [3:7574436114506237225:0] socket: 398 qp: -1
2025-11-19T14:00:35.712380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436114704500732:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:35.712459Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436113386891011:2115] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:35.712388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:35.712886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436113386890935:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.712904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:35.712910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436113386890935:2050] [node 3] ICP22 created new session: [5:7574436113386891019:2050]
2025-11-19T14:00:35.712917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436113386891019:2050] [node 3] ICS09 handshake done sender: [5:7574436113386891011:2115] self: [5:7574436113386891015:0] peer: [3:7574436114506237201:0] socket: 417 qp: -1
2025-11-19T14:00:35.712922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436113386891019:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:35.712928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:35.715459Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436113386890954:2064] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:35.712597Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436114571057947:2058] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.712674Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436114571058005:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.713874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436114571057936:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.713890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436114571057938:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.712790Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436114704500687:2085] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:35.715561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436113386890933:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.715577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:35.715582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436113386890933:2048] [node 1] ICP22 created new session: [5:7574436113386891021:2048]
2025-11-19T14:00:35.715587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436113386891021:2048] [node 1] ICS09 handshake done sender: [5:7574436113386890954:2064] self: [5:7574436113386891014:0] peer: [1:7574436114704500686:0] socket: 403 qp: -1
2025-11-19T14:00:35.715593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436113386891021:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:35.715600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:35.740053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436114506237158:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.740088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:35.740102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436114506237158:2048] [node 1] ICP22 created new session: [3:7574436114506237247:2048]
2025-11-19T14:00:35.740112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436114506237247:2048] [node 1] ICS09 handshake done sender: [3:7574436114506237178:2067] self: [3:7574436114506237225:0] peer: [1:7574436114704500681:0] socket: 399 qp: -1
2025-11-19T14:00:35.740122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436114506237247:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:35.740137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.746213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436114704500683:2078] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.746258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:35.746267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436114704500683:2078] [node 5] ICP22 created new session: [1:7574436114704500734:2078]
2025-11-19T14:00:35.746277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436114704500734:2078] [node 5] ICS09 handshake done sender: [1:7574436114704500687:2085] self: [1:7574436114704500686:0] peer: [5:7574436113386891014:0] socket: 400 qp: -1
2025-11-19T14:00:35.746286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436114704500734:2078] [node 5] ICS10 traffic start
2025-11-19T14:00:35.746302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:35.746834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436114704500731:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40271
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.759137Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436112803453701:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.759233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436112803453704:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
2025-11-19T14:00:35.759266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436112803453705:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
2025-11-19T14:00:35.759544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112254688855:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22293
2025-11-19T14:00:35.759577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112254688856:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22293
2025-11-19T14:00:35.760014Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436114704500690:2087] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.760029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436114704500690:2087] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.760495Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436114704500722:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.762772Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436114704500690:2087] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.763140Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436114571058004:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.763202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114571058019:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35873
2025-11-19T14:00:35.763231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114571058020:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35873
2025-11-19T14:00:35.763449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436114571058003:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.763510Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436114571057947:2058] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:35.763574Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436112254688825:2095] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.763585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436112254688825:2095] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.764272Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436112254688825:2095] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:35.764568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574436114506237158:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:35.764963Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436114506237196:2083] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.764966Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436114506237196:2083] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.766310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436112254688791:2062] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.766329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:35.766337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436112254688791:2062] [node 6] ICP22 created new session: [4:7574436112254688859:2062]
2025-11-19T14:00:35.766343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436112254688859:2062] [node 6] ICS09 handshake done sender: [4:7574436112254688825:2095] self: [4:7574436112254688802:0] peer: [6:7574436114571058021:0] socket: 407 qp: -1
2025-11-19T14:00:35.766351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436112254688859:2062] [node 6] ICS10 traffic start
2025-11-19T14:00:35.766360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:35.766627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112254688858:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22293
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.767074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436114571057936:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.767092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:35.767097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436114571057936:2048] [node 1] ICP22 created new session: [6:7574436114571058024:2048]
2025-11-19T14:00:35.767102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436114571058024:2048] [node 1] ICS09 handshake done sender: [6:7574436114571057947:2058] self: [6:7574436114571058017:0] peer: [1:7574436114704500689:0] socket: 402 qp: -1
2025-11-19T14:00:35.767107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436114571058024:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:35.767113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:35.767452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114571058022:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35873
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.768929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114571058023:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35873
2025-11-19T14:00:35.768981Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436114571058004:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:35.772285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436114704500684:2082] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.772316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:35.772326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436114704500684:2082] [node 6] ICP22 created new session: [1:7574436114704500740:2082]
2025-11-19T14:00:35.772333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436114704500740:2082] [node 6] ICS09 handshake done sender: [1:7574436114704500690:2087] self: [1:7574436114704500689:0] peer: [6:7574436114571058017:0] socket: 401 qp: -1
2025-11-19T14:00:35.772339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436114704500740:2082] [node 6] ICS10 traffic start
2025-11-19T14:00:35.772350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:35.773341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436114571058003:2064] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.773357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:35.773362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436114571058003:2064] [node 4] ICP22 created new session: [6:7574436114571058027:2064]
2025-11-19T14:00:35.773368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436114571058027:2064] [node 4] ICS09 handshake done sender: [6:7574436114571058004:2112] self: [6:7574436114571058021:0] peer: [4:7574436112254688802:0] socket: 418 qp: -1
2025-11-19T14:00:35.773372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436114571058027:2064] [node 4] ICS10 traffic start
2025-11-19T14:00:35.773377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:35.774110Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436113386891010:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.774137Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436113386891016:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.774202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436113386890951:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.774455Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436114506237196:2083] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:35.774597Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436114506237239:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.774615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436114506237184:2068] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.774623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:35.774629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436114506237184:2068] [node 6] ICP22 created new session: [3:7574436114506237252:2068]
2025-11-19T14:00:35.774634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436114506237252:2068] [node 6] ICS09 handshake done sender: [3:7574436114506237196:2083] self: [3:7574436114506237185:0] peer: [6:7574436114571058018:0] socket: 410 qp: -1
2025-11-19T14:00:35.774638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436114506237252:2068] [node 6] ICS10 traffic start
2025-11-19T14:00:35.774647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:35.776105Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436114571058019:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.776482Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436112254688856:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.777941Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436114506237241:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.777982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574436114506237184:2068] [node 6] ICP18 handshake (actor [3:7574436114506237239:2115]) is too old
2025-11-19T14:00:35.778376Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436114704500731:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.778633Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436114571058005:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.778801Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436112803453704:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.778834Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436112803453705:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.778895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436112803453707:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
2025-11-19T14:00:35.778928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436112803453708:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
2025-11-19T14:00:35.778971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436112803453624:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.779299Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436114704500700:2092] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.779308Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436114704500700:2092] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.780060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436112803453711:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48501
2025-11-19T14:00:35.780176Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436114704500700:2092] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:35.780402Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436112254688829:2097] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.780415Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436112254688829:2097] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.780381Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436112803453705:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.780993Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436112254688855:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.781275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436112254688775:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.781287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436112254688776:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.781307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112254688862:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22293
2025-11-19T14:00:35.783811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436114704500645:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.783842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:35.783851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436114704500645:2048] [node 2] ICP22 created new session: [1:7574436114704500744:2048]
2025-11-19T14:00:35.783859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436114704500744:2048] [node 2] ICS09 handshake done sender: [1:7574436114704500700:2092] self: [1:7574436114704500699:0] peer: [2:7574436112803453710:0] socket: 413 qp: -1
2025-11-19T14:00:35.783867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436114704500744:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:35.783878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.784435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436112803453624:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.784451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:35.784456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436112803453624:2048] [node 1] ICP22 created new session: [2:7574436112803453712:2048]
2025-11-19T14:00:35.784461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436112803453712:2048] [node 1] ICS09 handshake done sender: [2:7574436112803453705:2117] self: [2:7574436112803453710:0] peer: [1:7574436114704500699:0] socket: 438 qp: -1
2025-11-19T14:00:35.784467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436112803453712:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:35.784474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:35.784813Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436112803453708:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.785620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113386891027:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58388
2025-11-19T14:00:35.785702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436114571058002:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.785776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436114571057938:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.785784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:35.785788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436114571057938:2050] [node 3] ICP22 created new session: [6:7574436114571058030:2050]
2025-11-19T14:00:35.785793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436114571058030:2050] [node 3] ICS09 handshake done sender: [6:7574436114571058005:2113] self: [6:7574436114571058018:0] peer: [3:7574436114506237185:0] socket: 419 qp: -1
2025-11-19T14:00:35.785798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436114571058030:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:35.785806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:35.787331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436114506237159:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.787658Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436112254688829:2097] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:35.787892Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436113386891010:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:35.787971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436113386890951:2062] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.787984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:35.787990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436113386890951:2062] [node 4] ICP22 created new session: [5:7574436113386891028:2062]
2025-11-19T14:00:35.787996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436113386891028:2062] [node 4] ICS09 handshake done sender: [5:7574436113386891010:2114] self: [5:7574436113386891025:0] peer: [4:7574436112254688828:0] socket: 414 qp: -1
2025-11-19T14:00:35.788002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436113386891028:2062] [node 4] ICS10 traffic start
2025-11-19T14:00:35.788010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:35.789166Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436112254688858:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.789181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436112254688826:2064] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.789193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:35.789198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436112254688826:2064] [node 5] ICP22 created new session: [4:7574436112254688866:2064]
2025-11-19T14:00:35.789202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436112254688866:2064] [node 5] ICS09 handshake done sender: [4:7574436112254688829:2097] self: [4:7574436112254688828:0] peer: [5:7574436113386891025:0] socket: 408 qp: -1
2025-11-19T14:00:35.789207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436112254688866:2064] [node 5] ICS10 traffic start
2025-11-19T14:00:35.789213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:35.790014Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436114704500702:2093] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.790028Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436114704500702:2093] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.790961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436112254688777:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.791073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112254688868:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22293
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.791201Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436114506237206:2087] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.791212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436114506237206:2087] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.791801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436114506237255:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44058
2025-11-19T14:00:35.792179Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436112803453707:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.792216Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436112803453655:2077] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.792218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436112803453655:2077] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.792812Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436112803453693:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.792817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436112803453693:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.793976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574436114571057938:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:35.794107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114571058032:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35873
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.795290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112254688871:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22293
2025-11-19T14:00:35.795306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436112254688872:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22293
2025-11-19T14:00:35.795447Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436114506237206:2087] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:35.795657Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436113386890989:2095] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.795668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436113386890989:2095] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.796790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436114506237200:2080] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.796806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:35.796812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436114506237200:2080] [node 4] ICP22 created new session: [3:7574436114506237258:2080]
2025-11-19T14:00:35.796818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436114506237258:2080] [node 4] ICS09 handshake done sender: [3:7574436114506237206:2087] self: [3:7574436114506237205:0] peer: [4:7574436112254688870:0] socket: 445 qp: -1
2025-11-19T14:00:35.796824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436114506237258:2080] [node 4] ICS10 traffic start
2025-11-19T14:00:35.796833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:35.797074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436114506237257:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44058
2025-11-19T14:00:35.797200Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436114704500702:2093] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:35.797316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436114704500698:2090] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.797333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:35.797338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436114704500698:2090] [node 4] ICP22 created new session: [1:7574436114704500747:2090]
2025-11-19T14:00:35.797343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436114704500747:2090] [node 4] ICS09 handshake done sender: [1:7574436114704500702:2093] self: [1:7574436114704500701:0] peer: [4:7574436112254688864:0] socket: 416 qp: -1
2025-11-19T14:00:35.797347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436114704500747:2090] [node 4] ICS10 traffic start
2025-11-19T14:00:35.797356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.797792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114571058033:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35873
2025-11-19T14:00:35.798000Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436113386890989:2095] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:35.798160Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436114571058019:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:35.798216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436114571058002:2061] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.798229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:35.798234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436114571058002:2061] [node 5] ICP22 created new session: [6:7574436114571058034:2061]
2025-11-19T14:00:35.798239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436114571058034:2061] [node 5] ICS09 handshake done sender: [6:7574436114571058019:2118] self: [6:7574436114571058029:0] peer: [5:7574436113386890988:0] socket: 441 qp: -1
2025-11-19T14:00:35.798244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436114571058034:2061] [node 5] ICS10 traffic start
2025-11-19T14:00:35.798250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:35.799377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436113386890983:2069] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.799396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:35.799402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436113386890983:2069] [node 6] ICP22 created new session: [5:7574436113386891031:2069]
2025-11-19T14:00:35.799407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436113386891031:2069] [node 6] ICS09 handshake done sender: [5:7574436113386890989:2095] self: [5:7574436113386890988:0] peer: [6:7574436114571058029:0] socket: 397 qp: -1
2025-11-19T14:00:35.799412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436113386891031:2069] [node 6] ICS10 traffic start
2025-11-19T14:00:35.799418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:35.800169Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436114506237255:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.800204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574436114506237200:2080] [node 4] ICP18 handshake (actor [3:7574436114506237255:2120]) is too old
2025-11-19T14:00:35.801594Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436112803453655:2077] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:35.801838Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436114506237241:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:35.801926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436114506237159:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.801942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:35.801947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436114506237159:2049] [node 2] ICP22 created new session: [3:7574436114506237260:2049]
2025-11-19T14:00:35.801953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436114506237260:2049] [node 2] ICS09 handshake done sender: [3:7574436114506237241:2117] self: [3:7574436114506237254:0] peer: [2:7574436112803453654:0] socket: 432 qp: -1
2025-11-19T14:00:35.801958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436114506237260:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:35.801969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:35.803803Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436112803453693:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:35.804208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436112803453625:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.804234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:35.804240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436112803453625:2049] [node 3] ICP22 created new session: [2:7574436112803453719:2049]
2025-11-19T14:00:35.804246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436112803453719:2049] [node 3] ICS09 handshake done sender: [2:7574436112803453655:2077] self: [2:7574436112803453654:0] peer: [3:7574436114506237254:0] socket: 426 qp: -1
2025-11-19T14:00:35.804251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436112803453719:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:35.804259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:35.804661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436112803453656:2075] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.804674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:35.804678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436112803453656:2075] [node 4] ICP22 created new session: [2:7574436112803453721:2075]
2025-11-19T14:00:35.804682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436112803453721:2075] [node 4] ICS09 handshake done sender: [2:7574436112803453693:2112] self: [2:7574436112803453667:0] peer: [4:7574436112254688865:0] socket: 429 qp: -1
2025-11-19T14:00:35.804686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436112803453721:2075] [node 4] ICS10 traffic start
2025-11-19T14:00:35.804694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.808205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113386891033:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58388
2025-11-19T14:00:35.809997Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436114571058032:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.810042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436114571057937:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.810291Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436112803453698:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.810301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436112803453698:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.810843Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436113386891033:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.810895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436113386890934:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:35.811036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436114571058037:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35873
2025-11-19T14:00:35.811163Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436112803453698:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:35.811342Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436114571058032:2122] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:35.811394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436114571057937:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.811415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:35.811421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436114571057937:2049] [node 2] ICP22 created new session: [6:7574436114571058038:2049]
2025-11-19T14:00:35.811428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436114571058038:2049] [node 2] ICS09 handshake done sender: [6:7574436114571058032:2122] self: [6:7574436114571058036:0] peer: [2:7574436112803453697:0] socket: 422 qp: -1
2025-11-19T14:00:35.811434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436114571058038:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:35.811443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:35.812506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436112254688775:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20381) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436112254688856:2117] held: no
2025-11-19T14:00:35.812773Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436112254688858:2118] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:35.812836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436112254688777:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6828) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436112254688858:2118] held: no
2025-11-19T14:00:35.812925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436112254688777:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.812935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:35.812940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436112254688777:2050] [node 3] ICP22 created new session: [4:7574436112254688873:2050]
2025-11-19T14:00:35.812946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436112254688873:2050] [node 3] ICS09 handshake done sender: [4:7574436112254688858:2118] self: [4:7574436112254688870:0] peer: [3:7574436114506237205:0] socket: 446 qp: -1
2025-11-19T14:00:35.812952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436112254688873:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:35.812960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:35.814257Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436112803453700:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:35.814268Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436112803453700:2114] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:35.814898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436112803453694:2079] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.814912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:35.814923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436112803453694:2079] [node 6] ICP22 created new session: [2:7574436112803453725:2079]
2025-11-19T14:00:35.814930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436112803453725:2079] [node 6] ICS09 handshake done sender: [2:7574436112803453698:2113] self: [2:7574436112803453697:0] peer: [6:7574436114571058036:0] socket: 459 qp: -1
2025-11-19T14:00:35.814935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436112803453725:2079] [node 6] ICS10 traffic start
2025-11-19T14:00:35.814943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:35.816353Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436112254688856:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:35.816556Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436112254688776:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30141) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436112254688855:2116] held: no
2025-11-19T14:00:35.816865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436112254688775:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.816876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:35.816883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436112254688775:2048] [node 1] ICP22 created new session: [4:7574436112254688875:2048]
2025-11-19T14:00:35.816890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436112254688875:2048] [node 1] ICS09 handshake done sender: [4:7574436112254688856:2117] self: [4:7574436112254688864:0] peer: [1:7574436114704500701:0] socket: 439 qp: -1
2025-11-19T14:00:35.816898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436112254688875:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:35.816905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:35.817116Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436112254688855:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:35.817629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436112254688776:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.817641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:35.817645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436112254688776:2049] [node 2] ICP22 created new session: [4:7574436112254688877:2049]
2025-11-19T14:00:35.817650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436112254688877:2049] [node 2] ICS09 handshake done sender: [4:7574436112254688855:2116] self: [4:7574436112254688865:0] peer: [2:7574436112803453667:0] socket: 396 qp: -1
2025-11-19T14:00:35.817654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436112254688877:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:35.817660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:35.818454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436113386891035:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58388
2025-11-19T14:00:35.818600Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436112803453700:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:35.818772Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436113386891033:2122] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:35.818831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436113386890934:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:35.818845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:35.818850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436113386890934:2049] [node 2] ICP22 created new session: [5:7574436113386891036:2049]
2025-11-19T14:00:35.818856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436113386891036:2049] [node 2] ICS09 handshake done sender: [5:7574436113386891033:2122] self: [5:7574436113386891034:0] peer: [2:7574436112803453699:0] socket: 411 qp: -1
2025-11-19T14:00:35.818861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436113386891036:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:35.818868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:35.823912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436112803453695:2085] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:35.823938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:35.823945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436112803453695:2085] [node 5] ICP22 created new session: [2:7574436112803453728:2085]
2025-11-19T14:00:35.823952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436112803453728:2085] [node 5] ICS09 handshake done sender: [2:7574436112803453700:2114] self: [2:7574436112803453699:0] peer: [5:7574436113386891034:0] socket: 409 qp: -1
2025-11-19T14:00:35.823959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436112803453728:2085] [node 5] ICS10 traffic start
2025-11-19T14:00:35.823971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:35.829157Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:35.829257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436112803453712:2048] [node 1] ICS01 socket: 438 reason# EndOfStream
2025-11-19T14:00:35.829265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436112803453624:2048] [node 1] ICP30 unregister session Session# [2:7574436112803453712:2048] VirtualId# [2:7574436112803453710:0]
2025-11-19T14:00:35.829272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436112803453712:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.829302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:35.829322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:35.831065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:35.831142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574436114506237247:2048] [node 1] ICS07 socket disconnect 399 reason# EndOfStream
2025-11-19T14:00:35.831149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436114506237247:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.831176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:35.831193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436114506237247:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:35.833628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436114506237158:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20381) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T14:00:35.833642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436114506237247:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:35.833996Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436114506237158:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20381) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:35.834010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436114506237247:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:35.834015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436114506237158:2048] [node 1] ICP30 unregister session Session# [3:7574436114506237247:2048] VirtualId# [3:7574436114506237225:0]
2025-11-19T14:00:35.834085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:35.834092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436114506237158:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20381) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:35.834095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20381) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:35.853790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:35.853905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436113386891021:2048] [node 1] ICS01 socket: 403 reason# EndOfStream
2025-11-19T14:00:35.853917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436113386890933:2048] [node 1] ICP30 unregister session Session# [5:7574436113386891021:2048] VirtualId# [5:7574436113386891014:0]
2025-11-19T14:00:35.853926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436113386891021:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.853976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:35.854003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:35.854046Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:35.854115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436114571058024:2048] [node 1] ICS01 socket: 402 reason# EndOfStream
2025-11-19T14:00:35.854120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436114571057936:2048] [node 1] ICP30 unregister session Session# [6:7574436114571058024:2048] VirtualId# [6:7574436114571058017:0]
2025-11-19T14:00:35.854124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436114571058024:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.854148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:35.854169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:35.854197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:35.854257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436112254688875:2048] [node 1] ICS01 socket: 439 reason# EndOfStream
2025-11-19T14:00:35.854261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436112254688775:2048] [node 1] ICP30 unregister session Session# [4:7574436112254688875:2048] VirtualId# [4:7574436112254688864:0]
2025-11-19T14:00:35.854265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436112254688875:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.854299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:35.854321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:35.863834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574436114506237158:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:35.864890Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:35.864990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436114506237260:2049] [node 2] ICS01 socket: 432 reason# EndOfStream
2025-11-19T14:00:35.865002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436114506237159:2049] [node 2] ICP30 unregister session Session# [3:7574436114506237260:2049] VirtualId# [3:7574436114506237254:0]
2025-11-19T14:00:35.865012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436114506237260:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.865058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:35.865082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:35.866402Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:35.866490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436112254688877:2049] [node 2] ICS01 socket: 396 reason# EndOfStream
2025-11-19T14:00:35.866498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436112254688776:2049] [node 2] ICP30 unregister session Session# [4:7574436112254688877:2049] VirtualId# [4:7574436112254688865:0]
2025-11-19T14:00:35.866506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436112254688877:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.866538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:35.866557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:35.868167Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:35.868231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436113386891036:2049] [node 2] ICS01 socket: 411 reason# EndOfStream
2025-11-19T14:00:35.868236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436113386890934:2049] [node 2] ICP30 unregister session Session# [5:7574436113386891036:2049] VirtualId# [5:7574436113386891034:0]
2025-11-19T14:00:35.868240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436113386891036:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.868262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:35.868281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:35.867989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:35.868064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436114571058038:2049] [node 2] ICS01 socket: 422 reason# EndOfStream
2025-11-19T14:00:35.868071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436114571057937:2049] [node 2] ICP30 unregister session Session# [6:7574436114571058038:2049] VirtualId# [6:7574436114571058036:0]
2025-11-19T14:00:35.868077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436114571058038:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.868115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:35.868137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:35.886077Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:35.886190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436114571058030:2050] [node 3] ICS01 socket: 419 reason# EndOfStream
2025-11-19T14:00:35.886202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436114571057938:2050] [node 3] ICP30 unregister session Session# [6:7574436114571058030:2050] VirtualId# [6:7574436114571058018:0]
2025-11-19T14:00:35.886211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436114571058030:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.886269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:35.886297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:35.886337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:35.886397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436113386891019:2050] [node 3] ICS01 socket: 417 reason# EndOfStream
2025-11-19T14:00:35.886402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436113386890935:2050] [node 3] ICP30 unregister session Session# [5:7574436113386891019:2050] VirtualId# [5:7574436113386891015:0]
2025-11-19T14:00:35.886407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436113386891019:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.886433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:35.886452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:35.913269Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:35.913419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436114571058027:2064] [node 4] ICS01 socket: 418 reason# EndOfStream
2025-11-19T14:00:35.913430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436114571058003:2064] [node 4] ICP30 unregister session Session# [6:7574436114571058027:2064] VirtualId# [6:7574436114571058021:0]
2025-11-19T14:00:35.913441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436114571058027:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:35.913501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:35.913527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:36.101854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436116855046085:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:36.101872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436116855046086:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:36.182264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436118294227792:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:36.182287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436118294227793:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:36.230638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436116185454635:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:36.230655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436116185454636:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:36.260027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436115161298670:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:36.260048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436115161298671:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:36.260053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436115161298672:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:36.263962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436115135537196:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:36.263960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436115135537195:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:36.263977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436115135537197:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:36.280069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436115135537211:2059] [node 4] ICP01 ready to work
2025-11-19T14:00:36.280508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436118294227837:2091] [node 5] ICP01 ready to work
2025-11-19T14:00:36.291249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436115135537231:2067] [node 5] ICP01 ready to work
2025-11-19T14:00:36.292236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436115161298713:2084] [node 4] ICP01 ready to work
2025-11-19T14:00:36.292533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436118294227852:2094] [node 4] ICP01 ready to work
2025-11-19T14:00:36.292648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436118294227853:2099] [node 6] ICP01 ready to work
2025-11-19T14:00:36.297250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436116855046141:2101] [node 6] ICP01 ready to work
2025-11-19T14:00:36.297429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436116855046152:2102] [node 5] ICP01 ready to work
2025-11-19T14:00:36.297623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436116855046153:2105] [node 4] ICP01 ready to work
2025-11-19T14:00:36.298768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436115161298730:2091] [node 6] ICP01 ready to work
2025-11-19T14:00:36.299726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436118294227837:2091] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(18748399ns) {Configure:104(17443997ns) {ProcessConfigured:135(17164097ns) {StartInitialHandshake:157(17163247ns) {DropHandshakes:469(313ns) {DropIncomingHandshake:441(82ns) DropOutgoingHandshake:456(75ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(1068ns) {SwitchToState:178(743ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.299854Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436116855046085:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(17545641ns) {Configure:104(17545199ns) {ProcessConfigured:135(17259776ns) {StartInitialHandshake:157(17259438ns) {DropHandshakes:469(161ns) {DropIncomingHandshake:441(30ns) DropOutgoingHandshake:456(26ns)} GenerateSessionVirtualId:988(156ns) PrepareNewSessionHandshake:476(1171ns) {SwitchToState:178(944ns)}}}}}
2025-11-19T14:00:36.301156Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436116185454635:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(18129370ns) {Configure:104(18128326ns) {ProcessConfigured:135(17689944ns) {StartInitialHandshake:157(17689773ns) {DropHandshakes:469(437ns) {DropIncomingHandshake:441(78ns) DropOutgoingHandshake:456(65ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:476(950ns) {SwitchToState:178(179ns)}}}}}
2025-11-19T14:00:36.301341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436116185454699:2100] [node 4] ICP01 ready to work
2025-11-19T14:00:36.301457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436116185454700:2102] [node 5] ICP01 ready to work
2025-11-19T14:00:36.302475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436116185454701:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54373
2025-11-19T14:00:36.302488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436116185454702:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54373
2025-11-19T14:00:36.302499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436116185454703:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54373
2025-11-19T14:00:36.302508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436116185454704:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54373
2025-11-19T14:00:36.303345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436118294227852:2094] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27806) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:36.303431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574436118294227852:2094] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:36.303435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.319132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7574436118294227852:2094] [node 4] ICP33 wake up from error state
2025-11-19T14:00:36.319307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436118294227871:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28188
2025-11-19T14:00:36.319354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436118294227872:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28188
2025-11-19T14:00:36.319365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436118294227873:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28188
2025-11-19T14:00:36.318532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436115135537272:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64092
2025-11-19T14:00:36.318558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436115135537273:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64092
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.320010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436116855046164:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58989
2025-11-19T14:00:36.320034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436116855046165:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58989
2025-11-19T14:00:36.320046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436116855046166:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58989
2025-11-19T14:00:36.320538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436116185454696:2097] [node 6] ICP01 ready to work
2025-11-19T14:00:36.322929Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436118294227871:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.320610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436116185454699:2100] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27806) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:36.320719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436116185454699:2100] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:36.320723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:36.328666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436115135537211:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(48507933ns) {Configure:104(48507055ns) {ProcessConfigured:135(47608282ns) {StartInitialHandshake:157(47607466ns) {DropHandshakes:469(181ns) {DropIncomingHandshake:441(22ns) DropOutgoingHandshake:456(23ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:476(1990ns) {SwitchToState:178(1043ns)}}}}}
2025-11-19T14:00:36.333448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436115161298671:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(49492138ns) {Configure:104(49490560ns) {ProcessConfigured:135(49125843ns) {StartInitialHandshake:157(49125034ns) {DropHandshakes:469(673ns) {DropIncomingHandshake:441(96ns) DropOutgoingHandshake:456(28ns)} GenerateSessionVirtualId:988(267ns) PrepareNewSessionHandshake:476(1921ns) {SwitchToState:178(917ns)}}}}}
2025-11-19T14:00:36.333876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574436116185454699:2100] [node 4] ICP33 wake up from error state
2025-11-19T14:00:36.333889Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436116185454702:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.336450Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436115161298730:2091] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(37590221ns) {Configure:104(37588974ns) {ProcessConfigured:135(37274868ns) {StartInitialHandshake:157(37273853ns) {DropHandshakes:469(1613ns) {DropIncomingHandshake:441(232ns) DropOutgoingHandshake:456(205ns)} GenerateSessionVirtualId:988(324ns) PrepareNewSessionHandshake:476(903ns) {SwitchToState:178(399ns)}}}}}
2025-11-19T14:00:36.341944Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436115135537272:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.342001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436115135537196:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.342358Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436115135537273:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.342377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436115135537195:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.357062Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436116855046156:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.357079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436116855046156:2112] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:36.357383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436115161298713:2084] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27806) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:36.357488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574436115161298713:2084] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:36.357491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27806) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:36.355454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436117788267429:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:36.355525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436117788267430:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:36.355991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436117788267491:2102] [node 5] ICP01 ready to work
2025-11-19T14:00:36.356154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436117788267492:2103] [node 6] ICP01 ready to work
2025-11-19T14:00:36.356686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436117788267428:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:36.356978Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436118294227857:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.357001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436118294227857:2105] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:36.357958Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436118294227872:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.357991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436118294227792:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.360399Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436116855046154:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.360412Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436116855046154:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.363765Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436116855046154:2111] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:36.363691Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436118294227857:2105] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:36.363277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436116185454704:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.363377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436116185454635:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.363412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436116185454720:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54373
2025-11-19T14:00:36.363492Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436116185454703:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.363509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436116185454636:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.363983Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436118294227851:2103] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.363991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436118294227851:2103] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:36.364234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436118294227853:2099] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.364253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:36.364258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436118294227853:2099] [node 6] ICP22 created new session: [2:7574436118294227880:2099]
2025-11-19T14:00:36.364264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436118294227880:2099] [node 6] ICS09 handshake done sender: [2:7574436118294227857:2105] self: [2:7574436118294227856:0] peer: [6:7574436115135537275:0] socket: 406 qp: -1
2025-11-19T14:00:36.364270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436118294227880:2099] [node 6] ICS10 traffic start
2025-11-19T14:00:36.364278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:36.364111Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436116855046156:2112] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:36.364155Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436116855046165:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.364225Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436116855046166:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.364232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436116855046085:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.364249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:36.364256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436116855046085:2048] [node 2] ICP22 created new session: [1:7574436116855046169:2048]
2025-11-19T14:00:36.364262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436116855046169:2048] [node 2] ICS09 handshake done sender: [1:7574436116855046154:2111] self: [1:7574436116855046144:0] peer: [2:7574436118294227876:0] socket: 410 qp: -1
2025-11-19T14:00:36.364270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436116855046169:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:36.364279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:36.364824Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436116855046143:2103] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.364830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436116855046143:2103] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:36.364703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436115135537278:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64092
2025-11-19T14:00:36.364805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436115135537277:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64092
2025-11-19T14:00:36.370028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436116855046141:2101] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.370077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:36.370086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436116855046141:2101] [node 6] ICP22 created new session: [1:7574436116855046171:2101]
2025-11-19T14:00:36.370092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436116855046171:2101] [node 6] ICS09 handshake done sender: [1:7574436116855046156:2112] self: [1:7574436116855046155:0] peer: [6:7574436115135537276:0] socket: 411 qp: -1
2025-11-19T14:00:36.370101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436116855046171:2101] [node 6] ICS10 traffic start
2025-11-19T14:00:36.370109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:36.370365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436116855046085:2048] [node 2] ICP18 handshake (actor [1:7574436116855046166:2117]) is too old
2025-11-19T14:00:36.376155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436115135537196:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436115135537272:2114] held: no
2025-11-19T14:00:36.376292Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436115135537273:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:36.376374Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436115135537272:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.379890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436118294227878:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28188
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.380060Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436116855046143:2103] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:36.380209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436115135537195:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.380239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:36.380246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436115135537195:2048] [node 1] ICP22 created new session: [6:7574436115135537279:2048]
2025-11-19T14:00:36.380253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436115135537279:2048] [node 1] ICS09 handshake done sender: [6:7574436115135537273:2115] self: [6:7574436115135537276:0] peer: [1:7574436116855046155:0] socket: 418 qp: -1
2025-11-19T14:00:36.380261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436115135537279:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:36.380271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:36.380667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436115135537196:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.380674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:36.380679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436115135537196:2049] [node 2] ICP22 created new session: [6:7574436115135537281:2049]
2025-11-19T14:00:36.380683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436115135537281:2049] [node 2] ICS09 handshake done sender: [6:7574436115135537272:2114] self: [6:7574436115135537275:0] peer: [2:7574436118294227856:0] socket: 405 qp: -1
2025-11-19T14:00:36.380687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436115135537281:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:36.380691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:36.380173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436115161298747:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23056
2025-11-19T14:00:36.380270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574436115161298713:2084] [node 4] ICP33 wake up from error state
2025-11-19T14:00:36.380406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436115161298749:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23056
2025-11-19T14:00:36.380446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436115161298750:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23056
2025-11-19T14:00:36.380483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436115161298751:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23056
2025-11-19T14:00:36.381058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436116185454723:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54373
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.385206Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436116185454704:2112] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:36.384666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436117788267492:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(24465431ns) {Configure:104(24463877ns) {ProcessConfigured:135(24164471ns) {StartInitialHandshake:157(24163233ns) {DropHandshakes:469(686ns) {DropIncomingHandshake:441(228ns) DropOutgoingHandshake:456(72ns)} GenerateSessionVirtualId:988(182ns) PrepareNewSessionHandshake:476(1546ns) {SwitchToState:178(413ns)}}}}}
2025-11-19T14:00:36.384881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436117788267505:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40556
2025-11-19T14:00:36.384698Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436115161298749:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.384482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436118294227879:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28188
2025-11-19T14:00:36.384585Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436118294227873:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.390556Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436116185454701:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.390594Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436116185454720:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.390604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436116185454635:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.390625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:36.390636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436116185454635:2048] [node 1] ICP22 created new session: [3:7574436116185454724:2048]
2025-11-19T14:00:36.390643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436116185454724:2048] [node 1] ICS09 handshake done sender: [3:7574436116185454704:2112] self: [3:7574436116185454721:0] peer: [1:7574436116855046138:0] socket: 415 qp: -1
2025-11-19T14:00:36.390652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436116185454724:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:36.390662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:36.390441Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436115161298750:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.390467Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436115161298747:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.390577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436115161298672:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.390677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436115161298713:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.396921Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436117788267497:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.396960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436117788267497:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:36.398422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436116855046086:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.398459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:36.398471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436116855046086:2049] [node 3] ICP22 created new session: [1:7574436116855046177:2049]
2025-11-19T14:00:36.398479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436116855046177:2049] [node 3] ICS09 handshake done sender: [1:7574436116855046143:2103] self: [1:7574436116855046138:0] peer: [3:7574436116185454721:0] socket: 409 qp: -1
2025-11-19T14:00:36.398487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436116855046177:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:36.398498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:36.398769Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436116855046164:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.401144Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436117788267497:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:36.400767Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436116185454710:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.400783Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436116185454710:2115] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:36.401790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436116185454699:2100] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.402408Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436116185454710:2115] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:36.402276Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436117788267495:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.402286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436117788267495:2110] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:36.402716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436117788267491:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.402732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:36.402741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436117788267491:2102] [node 5] ICP22 created new session: [4:7574436117788267508:2102]
2025-11-19T14:00:36.402749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436117788267508:2102] [node 5] ICS09 handshake done sender: [4:7574436117788267497:2111] self: [4:7574436117788267494:0] peer: [5:7574436115161298755:0] socket: 432 qp: -1
2025-11-19T14:00:36.402758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436117788267508:2102] [node 5] ICS10 traffic start
2025-11-19T14:00:36.402768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:36.403103Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436117788267505:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.403265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436117788267428:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.403898Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436115161298751:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.403940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436115161298670:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.403437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436118294227792:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28134) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7574436118294227872:2116] held: no
2025-11-19T14:00:36.403652Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436118294227872:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:36.403977Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436116855046160:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.403990Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436116855046160:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:36.404740Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436116855046158:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.404743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436116855046158:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.405241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436115161298757:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23056
2025-11-19T14:00:36.405510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436115161298758:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23056
2025-11-19T14:00:36.405325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436116185454700:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.405348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:36.405354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436116185454700:2102] [node 5] ICP22 created new session: [3:7574436116185454730:2102]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.406034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436115135537285:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64092
2025-11-19T14:00:36.405227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436116855046180:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58989
2025-11-19T14:00:36.405363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436116185454730:2102] [node 5] ICS09 handshake done sender: [3:7574436116185454710:2115] self: [3:7574436116185454709:0] peer: [5:7574436115161298753:0] socket: 425 qp: -1
2025-11-19T14:00:36.405369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436116185454730:2102] [node 5] ICS10 traffic start
2025-11-19T14:00:36.405378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:36.406415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436118294227792:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.406431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:36.406436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436118294227792:2048] [node 1] ICP22 created new session: [2:7574436118294227886:2048]
2025-11-19T14:00:36.406442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436118294227886:2048] [node 1] ICS09 handshake done sender: [2:7574436118294227872:2116] self: [2:7574436118294227876:0] peer: [1:7574436116855046144:0] socket: 419 qp: -1
2025-11-19T14:00:36.406448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436118294227886:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:36.406454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.406692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436115161298759:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23056
2025-11-19T14:00:36.406717Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436115161298749:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:36.406877Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436115161298750:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:36.407572Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436117788267495:2110] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:36.407508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436115161298672:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.407528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:36.407535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436115161298672:2050] [node 3] ICP22 created new session: [5:7574436115161298760:2050]
2025-11-19T14:00:36.407541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436115161298760:2050] [node 3] ICS09 handshake done sender: [5:7574436115161298749:2116] self: [5:7574436115161298753:0] peer: [3:7574436116185454709:0] socket: 403 qp: -1
2025-11-19T14:00:36.407548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436115161298760:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:36.407555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:36.408296Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436116855046160:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:36.408847Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436118294227851:2103] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:36.407869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436117788267430:2050] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.407887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:36.407895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436117788267430:2050] [node 3] ICP22 created new session: [4:7574436117788267513:2050]
2025-11-19T14:00:36.407908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436117788267513:2050] [node 3] ICS09 handshake done sender: [4:7574436117788267495:2110] self: [4:7574436117788267493:0] peer: [3:7574436116185454729:0] socket: 431 qp: -1
2025-11-19T14:00:36.407919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436117788267513:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:36.407928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:36.409384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436115161298713:2084] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.409400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:36.409407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436115161298713:2084] [node 4] ICP22 created new session: [5:7574436115161298762:2084]
2025-11-19T14:00:36.409413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436115161298762:2084] [node 4] ICS09 handshake done sender: [5:7574436115161298750:2117] self: [5:7574436115161298755:0] peer: [4:7574436117788267494:0] socket: 440 qp: -1
2025-11-19T14:00:36.409419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436115161298762:2084] [node 4] ICS10 traffic start
2025-11-19T14:00:36.409426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.411910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436116185454732:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54373
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.412366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436115135537286:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64092
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.413088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436116185454733:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54373
2025-11-19T14:00:36.411144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436117788267515:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40556
2025-11-19T14:00:36.411188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436117788267516:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40556
2025-11-19T14:00:36.412625Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436117788267515:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.412647Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436117788267505:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:36.411636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436118294227884:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28188
2025-11-19T14:00:36.411732Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436118294227879:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.412038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436118294227793:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.412070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:36.412077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436118294227793:2049] [node 3] ICP22 created new session: [2:7574436118294227888:2049]
2025-11-19T14:00:36.412084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436118294227888:2049] [node 3] ICS09 handshake done sender: [2:7574436118294227851:2103] self: [2:7574436118294227838:0] peer: [3:7574436116185454722:0] socket: 404 qp: -1
2025-11-19T14:00:36.412091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436118294227888:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:36.412101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:36.412609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436118294227852:2094] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.412861Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436118294227884:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.413887Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436115135537285:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.413922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436115135537197:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.414195Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436116185454636:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436116185454703:2111] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.414770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436115161298764:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23056
2025-11-19T14:00:36.415636Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436115161298751:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:36.415706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436115161298670:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.415719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:36.415723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436115161298670:2048] [node 1] ICP22 created new session: [5:7574436115161298765:2048]
2025-11-19T14:00:36.415728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436115161298765:2048] [node 1] ICS09 handshake done sender: [5:7574436115161298751:2118] self: [5:7574436115161298756:0] peer: [1:7574436116855046157:0] socket: 400 qp: -1
2025-11-19T14:00:36.415734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436115161298765:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:36.415796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:36.414233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436116855046153:2105] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.414252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:36.414258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436116855046153:2105] [node 4] ICP22 created new session: [1:7574436116855046182:2105]
2025-11-19T14:00:36.414267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436116855046182:2105] [node 4] ICS09 handshake done sender: [1:7574436116855046160:2114] self: [1:7574436116855046159:0] peer: [4:7574436117788267512:0] socket: 423 qp: -1
2025-11-19T14:00:36.414273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436116855046182:2105] [node 4] ICS10 traffic start
2025-11-19T14:00:36.414281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:36.414881Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436116855046158:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:36.415127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436116855046152:2102] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.415140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:36.415145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436116855046152:2102] [node 5] ICP22 created new session: [1:7574436116855046185:2102]
2025-11-19T14:00:36.415150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436116855046185:2102] [node 5] ICS09 handshake done sender: [1:7574436116855046158:2113] self: [1:7574436116855046157:0] peer: [5:7574436115161298756:0] socket: 407 qp: -1
2025-11-19T14:00:36.415155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436116855046185:2102] [node 5] ICS10 traffic start
2025-11-19T14:00:36.415162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:36.423151Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436117788267499:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.423167Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436117788267499:2112] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:36.423910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436117788267428:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.423935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:36.423947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436117788267428:2048] [node 1] ICP22 created new session: [4:7574436117788267518:2048]
2025-11-19T14:00:36.423956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436117788267518:2048] [node 1] ICS09 handshake done sender: [4:7574436117788267505:2114] self: [4:7574436117788267512:0] peer: [1:7574436116855046159:0] socket: 443 qp: -1
2025-11-19T14:00:36.423967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436117788267518:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:36.423980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:36.425175Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436116185454720:2119] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:36.426165Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436116185454703:2111] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:36.426397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436116185454699:2100] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.426414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:36.426423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436116185454699:2100] [node 4] ICP22 created new session: [3:7574436116185454734:2100]
2025-11-19T14:00:36.426430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436116185454734:2100] [node 4] ICS09 handshake done sender: [3:7574436116185454720:2119] self: [3:7574436116185454729:0] peer: [4:7574436117788267493:0] socket: 438 qp: -1
2025-11-19T14:00:36.426439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436116185454734:2100] [node 4] ICS10 traffic start
2025-11-19T14:00:36.426448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:36.426651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436116185454636:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.426657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:36.426662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436116185454636:2049] [node 2] ICP22 created new session: [3:7574436116185454736:2049]
2025-11-19T14:00:36.426666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436116185454736:2049] [node 2] ICS09 handshake done sender: [3:7574436116185454703:2111] self: [3:7574436116185454722:0] peer: [2:7574436118294227838:0] socket: 414 qp: -1
2025-11-19T14:00:36.426670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436116185454736:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:36.426677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:36.428796Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436116185454717:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.428813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436116185454717:2118] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:36.430554Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436117788267499:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:36.430671Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436116185454717:2118] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:36.430757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436116185454696:2097] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.430779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:36.430786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436116185454696:2097] [node 6] ICP22 created new session: [3:7574436116185454739:2097]
2025-11-19T14:00:36.430792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436116185454739:2097] [node 6] ICS09 handshake done sender: [3:7574436116185454717:2118] self: [3:7574436116185454714:0] peer: [6:7574436115135537287:0] socket: 421 qp: -1
2025-11-19T14:00:36.430798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436116185454739:2097] [node 6] ICS10 traffic start
2025-11-19T14:00:36.430806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:36.431254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436117788267429:2049] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.431267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:36.431272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436117788267429:2049] [node 2] ICP22 created new session: [4:7574436117788267522:2049]
2025-11-19T14:00:36.431278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436117788267522:2049] [node 2] ICS09 handshake done sender: [4:7574436117788267499:2112] self: [4:7574436117788267498:0] peer: [2:7574436118294227890:0] socket: 433 qp: -1
2025-11-19T14:00:36.431283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436117788267522:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:36.431289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.433179Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436115135537197:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26068) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436115135537285:2119] held: no
2025-11-19T14:00:36.433351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436115135537288:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64092
2025-11-19T14:00:36.433394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436115135537289:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64092
2025-11-19T14:00:36.433536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436118294227892:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28188
2025-11-19T14:00:36.433783Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436115161298759:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.433827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436115161298671:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.434087Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436118294227869:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.434100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436118294227869:2113] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:36.434136Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7574436116855046153:2105] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:36.435209Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436115135537288:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.435233Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436115135537285:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:36.435903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436115135537197:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.435918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:36.435924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436115135537197:2050] [node 3] ICP22 created new session: [6:7574436115135537290:2050]
2025-11-19T14:00:36.435946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436115135537290:2050] [node 3] ICS09 handshake done sender: [6:7574436115135537285:2119] self: [6:7574436115135537287:0] peer: [3:7574436116185454714:0] socket: 451 qp: -1
2025-11-19T14:00:36.435952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436115135537290:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:36.435958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:36.436156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436115135537231:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:36.434678Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436118294227879:2119] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:36.435320Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436118294227869:2113] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:36.435384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436118294227837:2091] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.435402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:36.435408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436118294227837:2091] [node 5] ICP22 created new session: [2:7574436118294227894:2091]
2025-11-19T14:00:36.435414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436118294227894:2091] [node 5] ICS09 handshake done sender: [2:7574436118294227869:2113] self: [2:7574436118294227840:0] peer: [5:7574436115161298768:0] socket: 453 qp: -1
2025-11-19T14:00:36.435420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436118294227894:2091] [node 5] ICS10 traffic start
2025-11-19T14:00:36.435428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:36.436714Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436115135537286:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.436772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436115135537211:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.438416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436115135537294:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64092
2025-11-19T14:00:36.438961Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436115135537286:2120] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:36.436868Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436117788267506:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.436879Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436117788267506:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:36.437873Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436117788267506:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:36.437948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436117788267492:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.437960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:36.437965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436117788267492:2103] [node 6] ICP22 created new session: [4:7574436117788267525:2103]
2025-11-19T14:00:36.437970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436117788267525:2103] [node 6] ICS09 handshake done sender: [4:7574436117788267506:2115] self: [4:7574436117788267496:0] peer: [6:7574436115135537293:0] socket: 461 qp: -1
2025-11-19T14:00:36.437975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436117788267525:2103] [node 6] ICS10 traffic start
2025-11-19T14:00:36.437983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.438376Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436115161298748:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:36.438388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436115161298748:2115] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:36.439491Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436115161298671:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436115161298759:2121] held: no
2025-11-19T14:00:36.439720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436115161298769:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23056
2025-11-19T14:00:36.438494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436118294227852:2094] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.438507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:36.438513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436118294227852:2094] [node 4] ICP22 created new session: [2:7574436118294227896:2094]
2025-11-19T14:00:36.438517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436118294227896:2094] [node 4] ICS09 handshake done sender: [2:7574436118294227879:2119] self: [2:7574436118294227890:0] peer: [4:7574436117788267498:0] socket: 441 qp: -1
2025-11-19T14:00:36.438525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436118294227896:2094] [node 4] ICS10 traffic start
2025-11-19T14:00:36.438533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:36.441215Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436115161298759:2121] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:36.441944Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436115161298748:2115] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:36.442091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436115161298671:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.442109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:36.442117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436115161298671:2049] [node 2] ICP22 created new session: [5:7574436115161298771:2049]
2025-11-19T14:00:36.442123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436115161298771:2049] [node 2] ICS09 handshake done sender: [5:7574436115161298759:2121] self: [5:7574436115161298768:0] peer: [2:7574436118294227840:0] socket: 458 qp: -1
2025-11-19T14:00:36.442130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436115161298771:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:36.442140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:36.442372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436115161298730:2091] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:36.442387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:36.442393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436115161298730:2091] [node 6] ICP22 created new session: [5:7574436115161298773:2091]
2025-11-19T14:00:36.442398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436115161298773:2091] [node 6] ICS09 handshake done sender: [5:7574436115161298748:2115] self: [5:7574436115161298735:0] peer: [6:7574436115135537292:0] socket: 398 qp: -1
2025-11-19T14:00:36.442402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436115161298773:2091] [node 6] ICS10 traffic start
2025-11-19T14:00:36.442408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:36.452200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436115135537211:2059] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.452246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:36.452264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436115135537211:2059] [node 4] ICP22 created new session: [6:7574436115135537295:2059]
2025-11-19T14:00:36.452273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436115135537295:2059] [node 4] ICS09 handshake done sender: [6:7574436115135537286:2120] self: [6:7574436115135537293:0] peer: [4:7574436117788267496:0] socket: 462 qp: -1
2025-11-19T14:00:36.452286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436115135537295:2059] [node 4] ICS10 traffic start
2025-11-19T14:00:36.452300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:36.460448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7574436115135537211:2059] [node 4] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:36.460616Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436115135537231:2067] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4186) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436115135537288:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:36.511162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436115135537297:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64092
2025-11-19T14:00:36.513026Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436115135537288:2121] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:36.513171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436115135537231:2067] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:36.513193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:36.513203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436115135537231:2067] [node 5] ICP22 created new session: [6:7574436115135537298:2067]
2025-11-19T14:00:36.513211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436115135537298:2067] [node 5] ICS09 handshake done sender: [6:7574436115135537288:2121] self: [6:7574436115135537292:0] peer: [5:7574436115161298735:0] socket: 413 qp: -1
2025-11-19T14:00:36.513221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436115135537298:2067] [node 5] ICS10 traffic start
2025-11-19T14:00:36.513235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:36.528017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:36.528128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436118294227886:2048] [node 1] ICS01 socket: 419 reason# EndOfStream
2025-11-19T14:00:36.528140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436118294227792:2048] [node 1] ICP30 unregister session Session# [2:7574436118294227886:2048] VirtualId# [2:7574436118294227876:0]
2025-11-19T14:00:36.528149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436118294227886:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:36.528220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:36.528250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:36.528334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:36.528419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574436116185454724:2048] [node 1] ICS07 socket disconnect 415 reason# EndOfStream
2025-11-19T14:00:36.528431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436116185454724:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:36.528468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:36.528487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436116185454724:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:36.531854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:36.531955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436115135537279:2048] [node 1] ICS01 socket: 418 reason# EndOfStream
2025-11-19T14:00:36.531977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436115135537195:2048] [node 1] ICP30 unregister session Session# [6:7574436115135537279:2048] VirtualId# [6:7574436115135537276:0]
2025-11-19T14:00:36.531984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436115135537279:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:36.532031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:36.532056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:36.532284Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:36.532352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436115161298765:2048] [node 1] ICS01 socket: 400 reason# EndOfStream
2025-11-19T14:00:36.532358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436115161298670:2048] [node 1] ICP30 unregister session Session# [5:7574436115161298765:2048] VirtualId# [5:7574436115161298756:0]
2025-11-19T14:00:36.532363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436115161298765:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:36.532395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:36.532415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:36.531538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436116185454635:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28134) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T14:00:36.531549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436116185454724:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:36.533078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436116185454635:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28134) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:36.533093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436116185454724:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:36.533097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436116185454635:2048] [node 1] ICP30 unregister session Session# [3:7574436116185454724:2048] VirtualId# [3:7574436116185454721:0]
2025-11-19T14:00:36.533145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:36.533151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436116185454635:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28134) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:36.533154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28134) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:36.533481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:36.533564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436117788267518:2048] [node 1] ICS01 socket: 443 reason# EndOfStream
2025-11-19T14:00:36.533573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436117788267428:2048] [node 1] ICP30 unregister session Session# [4:7574436117788267518:2048] VirtualId# [4:7574436117788267512:0]
2025-11-19T14:00:36.533579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436117788267518:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:36.533630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:36.533655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:36.544527Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:36.544638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436116185454736:2049] [node 2] ICS01 socket: 414 reason# EndOfStream
2025-11-19T14:00:36.544650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436116185454636:2049] [node 2] ICP30 unregister session Session# [3:7574436116185454736:2049] VirtualId# [3:7574436116185454722:0]
2025-11-19T14:00:36.544658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436116185454736:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:36.544693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:36.544715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:36.544895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574436116185454635:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:36.588048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:36.588179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436117788267522:2049] [node 2] ICS01 socket: 433 reason# EndOfStream
2025-11-19T14:00:36.588192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436117788267429:2049] [node 2] ICP30 unregister session Session# [4:7574436117788267522:2049] VirtualId# [4:7574436117788267498:0]
2025-11-19T14:00:36.588200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436117788267522:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:36.588253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:36.588292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:36.588710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:36.588797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436115161298771:2049] [node 2] ICS01 socket: 458 reason# EndOfStream
2025-11-19T14:00:36.588804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436115161298671:2049] [node 2] ICP30 unregister session Session# [5:7574436115161298771:2049] VirtualId# [5:7574436115161298768:0]
2025-11-19T14:00:36.588810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436115161298771:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:36.588858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:36.588883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:36.588920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:36.588985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436115135537281:2049] [node 2] ICS01 socket: 405 reason# EndOfStream
2025-11-19T14:00:36.588990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436115135537196:2049] [node 2] ICP30 unregister session Session# [6:7574436115135537281:2049] VirtualId# [6:7574436115135537275:0]
2025-11-19T14:00:36.588994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436115135537281:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:36.589022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:36.589043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:36.608101Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:36.608228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436115135537290:2050] [node 3] ICS01 socket: 451 reason# EndOfStream
2025-11-19T14:00:36.608239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436115135537197:2050] [node 3] ICP30 unregister session Session# [6:7574436115135537290:2050] VirtualId# [6:7574436115135537287:0]
2025-11-19T14:00:36.608249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436115135537290:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:36.608294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:36.608319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:36.608361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:36.608469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436115161298760:2050] [node 3] ICS01 socket: 403 reason# EndOfStream
2025-11-19T14:00:36.608474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436115161298672:2050] [node 3] ICP30 unregister session Session# [5:7574436115161298760:2050] VirtualId# [5:7574436115161298753:0]
2025-11-19T14:00:36.608477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436115161298760:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:36.608504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:36.608525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:36.628813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:36.628946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436115135537295:2059] [node 4] ICS01 socket: 462 reason# EndOfStream
2025-11-19T14:00:36.628957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436115135537211:2059] [node 4] ICP30 unregister session Session# [6:7574436115135537295:2059] VirtualId# [6:7574436115135537293:0]
2025-11-19T14:00:36.628968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436115135537295:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:36.629023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:36.629051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:36.648322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T14:00:36.648448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436115135537298:2067] [node 5] ICS01 socket: 413 reason# EndOfStream
2025-11-19T14:00:36.648460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436115135537231:2067] [node 5] ICP30 unregister session Session# [6:7574436115135537298:2067] VirtualId# [6:7574436115135537292:0]
2025-11-19T14:00:36.648469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436115135537298:2067] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:36.648512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T14:00:36.648537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:36.795964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436119302506161:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:36.795983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436119302506162:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:36.847929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436118658355647:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:36.847948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436118658355648:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:36.868462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436117947372670:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:36.868479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436117947372671:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:36.903935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436115473451604:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:36.903948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436115473451605:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:36.903953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436115473451606:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:36.943954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436118112061853:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:36.943971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436118112061854:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:36.943977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436118112061855:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:37.007792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436118166035853:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:37.007920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436118166035854:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:37.012072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436118166035855:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:37.076607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436122242340022:2102] [node 6] ICP01 ready to work
2025-11-19T14:00:37.076795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436122461003202:2099] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.078194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436122407029210:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T14:00:37.080213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436123597473519:2107] [node 4] ICP01 ready to work
2025-11-19T14:00:37.080321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436123597473520:2108] [node 5] ICP01 ready to work
2025-11-19T14:00:37.090331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436122953323005:2107] [node 5] ICP01 ready to work
2025-11-19T14:00:37.090414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436122953323006:2108] [node 6] ICP01 ready to work
2025-11-19T14:00:37.089625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436122461003214:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.093471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436122953323010:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45348
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.093862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436123597473531:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14363
2025-11-19T14:00:37.101771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436122407029213:2109] [node 4] ICP01 ready to work
2025-11-19T14:00:37.101878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436122407029214:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:37.109042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436122242340036:2110] [node 5] ICP01 ready to work
2025-11-19T14:00:37.111000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436122242340041:2112] [node 4] ICP01 ready to work
2025-11-19T14:00:37.108472Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436122407029210:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.117582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436122242340044:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12285
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.118823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436122242340045:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12285
2025-11-19T14:00:37.116519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436122461003221:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40690
2025-11-19T14:00:37.116593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436122461003222:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40690
2025-11-19T14:00:37.116605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436122461003223:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40690
2025-11-19T14:00:37.116825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436122953323005:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(24943950ns) {Configure:104(21065782ns) {ProcessConfigured:135(20770056ns) {StartInitialHandshake:157(20769223ns) {DropHandshakes:469(206ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:476(997ns) {SwitchToState:178(770ns)}}}}}
2025-11-19T14:00:37.117014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436122953323015:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.129152Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436122461003221:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.130407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436118166035855:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.132316Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122242340024:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.132331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436122242340024:2103] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.133324Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436122242340024:2103] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:37.133548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436122242340022:2102] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.133568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:37.133578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436122242340022:2102] [node 6] ICP22 created new session: [3:7574436122242340047:2102]
2025-11-19T14:00:37.133586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436122242340047:2102] [node 6] ICS09 handshake done sender: [3:7574436122242340024:2103] self: [3:7574436122242340023:0] peer: [6:7574436122461003229:0] socket: 408 qp: -1
2025-11-19T14:00:37.133594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436122242340047:2102] [node 6] ICS10 traffic start
2025-11-19T14:00:37.133605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:37.128154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436119768418970:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:37.128910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119768418965:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18712
2025-11-19T14:00:37.128923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119768418966:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18712
2025-11-19T14:00:37.128934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119768418967:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18712
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.138106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436123597473534:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14363
2025-11-19T14:00:37.138131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436123597473535:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14363
2025-11-19T14:00:37.138965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436119768418962:2103] [node 5] ICP01 ready to work
2025-11-19T14:00:37.138943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436123597473533:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.144055Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436119302506162:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(57537463ns) {Configure:104(57535513ns) {ProcessConfigured:135(57165393ns) {StartInitialHandshake:157(57164169ns) {DropHandshakes:469(1834ns) {DropIncomingHandshake:441(210ns) DropOutgoingHandshake:456(263ns)} GenerateSessionVirtualId:988(282ns) PrepareNewSessionHandshake:476(1918ns) {SwitchToState:178(845ns)}}}}}
2025-11-19T14:00:37.144127Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436123597473531:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.144328Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436115473451606:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(40632138ns) {Configure:104(40631665ns) {ProcessConfigured:135(40213934ns) {StartInitialHandshake:157(40213272ns) {DropHandshakes:469(692ns) {DropIncomingHandshake:441(113ns) DropOutgoingHandshake:456(117ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:476(1277ns) {SwitchToState:178(320ns)}}}}}
2025-11-19T14:00:37.144369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119768418979:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18712
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.145608Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436119768418965:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.145639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436115473451604:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.145710Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122242340045:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.145745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7574436122242340022:2102] [node 6] ICP18 handshake (actor [3:7574436122242340045:2115]) is too old
2025-11-19T14:00:37.146053Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436119768418979:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.146074Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436119768418966:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.146096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436115473451606:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.146978Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436123597473524:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.146986Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436123597473524:2109] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.147785Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436123597473524:2109] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:37.147949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436123597473519:2107] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.147976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:37.147985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436123597473519:2107] [node 4] ICP22 created new session: [1:7574436123597473540:2107]
2025-11-19T14:00:37.147993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436123597473540:2107] [node 4] ICS09 handshake done sender: [1:7574436123597473524:2109] self: [1:7574436123597473521:0] peer: [4:7574436119768418981:0] socket: 413 qp: -1
2025-11-19T14:00:37.148003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436123597473540:2107] [node 4] ICS10 traffic start
2025-11-19T14:00:37.148028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.146194Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122242340043:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.146204Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436122242340043:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.148566Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436122242340043:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:37.148821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436122242340041:2112] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.148065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436118112061854:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(36337069ns) {Configure:104(36336040ns) {ProcessConfigured:135(35768737ns) {StartInitialHandshake:157(35767763ns) {DropHandshakes:469(488ns) {DropIncomingHandshake:441(62ns) DropOutgoingHandshake:456(62ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(1255ns) {SwitchToState:178(439ns)}}}}}
2025-11-19T14:00:37.148832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:37.148839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436122242340041:2112] [node 4] ICP22 created new session: [3:7574436122242340052:2112]
2025-11-19T14:00:37.148846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436122242340052:2112] [node 4] ICS09 handshake done sender: [3:7574436122242340043:2113] self: [3:7574436122242340042:0] peer: [4:7574436119768418982:0] socket: 439 qp: -1
2025-11-19T14:00:37.148852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436122242340052:2112] [node 4] ICS10 traffic start
2025-11-19T14:00:37.148870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:37.156148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436122242340051:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12285
2025-11-19T14:00:37.156393Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436123597473534:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.156698Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122242340051:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.156729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436117947372670:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.145280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436122953323022:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45348
2025-11-19T14:00:37.152016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436118658355647:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(61540341ns) {Configure:104(61539804ns) {ProcessConfigured:135(61186539ns) {StartInitialHandshake:157(61185847ns) {DropHandshakes:469(275ns) {DropIncomingHandshake:441(56ns) DropOutgoingHandshake:456(36ns)} GenerateSessionVirtualId:988(48ns) PrepareNewSessionHandshake:476(1277ns) {SwitchToState:178(868ns)}}}}}
2025-11-19T14:00:37.160721Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436123597473533:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(21636287ns) {Configure:104(21634938ns) {ProcessConfigured:135(21276200ns) {StartInitialHandshake:157(21275029ns) {DropHandshakes:469(1584ns) {DropIncomingHandshake:441(255ns) DropOutgoingHandshake:456(309ns)} GenerateSessionVirtualId:988(359ns) PrepareNewSessionHandshake:476(870ns) {SwitchToState:178(506ns)}}}}}
2025-11-19T14:00:37.160816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436119768418962:2103] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(21693251ns) {Configure:104(21692540ns) {ProcessConfigured:135(21342632ns) {StartInitialHandshake:157(21342342ns) {DropHandshakes:469(792ns) {DropIncomingHandshake:441(121ns) DropOutgoingHandshake:456(137ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:476(1181ns) {SwitchToState:178(621ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.161122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436122407029228:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T14:00:37.159990Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122242340044:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.161194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436122407029229:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T14:00:37.161228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436122407029230:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T14:00:37.160332Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [3:7574436117947372671:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(64996948ns) {Configure:104(64995744ns) {ProcessConfigured:135(64614338ns) {StartInitialHandshake:157(64613264ns) {DropHandshakes:469(1273ns) {DropIncomingHandshake:441(112ns) DropOutgoingHandshake:456(82ns)} GenerateSessionVirtualId:988(269ns) PrepareNewSessionHandshake:476(811ns) {SwitchToState:178(209ns)}}}}}
2025-11-19T14:00:37.161376Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436118166035855:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64815) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436122461003221:2112] held: no
2025-11-19T14:00:37.161498Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436122461003223:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.161598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436122461003230:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40690
2025-11-19T14:00:37.161623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436122461003231:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40690
2025-11-19T14:00:37.163891Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122953323010:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.163982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436118658355647:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.165072Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436118166035853:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(53089495ns) {Configure:104(53088529ns) {ProcessConfigured:135(52721900ns) {StartInitialHandshake:157(52721057ns) {DropHandshakes:469(979ns) {DropIncomingHandshake:441(107ns) DropOutgoingHandshake:456(55ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:476(965ns) {SwitchToState:178(700ns)}}}}}
2025-11-19T14:00:37.165109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436122461003202:2099] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.165224Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436122461003221:2112] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:37.166117Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122242340038:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.166128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436122242340038:2111] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.165325Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436122407029229:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.165361Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436122407029228:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.165391Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436122407029217:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.165395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436122407029217:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.165891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436118112061855:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.165904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436118112061853:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.166932Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436122407029217:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:37.167121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436122407029214:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.167144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:37.167152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436122407029214:2110] [node 6] ICP22 created new session: [5:7574436122407029235:2110]
2025-11-19T14:00:37.167158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436122407029235:2110] [node 6] ICS09 handshake done sender: [5:7574436122407029217:2111] self: [5:7574436122407029216:0] peer: [6:7574436122461003233:0] socket: 422 qp: -1
2025-11-19T14:00:37.167165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436122407029235:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:37.167176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:37.167671Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436122242340038:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.167795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436122242340036:2110] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.167816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:37.167822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436122242340036:2110] [node 5] ICP22 created new session: [3:7574436122242340059:2110]
2025-11-19T14:00:37.167828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436122242340059:2110] [node 5] ICS09 handshake done sender: [3:7574436122242340038:2111] self: [3:7574436122242340037:0] peer: [5:7574436122407029232:0] socket: 425 qp: -1
2025-11-19T14:00:37.167833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436122242340059:2110] [node 5] ICS10 traffic start
2025-11-19T14:00:37.167856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.167994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436122407029213:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(66060220ns) {Configure:104(66058938ns) {ProcessConfigured:135(65662511ns) {StartInitialHandshake:157(65661652ns) {DropHandshakes:469(1825ns) {DropIncomingHandshake:441(221ns) DropOutgoingHandshake:456(329ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:476(715ns) {SwitchToState:178(450ns)}}}}}
2025-11-19T14:00:37.168163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436122407029234:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T14:00:37.168574Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436123597473529:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.168588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436123597473529:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.167973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436118166035855:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.167990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:37.167996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436118166035855:2050] [node 3] ICP22 created new session: [6:7574436122461003235:2050]
2025-11-19T14:00:37.168001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436122461003235:2050] [node 3] ICS09 handshake done sender: [6:7574436122461003221:2112] self: [6:7574436122461003229:0] peer: [3:7574436122242340023:0] socket: 411 qp: -1
2025-11-19T14:00:37.168006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436122461003235:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:37.168021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:37.168322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436122461003234:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40690
2025-11-19T14:00:37.169309Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436123597473537:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.169318Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436123597473537:2116] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.168452Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436122407029229:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:37.168536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436118112061855:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.168548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:37.168554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436118112061855:2050] [node 3] ICP22 created new session: [5:7574436122407029239:2050]
2025-11-19T14:00:37.168560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436122407029239:2050] [node 3] ICS09 handshake done sender: [5:7574436122407029229:2116] self: [5:7574436122407029232:0] peer: [3:7574436122242340037:0] socket: 450 qp: -1
2025-11-19T14:00:37.168566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436122407029239:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:37.168581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:37.171978Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436123597473526:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.171995Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436123597473526:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.172169Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436123597473529:2111] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:37.172293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436123597473520:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.172319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:37.172326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436123597473520:2108] [node 5] ICP22 created new session: [1:7574436123597473546:2108]
2025-11-19T14:00:37.172333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436123597473546:2108] [node 5] ICS09 handshake done sender: [1:7574436123597473529:2111] self: [1:7574436123597473522:0] peer: [5:7574436122407029233:0] socket: 416 qp: -1
2025-11-19T14:00:37.172339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436123597473546:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:37.172350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.179596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436122953323025:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45348
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.180141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436122407029241:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T14:00:37.181440Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436122407029228:2115] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:37.182107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436118112061853:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.182130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:37.182137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436118112061853:2048] [node 1] ICP22 created new session: [5:7574436122407029243:2048]
2025-11-19T14:00:37.182149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436122407029243:2048] [node 1] ICS09 handshake done sender: [5:7574436122407029228:2115] self: [5:7574436122407029233:0] peer: [1:7574436123597473522:0] socket: 420 qp: -1
2025-11-19T14:00:37.182158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436122407029243:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:37.180813Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436122461003202:2099] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15489) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436122461003223:2114] held: no
2025-11-19T14:00:37.180966Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436122461003223:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:37.182169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436122461003202:2099] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.182181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:37.182186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436122461003202:2099] [node 5] ICP22 created new session: [6:7574436122461003239:2099]
2025-11-19T14:00:37.182191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436122461003239:2099] [node 5] ICS09 handshake done sender: [6:7574436122461003223:2114] self: [6:7574436122461003233:0] peer: [5:7574436122407029216:0] socket: 427 qp: -1
2025-11-19T14:00:37.182197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436122461003239:2099] [node 5] ICS10 traffic start
2025-11-19T14:00:37.182204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.183570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119768418986:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18712
2025-11-19T14:00:37.184051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436122953323026:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45348
2025-11-19T14:00:37.183963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119768418985:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18712
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.186613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436123597473548:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14363
2025-11-19T14:00:37.186747Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436123597473526:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:37.185654Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436122461003222:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.185736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436118166035854:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.186833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436119302506161:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.186849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:37.186856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436119302506161:2048] [node 2] ICP22 created new session: [1:7574436123597473551:2048]
2025-11-19T14:00:37.186863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436123597473551:2048] [node 2] ICS09 handshake done sender: [1:7574436123597473526:2110] self: [1:7574436123597473525:0] peer: [2:7574436122953323024:0] socket: 414 qp: -1
2025-11-19T14:00:37.186869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436123597473551:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:37.186876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:37.182165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.190576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119768418987:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18712
2025-11-19T14:00:37.190155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436123597473520:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(17756139ns) {HandleHandshakeStatus:329(17755012ns) {DropIncomingHandshake:441(5059ns) DropOutgoingHandshake:456(282ns) SwitchToState:178(343ns) ProcessPendingSessionEvents:495(17650984ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11712852ns max# 11629031ns}}}
2025-11-19T14:00:37.192073Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122953323026:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.198093Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436119768418979:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:37.204123Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122953323025:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.204921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436122953323029:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45348
2025-11-19T14:00:37.204979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436122953323030:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45348
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.205410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436122242340061:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12285
2025-11-19T14:00:37.205448Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436122407029230:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.205495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436118112061854:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.205379Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436122953323010:2110] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:37.205703Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122953323014:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.205712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436122953323014:2112] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.206217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436118658355647:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.206238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:37.206246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436118658355647:2048] [node 1] ICP22 created new session: [2:7574436122953323031:2048]
2025-11-19T14:00:37.206254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436122953323031:2048] [node 1] ICS09 handshake done sender: [2:7574436122953323010:2110] self: [2:7574436122953323024:0] peer: [1:7574436123597473525:0] socket: 417 qp: -1
2025-11-19T14:00:37.206264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436122953323031:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:37.206274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.207272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436122407029246:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T14:00:37.207411Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436122953323014:2112] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:37.207600Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436122407029230:2117] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:37.207707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436118112061854:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.207724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:37.207730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436118112061854:2049] [node 2] ICP22 created new session: [5:7574436122407029247:2049]
2025-11-19T14:00:37.207735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436122407029247:2049] [node 2] ICS09 handshake done sender: [5:7574436122407029230:2117] self: [5:7574436122407029245:0] peer: [2:7574436122953323012:0] socket: 451 qp: -1
2025-11-19T14:00:37.207741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436122407029247:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:37.207779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:37.208287Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436123597473537:2116] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:37.208236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436122953323005:2107] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.208257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:37.208263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436122953323005:2107] [node 5] ICP22 created new session: [2:7574436122953323035:2107]
2025-11-19T14:00:37.208269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436122953323035:2107] [node 5] ICS09 handshake done sender: [2:7574436122953323014:2112] self: [2:7574436122953323012:0] peer: [5:7574436122407029245:0] socket: 430 qp: -1
2025-11-19T14:00:37.208274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436122953323035:2107] [node 5] ICS10 traffic start
2025-11-19T14:00:37.208296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:37.208628Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122953323009:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.208641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436122953323009:2109] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.208810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436119302506162:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.208834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:37.208839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436119302506162:2049] [node 3] ICP22 created new session: [1:7574436123597473554:2049]
2025-11-19T14:00:37.208845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436123597473554:2049] [node 3] ICS09 handshake done sender: [1:7574436123597473537:2116] self: [1:7574436123597473523:0] peer: [3:7574436122242340054:0] socket: 446 qp: -1
2025-11-19T14:00:37.208850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436123597473554:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:37.208857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:37.209071Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436123597473548:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.209816Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436119768418965:2110] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:37.209984Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436119768418967:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.210116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436115473451606:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.210128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:37.210132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436115473451606:2050] [node 3] ICP22 created new session: [4:7574436119768418988:2050]
2025-11-19T14:00:37.210137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436119768418988:2050] [node 3] ICS09 handshake done sender: [4:7574436119768418979:2116] self: [4:7574436119768418982:0] peer: [3:7574436122242340042:0] socket: 443 qp: -1
2025-11-19T14:00:37.210142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436119768418988:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:37.210149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:37.210656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436115473451605:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.211238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436122407029249:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T14:00:37.211390Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436119768418987:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.211402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436115473451604:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.211412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:37.211416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436115473451604:2048] [node 1] ICP22 created new session: [4:7574436119768418992:2048]
2025-11-19T14:00:37.211422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436119768418992:2048] [node 1] ICS09 handshake done sender: [4:7574436119768418965:2110] self: [4:7574436119768418981:0] peer: [1:7574436123597473521:0] socket: 415 qp: -1
2025-11-19T14:00:37.211426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436119768418992:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:37.211441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:37.212677Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436122242340051:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:37.212419Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436123597473535:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.212708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436123597473519:2107] [node 4] ICP18 handshake (actor [1:7574436123597473535:2115]) is too old
2025-11-19T14:00:37.212899Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436122461003230:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.212943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436122461003214:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.212771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436117947372670:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.212786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:37.212791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436117947372670:2048] [node 1] ICP22 created new session: [3:7574436122242340062:2048]
2025-11-19T14:00:37.212797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436122242340062:2048] [node 1] ICS09 handshake done sender: [3:7574436122242340051:2116] self: [3:7574436122242340054:0] peer: [1:7574436123597473523:0] socket: 447 qp: -1
2025-11-19T14:00:37.212803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436122242340062:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:37.212808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.220150Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122953323029:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.220186Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122953323022:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.220218Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122953323017:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.220222Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436122953323017:2113] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.222647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436122461003243:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40690
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.224164Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436122953323017:2113] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:37.224369Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436122953323009:2109] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:37.224139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436122461003244:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40690
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.224462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436122953323015:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.224485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:37.224491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436122953323015:2111] [node 4] ICP22 created new session: [2:7574436122953323041:2111]
2025-11-19T14:00:37.224498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436122953323041:2111] [node 4] ICS09 handshake done sender: [2:7574436122953323017:2113] self: [2:7574436122953323016:0] peer: [4:7574436119768418990:0] socket: 431 qp: -1
2025-11-19T14:00:37.224505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436122953323041:2111] [node 4] ICS10 traffic start
2025-11-19T14:00:37.224514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:37.224718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436122953323006:2108] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.224727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:37.224730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436122953323006:2108] [node 6] ICP22 created new session: [2:7574436122953323043:2108]
2025-11-19T14:00:37.224734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436122953323043:2108] [node 6] ICS09 handshake done sender: [2:7574436122953323009:2109] self: [2:7574436122953323008:0] peer: [6:7574436122461003241:0] socket: 418 qp: -1
2025-11-19T14:00:37.224738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436122953323043:2108] [node 6] ICS10 traffic start
2025-11-19T14:00:37.224743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:37.224767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436122242340064:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12285
2025-11-19T14:00:37.224957Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436122461003243:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.224996Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436122461003222:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:37.225051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436118166035853:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.225208Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436123597473544:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.225220Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436123597473544:2117] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.226754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436118166035854:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9393) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436122461003222:2113] held: no
2025-11-19T14:00:37.226992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436118166035854:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.227008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:37.227015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436118166035854:2049] [node 2] ICP22 created new session: [6:7574436122461003246:2049]
2025-11-19T14:00:37.227021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436122461003246:2049] [node 2] ICS09 handshake done sender: [6:7574436122461003222:2113] self: [6:7574436122461003241:0] peer: [2:7574436122953323008:0] socket: 426 qp: -1
2025-11-19T14:00:37.227029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436122461003246:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:37.227036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.227908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436115473451605:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9393) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436119768418967:2112] held: no
2025-11-19T14:00:37.228663Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436119768418972:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.228672Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436119768418972:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.230452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119768418995:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18712
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.231163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436122461003248:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40690
2025-11-19T14:00:37.231188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436122461003249:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40690
2025-11-19T14:00:37.231304Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436119768418972:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:37.228380Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436119302506162:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(19458823ns) {HandleHandshakeStatus:329(19458196ns) {DropIncomingHandshake:441(3966ns) DropOutgoingHandshake:456(83ns) SwitchToState:178(60ns) ProcessPendingSessionEvents:495(19378036ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19354046ns max# 19266378ns}}}
2025-11-19T14:00:37.231649Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436119768418967:2112] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:37.231515Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436122407029249:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.231566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436122407029213:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.231448Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436123597473544:2117] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:37.231502Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436122461003230:2117] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:37.232589Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122242340064:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.232632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436117947372671:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.231825Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436119768418984:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.231835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436119768418984:2118] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.232315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436119768418970:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.232333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:37.232338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436119768418970:2109] [node 6] ICP22 created new session: [4:7574436119768418997:2109]
2025-11-19T14:00:37.232344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436119768418997:2109] [node 6] ICS09 handshake done sender: [4:7574436119768418972:2114] self: [4:7574436119768418971:0] peer: [6:7574436122461003242:0] socket: 436 qp: -1
2025-11-19T14:00:37.232351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436119768418997:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:37.232361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:37.232807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436123597473533:2113] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.232826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:37.232838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436123597473533:2113] [node 6] ICP22 created new session: [1:7574436123597473558:2113]
2025-11-19T14:00:37.232843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436123597473558:2113] [node 6] ICS09 handshake done sender: [1:7574436123597473544:2117] self: [1:7574436123597473536:0] peer: [6:7574436122461003245:0] socket: 463 qp: -1
2025-11-19T14:00:37.232849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436123597473558:2113] [node 6] ICS10 traffic start
2025-11-19T14:00:37.232857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:37.232957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436115473451605:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.232970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:37.232975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436115473451605:2049] [node 2] ICP22 created new session: [4:7574436119768419000:2049]
2025-11-19T14:00:37.232980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436119768419000:2049] [node 2] ICS09 handshake done sender: [4:7574436119768418967:2112] self: [4:7574436119768418990:0] peer: [2:7574436122953323016:0] socket: 434 qp: -1
2025-11-19T14:00:37.232985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436119768419000:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:37.232991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:37.233367Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436122461003243:2121] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.233479Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436119768418984:2118] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:37.233504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436122407029251:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33084
2025-11-19T14:00:37.233557Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436122407029249:2122] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:37.233672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436118166035853:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.233694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:37.233700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436118166035853:2048] [node 1] ICP22 created new session: [6:7574436122461003250:2048]
2025-11-19T14:00:37.233706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436122461003250:2048] [node 1] ICS09 handshake done sender: [6:7574436122461003243:2121] self: [6:7574436122461003245:0] peer: [1:7574436123597473536:0] socket: 464 qp: -1
2025-11-19T14:00:37.233712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436122461003250:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:37.233719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:37.233700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436122407029213:2109] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.233717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:37.233723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436122407029213:2109] [node 4] ICP22 created new session: [5:7574436122407029252:2109]
2025-11-19T14:00:37.233728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436122407029252:2109] [node 4] ICS09 handshake done sender: [5:7574436122407029249:2122] self: [5:7574436122407029250:0] peer: [4:7574436119768418978:0] socket: 472 qp: -1
2025-11-19T14:00:37.233733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436122407029252:2109] [node 4] ICS10 traffic start
2025-11-19T14:00:37.233740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:37.234725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436119768418962:2103] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.234744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:37.234748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436119768418962:2103] [node 5] ICP22 created new session: [4:7574436119768419002:2103]
2025-11-19T14:00:37.234753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436119768419002:2103] [node 5] ICS09 handshake done sender: [4:7574436119768418984:2118] self: [4:7574436119768418978:0] peer: [5:7574436122407029250:0] socket: 471 qp: -1
2025-11-19T14:00:37.234758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436119768419002:2103] [node 5] ICS10 traffic start
2025-11-19T14:00:37.234764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:37.234678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436122461003214:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.234694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:37.234701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436122461003214:2108] [node 4] ICP22 created new session: [6:7574436122461003252:2108]
2025-11-19T14:00:37.234713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436122461003252:2108] [node 4] ICS09 handshake done sender: [6:7574436122461003230:2117] self: [6:7574436122461003242:0] peer: [4:7574436119768418971:0] socket: 442 qp: -1
2025-11-19T14:00:37.234721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436122461003252:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:37.234728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:37.234853Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122953323019:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.234867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436122953323019:2114] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.235912Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436122953323019:2114] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:37.235997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436118658355648:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.236018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:37.236023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436118658355648:2049] [node 3] ICP22 created new session: [2:7574436122953323046:2049]
2025-11-19T14:00:37.236029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436122953323046:2049] [node 3] ICS09 handshake done sender: [2:7574436122953323019:2114] self: [2:7574436122953323018:0] peer: [3:7574436122242340065:0] socket: 469 qp: -1
2025-11-19T14:00:37.236034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436122953323046:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:37.236041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.241217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436117947372671:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9393) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436122242340064:2119] held: no
2025-11-19T14:00:37.241400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436122242340066:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12285
2025-11-19T14:00:37.242036Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436122242340064:2119] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:37.242365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436117947372671:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.242387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:37.242395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436117947372671:2049] [node 2] ICP22 created new session: [3:7574436122242340067:2049]
2025-11-19T14:00:37.242403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436122242340067:2049] [node 2] ICS09 handshake done sender: [3:7574436122242340064:2119] self: [3:7574436122242340065:0] peer: [2:7574436122953323018:0] socket: 473 qp: -1
2025-11-19T14:00:37.242411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436122242340067:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:37.242418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:37.263735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:37.263860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436122953323031:2048] [node 1] ICS01 socket: 417 reason# EndOfStream
2025-11-19T14:00:37.263873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436118658355647:2048] [node 1] ICP30 unregister session Session# [2:7574436122953323031:2048] VirtualId# [2:7574436122953323024:0]
2025-11-19T14:00:37.263881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436122953323031:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.263913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:37.263935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:37.268173Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:37.268251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436122242340062:2048] [node 1] ICS01 socket: 447 reason# EndOfStream
2025-11-19T14:00:37.268258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436117947372670:2048] [node 1] ICP30 unregister session Session# [3:7574436122242340062:2048] VirtualId# [3:7574436122242340054:0]
2025-11-19T14:00:37.268264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436122242340062:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.268293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:37.268312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:37.274669Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:37.274768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436119768418992:2048] [node 1] ICS01 socket: 415 reason# EndOfStream
2025-11-19T14:00:37.274778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436115473451604:2048] [node 1] ICP30 unregister session Session# [4:7574436119768418992:2048] VirtualId# [4:7574436119768418981:0]
2025-11-19T14:00:37.274787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436119768418992:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.274828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:37.274852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:37.274887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:37.274946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436122407029243:2048] [node 1] ICS01 socket: 420 reason# EndOfStream
2025-11-19T14:00:37.274952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436118112061853:2048] [node 1] ICP30 unregister session Session# [5:7574436122407029243:2048] VirtualId# [5:7574436122407029233:0]
2025-11-19T14:00:37.274955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436122407029243:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.274979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:37.274998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:37.275025Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:37.275082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436122461003250:2048] [node 1] ICS01 socket: 464 reason# EndOfStream
2025-11-19T14:00:37.275086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436118166035853:2048] [node 1] ICP30 unregister session Session# [6:7574436122461003250:2048] VirtualId# [6:7574436122461003245:0]
2025-11-19T14:00:37.275090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436122461003250:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.275118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:37.275138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:37.283815Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:37.283921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436122242340067:2049] [node 2] ICS01 socket: 473 reason# EndOfStream
2025-11-19T14:00:37.283936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436117947372671:2049] [node 2] ICP30 unregister session Session# [3:7574436122242340067:2049] VirtualId# [3:7574436122242340065:0]
2025-11-19T14:00:37.283945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436122242340067:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.283983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:37.284011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:37.296909Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:37.297027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436122461003246:2049] [node 2] ICS01 socket: 426 reason# EndOfStream
2025-11-19T14:00:37.297039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436118166035854:2049] [node 2] ICP30 unregister session Session# [6:7574436122461003246:2049] VirtualId# [6:7574436122461003241:0]
2025-11-19T14:00:37.297047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436122461003246:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.297091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:37.297114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:37.297230Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:37.297296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436119768419000:2049] [node 2] ICS01 socket: 434 reason# EndOfStream
2025-11-19T14:00:37.297300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436115473451605:2049] [node 2] ICP30 unregister session Session# [4:7574436119768419000:2049] VirtualId# [4:7574436119768418990:0]
2025-11-19T14:00:37.297305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436119768419000:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.297335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:37.297354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:37.302141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:37.302238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436122407029247:2049] [node 2] ICS01 socket: 451 reason# EndOfStream
2025-11-19T14:00:37.302247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436118112061854:2049] [node 2] ICP30 unregister session Session# [5:7574436122407029247:2049] VirtualId# [5:7574436122407029245:0]
2025-11-19T14:00:37.302254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436122407029247:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.302304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:37.302327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:37.319867Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:37.319984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [5:7574436122407029239:2050] [node 3] ICS07 socket disconnect 450 reason# EndOfStream
2025-11-19T14:00:37.319996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436122407029239:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.320040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:37.320063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [5:7574436122407029239:2050] [node 3] ICS15 start handshake
2025-11-19T14:00:37.320657Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436118112061855:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64815) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:37.320676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436122407029239:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:37.320682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436118112061855:2050] [node 3] ICP30 unregister session Session# [5:7574436122407029239:2050] VirtualId# [5:7574436122407029232:0]
2025-11-19T14:00:37.320737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:37.320744Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7574436118112061855:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64815) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:37.320747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64815) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:37.321654Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:37.321733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436122461003235:2050] [node 3] ICS01 socket: 411 reason# EndOfStream
2025-11-19T14:00:37.321740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436118166035855:2050] [node 3] ICP30 unregister session Session# [6:7574436122461003235:2050] VirtualId# [6:7574436122461003229:0]
2025-11-19T14:00:37.321746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436122461003235:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.321798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:37.321822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:37.325499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7574436118112061855:2050] [node 3] ICP33 wake up from error state
2025-11-19T14:00:37.338686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:37.338793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436122461003252:2108] [node 4] ICS01 socket: 442 reason# EndOfStream
2025-11-19T14:00:37.338804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436122461003214:2108] [node 4] ICP30 unregister session Session# [6:7574436122461003252:2108] VirtualId# [6:7574436122461003242:0]
2025-11-19T14:00:37.338812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436122461003252:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.338867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:37.338894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
2025-11-19T14:00:37.378206Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-11-19T14:00:37.378332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436122461003239:2099] [node 5] ICS01 socket: 427 reason# EndOfStream
2025-11-19T14:00:37.378344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436122461003202:2099] [node 5] ICP30 unregister session Session# [6:7574436122461003239:2099] VirtualId# [6:7574436122461003233:0]
2025-11-19T14:00:37.378352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436122461003239:2099] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.378412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [5] disconnected
2025-11-19T14:00:37.378436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-11-19T14:00:37.556202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436120413225697:2048] [node 2] ICP01 ready to work
2025-11-19T14:00:37.556219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436120413225698:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:37.571989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436122094751827:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:37.572007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436122094751828:2049] [node 3] ICP01 ready to work
2025-11-19T14:00:37.623964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436122766051550:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:37.624250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436122766051549:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:37.659877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436119898753183:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:37.659904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436119898753184:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:37.659911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436119898753185:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:37.704049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436120050619340:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:37.704066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436120050619342:2050] [node 3] ICP01 ready to work
2025-11-19T14:00:37.704091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436120050619341:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:37.738163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436120921961143:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:37.738178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436120921961144:2049] [node 2] ICP01 ready to work
2025-11-19T14:00:37.738185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436120921961145:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.781419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436120413225759:2106] [node 6] ICP01 ready to work
2025-11-19T14:00:37.782835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436120921961200:2102] [node 5] ICP01 ready to work
2025-11-19T14:00:37.783343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436120921961199:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17959
2025-11-19T14:00:37.783960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436120050619397:2103] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.784309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436122766051607:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29545
2025-11-19T14:00:37.784702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436119898753245:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.790723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436122766051612:2107] [node 6] ICP01 ready to work
2025-11-19T14:00:37.790846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436122766051613:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:37.790913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7574436122766051614:2109] [node 5] ICP01 ready to work
2025-11-19T14:00:37.792467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436122766051615:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29545
2025-11-19T14:00:37.784655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436120413225769:2111] [node 4] ICP01 ready to work
2025-11-19T14:00:37.800622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436120413225773:2113] [node 5] ICP01 ready to work
2025-11-19T14:00:37.801202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436120050619398:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-19T14:00:37.801747Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122766051607:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.801798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436122766051549:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.804520Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436120413225758:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.804538Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436120413225758:2107] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.805215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7574436119898753251:2110] [node 6] ICP01 ready to work
2025-11-19T14:00:37.804634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7574436120050619406:2108] [node 4] ICP01 ready to work
2025-11-19T14:00:37.807985Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436120921961199:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.811892Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436120921961143:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(24752861ns) {Configure:104(24752446ns) {ProcessConfigured:135(24558622ns) {StartInitialHandshake:157(24558035ns) {DropHandshakes:469(168ns) {DropIncomingHandshake:441(21ns) DropOutgoingHandshake:456(16ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:476(1832ns) {SwitchToState:178(582ns)}}}}}
2025-11-19T14:00:37.811944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436120921961143:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.812459Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436120050619398:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.812986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436120050619397:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(28910593ns) {Configure:104(28909651ns) {ProcessConfigured:135(28508062ns) {StartInitialHandshake:157(28507289ns) {DropHandshakes:469(1823ns) {DropIncomingHandshake:441(237ns) DropOutgoingHandshake:456(350ns)} GenerateSessionVirtualId:988(179ns) PrepareNewSessionHandshake:476(926ns) {SwitchToState:178(806ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.813644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436120413225775:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4216
2025-11-19T14:00:37.813699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436120413225776:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4216
2025-11-19T14:00:37.813710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436120413225777:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4216
2025-11-19T14:00:37.818367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436122094751891:2106] [node 5] ICP01 ready to work
2025-11-19T14:00:37.818590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436122094751894:2109] [node 6] ICP01 ready to work
2025-11-19T14:00:37.819270Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436120413225761:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.819290Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436120413225761:2108] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.819430Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436120413225775:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.820410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436120050619411:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-19T14:00:37.820716Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436120050619411:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.820696Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122766051615:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.821185Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436120413225777:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.823015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436122094751892:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.824766Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436120413225758:2107] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:37.824442Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [5:7574436120050619406:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:261(19735790ns) {Configure:104(19735159ns) {ProcessConfigured:135(19417746ns) {StartInitialHandshake:157(19417066ns) {DropHandshakes:469(619ns) {DropIncomingHandshake:441(63ns) DropOutgoingHandshake:456(58ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:476(690ns) {SwitchToState:178(640ns)}}}}}
2025-11-19T14:00:37.824495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436120050619406:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.824601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436122766051633:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29545
2025-11-19T14:00:37.824644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436122766051634:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29545
2025-11-19T14:00:37.825004Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436122766051607:2105] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:37.824738Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [4:7574436119898753251:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:261(19469290ns) {Configure:104(19468181ns) {ProcessConfigured:135(19152127ns) {StartInitialHandshake:157(19151532ns) {DropHandshakes:469(592ns) {DropIncomingHandshake:441(55ns) DropOutgoingHandshake:456(64ns)} GenerateSessionVirtualId:988(86ns) PrepareNewSessionHandshake:476(951ns) {SwitchToState:178(374ns)}}}}}
2025-11-19T14:00:37.825383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436120413225698:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.825408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:37.825414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436120413225698:2049] [node 3] ICP22 created new session: [1:7574436120413225780:2049]
2025-11-19T14:00:37.825422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436120413225780:2049] [node 3] ICS09 handshake done sender: [1:7574436120413225758:2107] self: [1:7574436120413225757:0] peer: [3:7574436122766051630:0] socket: 414 qp: -1
2025-11-19T14:00:37.825430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436120413225780:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:37.825448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.829385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436122094751893:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4361
2025-11-19T14:00:37.829477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436122094751897:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4361
2025-11-19T14:00:37.829509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436122094751898:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4361
2025-11-19T14:00:37.831919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436122766051632:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29545
2025-11-19T14:00:37.834326Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436119898753247:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.834351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436119898753247:2109] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.837012Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436122094751828:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:261(18322903ns) {Configure:104(18321463ns) {ProcessConfigured:135(17947712ns) {StartInitialHandshake:157(17946722ns) {DropHandshakes:469(1209ns) {DropIncomingHandshake:441(195ns) DropOutgoingHandshake:456(42ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:476(1437ns) {SwitchToState:178(570ns)}}}}}
2025-11-19T14:00:37.837092Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122094751898:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.839074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436122094751827:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:261(40001135ns) {Configure:104(39999582ns) {ProcessConfigured:135(39543572ns) {StartInitialHandshake:157(39542632ns) {DropHandshakes:469(1534ns) {DropIncomingHandshake:441(212ns) DropOutgoingHandshake:456(191ns)} GenerateSessionVirtualId:988(408ns) PrepareNewSessionHandshake:476(831ns) {SwitchToState:178(578ns)}}}}}
2025-11-19T14:00:37.839400Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122766051633:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.839584Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122766051632:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.840258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436120921961219:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17959
2025-11-19T14:00:37.840253Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122094751897:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.840277Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122094751893:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.840309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436122094751827:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.840468Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436120413225761:2108] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:37.840626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436120921961217:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17959
2025-11-19T14:00:37.841046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436120921961218:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17959
2025-11-19T14:00:37.844052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436120921961218:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.844112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436120921961200:2102] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.848486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7574436120921961223:2117] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.850825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [2:7574436122094751891:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(27705918ns) {Configure:104(27704850ns) {ProcessConfigured:135(27340909ns) {StartInitialHandshake:157(27340035ns) {DropHandshakes:469(893ns) {DropIncomingHandshake:441(77ns) DropOutgoingHandshake:456(52ns)} GenerateSessionVirtualId:988(336ns) PrepareNewSessionHandshake:476(1698ns) {SwitchToState:178(975ns)}}}}}
2025-11-19T14:00:37.848215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436120413225698:2049] [node 3] ICP35 event processing took too much time PendingConnection:278(22720687ns) {HandleHandshakeStatus:329(22719637ns) {DropIncomingHandshake:441(3706ns) DropOutgoingHandshake:456(475ns) SwitchToState:178(64ns) ProcessPendingSessionEvents:495(16017231ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 15996073ns max# 15902038ns}}}
2025-11-19T14:00:37.848296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436120413225776:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.848323Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436120413225766:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.848326Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436120413225766:2110] [node 2] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.851182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436120050619422:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-19T14:00:37.848346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119898753261:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31250
2025-11-19T14:00:37.848370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119898753262:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31250
2025-11-19T14:00:37.852067Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436120050619414:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.852081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [5:7574436120050619414:2114] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.852153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436122766051549:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.852188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:37.852197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436122766051549:2048] [node 1] ICP22 created new session: [3:7574436122766051637:2048]
2025-11-19T14:00:37.852205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436122766051637:2048] [node 1] ICS09 handshake done sender: [3:7574436122766051607:2105] self: [3:7574436122766051630:0] peer: [1:7574436120413225757:0] socket: 419 qp: -1
2025-11-19T14:00:37.852214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436122766051637:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:37.852237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:37.852853Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436119898753247:2109] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:37.855903Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436120921961217:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.858973Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122766051617:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.858984Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436122766051617:2111] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.856387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436120921961220:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17959
2025-11-19T14:00:37.855862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [1:7574436120413225773:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:261(37981396ns) {Configure:104(37979657ns) {ProcessConfigured:135(37440689ns) {StartInitialHandshake:157(37440062ns) {DropHandshakes:469(728ns) {DropIncomingHandshake:441(57ns) DropOutgoingHandshake:456(63ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:476(758ns) {SwitchToState:178(240ns)}}}}}
2025-11-19T14:00:37.855916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436120413225759:2106] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.855934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:37.855942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436120413225759:2106] [node 6] ICP22 created new session: [1:7574436120413225785:2106]
2025-11-19T14:00:37.855952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436120413225785:2106] [node 6] ICS09 handshake done sender: [1:7574436120413225761:2108] self: [1:7574436120413225760:0] peer: [6:7574436120921961214:0] socket: 415 qp: -1
2025-11-19T14:00:37.855963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436120413225785:2106] [node 6] ICS10 traffic start
2025-11-19T14:00:37.856002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:37.859877Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436122766051617:2111] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:37.864038Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436120050619411:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:37.866488Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436120921961199:2103] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:37.867325Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436120413225766:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:37.868007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436120413225697:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.868024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:37.868031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436120413225697:2048] [node 2] ICP22 created new session: [1:7574436120413225790:2048]
2025-11-19T14:00:37.868038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436120413225790:2048] [node 2] ICS09 handshake done sender: [1:7574436120413225766:2110] self: [1:7574436120413225764:0] peer: [2:7574436122094751910:0] socket: 418 qp: -1
2025-11-19T14:00:37.868044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436120413225790:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:37.868052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:37.870554Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436119898753262:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.870613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436119898753183:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.871253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436120921961145:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.871304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436120921961143:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.871313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:37.871320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436120921961143:2048] [node 1] ICP22 created new session: [6:7574436120921961229:2048]
2025-11-19T14:00:37.871325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436120921961229:2048] [node 1] ICS09 handshake done sender: [6:7574436120921961199:2103] self: [6:7574436120921961214:0] peer: [1:7574436120413225760:0] socket: 416 qp: -1
2025-11-19T14:00:37.871329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436120921961229:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:37.871337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:37.871023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436119898753245:2108] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.871041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:37.871051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436119898753245:2108] [node 5] ICP22 created new session: [4:7574436119898753266:2108]
2025-11-19T14:00:37.871057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436119898753266:2108] [node 5] ICS09 handshake done sender: [4:7574436119898753247:2109] self: [4:7574436119898753246:0] peer: [5:7574436120050619421:0] socket: 423 qp: -1
2025-11-19T14:00:37.871065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436119898753266:2108] [node 5] ICS10 traffic start
2025-11-19T14:00:37.871077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:37.867102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436120050619406:2108] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.867127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:37.867134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436120050619406:2108] [node 4] ICP22 created new session: [5:7574436120050619423:2108]
2025-11-19T14:00:37.867140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436120050619423:2108] [node 4] ICS09 handshake done sender: [5:7574436120050619411:2112] self: [5:7574436120050619421:0] peer: [4:7574436119898753246:0] socket: 428 qp: -1
2025-11-19T14:00:37.867151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436120050619423:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:37.867165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.876390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436120050619425:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-19T14:00:37.877252Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436120050619414:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:37.877371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7574436120050619397:2103] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.877392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:37.877398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436120050619397:2103] [node 6] ICP22 created new session: [5:7574436120050619428:2103]
2025-11-19T14:00:37.877405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436120050619428:2103] [node 6] ICS09 handshake done sender: [5:7574436120050619414:2114] self: [5:7574436120050619399:0] peer: [6:7574436120921961221:0] socket: 437 qp: -1
2025-11-19T14:00:37.877409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436120050619428:2103] [node 6] ICS10 traffic start
2025-11-19T14:00:37.877433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
2025-11-19T14:00:37.876550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436122766051612:2107] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.876583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:37.876592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436122766051612:2107] [node 6] ICP22 created new session: [3:7574436122766051641:2107]
2025-11-19T14:00:37.876601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436122766051641:2107] [node 6] ICS09 handshake done sender: [3:7574436122766051617:2111] self: [3:7574436122766051616:0] peer: [6:7574436120921961226:0] socket: 425 qp: -1
2025-11-19T14:00:37.876610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436122766051641:2107] [node 6] ICS10 traffic start
2025-11-19T14:00:37.876623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.879659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436122766051640:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29545
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.880328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436122094751912:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4361
2025-11-19T14:00:37.880709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436122094751913:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4361
2025-11-19T14:00:37.880986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [3:7574436122766051549:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(48863475ns) {HandleHandshakeStatus:329(48862180ns) {DropIncomingHandshake:441(851ns) DropOutgoingHandshake:456(3564ns) SwitchToState:178(312ns) ProcessPendingSessionEvents:495(19755937ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 19741622ns max# 19680262ns}}}
2025-11-19T14:00:37.881355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7574436122766051549:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:37.883607Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436119898753261:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.883657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436119898753184:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.883520Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436122094751893:2110] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:37.883918Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122094751902:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.883926Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436122094751902:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.884644Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436122094751902:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:37.884732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436122094751892:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.884754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:37.884760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436122094751892:2108] [node 4] ICP22 created new session: [2:7574436122094751915:2108]
2025-11-19T14:00:37.884767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436122094751915:2108] [node 4] ICS09 handshake done sender: [2:7574436122094751902:2114] self: [2:7574436122094751901:0] peer: [4:7574436119898753268:0] socket: 446 qp: -1
2025-11-19T14:00:37.884775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436122094751915:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:37.884796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:37.885408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436122094751827:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.885424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:37.885429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436122094751827:2048] [node 1] ICP22 created new session: [2:7574436122094751918:2048]
2025-11-19T14:00:37.885435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436122094751918:2048] [node 1] ICS09 handshake done sender: [2:7574436122094751893:2110] self: [2:7574436122094751910:0] peer: [1:7574436120413225764:0] socket: 421 qp: -1
2025-11-19T14:00:37.885440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436122094751918:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:37.885470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:37.888002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436119898753184:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436119898753261:2115] held: no
2025-11-19T14:00:37.891234Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122766051640:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.891299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7574436122766051550:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.891511Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122094751905:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.891525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436122094751905:2115] [node 3] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.892344Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436122094751905:2115] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:37.892456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436122094751828:2049] [node 3] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.892481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:37.892488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436122094751828:2049] [node 3] ICP22 created new session: [2:7574436122094751922:2049]
2025-11-19T14:00:37.892496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436122094751922:2049] [node 3] ICS09 handshake done sender: [2:7574436122094751905:2115] self: [2:7574436122094751895:0] peer: [3:7574436122766051645:0] socket: 449 qp: -1
2025-11-19T14:00:37.892502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436122094751922:2049] [node 3] ICS10 traffic start
2025-11-19T14:00:37.892510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.890955Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436120413225771:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.890974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436120413225771:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.893046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [3:7574436122766051646:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29545
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.893227Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436120413225759:2106] [node 6] ICP35 event processing took too much time PendingConnection:278(37230729ns) {HandleHandshakeStatus:329(37229558ns) {DropIncomingHandshake:441(189ns) DropOutgoingHandshake:456(263ns) SwitchToState:178(62ns) ProcessPendingSessionEvents:495(26823528ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 26806324ns max# 26756822ns}}}
2025-11-19T14:00:37.893270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436120413225793:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4216
2025-11-19T14:00:37.894114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436120413225759:2106] [node 6] ICP18 handshake (actor [1:7574436120413225776:2115]) is too old
2025-11-19T14:00:37.894479Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436120413225771:2112] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:37.894570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436120413225769:2111] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.894586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:37.894593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436120413225769:2111] [node 4] ICP22 created new session: [1:7574436120413225795:2111]
2025-11-19T14:00:37.894599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436120413225795:2111] [node 4] ICS09 handshake done sender: [1:7574436120413225771:2112] self: [1:7574436120413225770:0] peer: [4:7574436119898753265:0] socket: 451 qp: -1
2025-11-19T14:00:37.894606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436120413225795:2111] [node 4] ICS10 traffic start
2025-11-19T14:00:37.894614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:37.892917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436120921961228:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17959
2025-11-19T14:00:37.895197Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436120921961219:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.896208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436120050619430:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-19T14:00:37.896252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436120050619431:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-19T14:00:37.896672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [1:7574436120413225697:2048] [node 2] ICP35 event processing took too much time PendingConnection:278(28569979ns) {HandleHandshakeStatus:329(28569198ns) {DropIncomingHandshake:441(67ns) DropOutgoingHandshake:456(300ns) SwitchToState:178(341ns) ProcessPendingSessionEvents:495(28492026ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28468149ns max# 28391090ns}}}
2025-11-19T14:00:37.897208Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436120050619430:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.897250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436120050619342:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.897429Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122766051625:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.897439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436122766051625:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.898954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436120050619433:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-19T14:00:37.899203Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:261: Proxy [6:7574436120921961144:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:261(50145798ns) {Configure:104(50144271ns) {ProcessConfigured:135(49935654ns) {StartInitialHandshake:157(49934856ns) {DropHandshakes:469(327ns) {DropIncomingHandshake:441(16ns) DropOutgoingHandshake:456(28ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:476(1228ns) {SwitchToState:178(460ns)}}}}}
2025-11-19T14:00:37.899734Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436122766051625:2115] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:37.900071Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436120050619425:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.900101Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [5:7574436120050619431:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.900857Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436120050619430:2119] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:37.903970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [1:7574436120413225792:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4216
2025-11-19T14:00:37.905048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [5:7574436120050619406:2108] [node 4] ICP35 event processing took too much time PendingConnection:278(37856471ns) {HandleHandshakeStatus:329(37855851ns) {DropIncomingHandshake:441(257ns) DropOutgoingHandshake:456(2165ns) SwitchToState:178(781ns) ProcessPendingSessionEvents:495(33653051ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 33633964ns max# 33568144ns}}}
2025-11-19T14:00:37.905076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436120050619340:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.905088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7574436120050619341:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.905097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436120050619342:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.905112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:37.905117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436120050619342:2050] [node 3] ICP22 created new session: [5:7574436120050619436:2050]
2025-11-19T14:00:37.905122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436120050619436:2050] [node 3] ICS09 handshake done sender: [5:7574436120050619430:2119] self: [5:7574436120050619432:0] peer: [3:7574436122766051624:0] socket: 444 qp: -1
2025-11-19T14:00:37.905130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436120050619436:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:37.905138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:37.904520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436120921961145:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436120921961217:2113] held: no
2025-11-19T14:00:37.904728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436120921961200:2102] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24683) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436120921961218:2114] held: no
2025-11-19T14:00:37.904814Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436120921961217:2113] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:37.904345Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436120413225792:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.904391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7574436120413225769:2111] [node 4] ICP18 handshake (actor [1:7574436120413225792:2118]) is too old
2025-11-19T14:00:37.906132Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122094751911:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.906148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436122094751911:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.907100Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436122094751911:2117] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:37.907211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436122094751891:2106] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.907225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:37.907233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436122094751891:2106] [node 5] ICP22 created new session: [2:7574436122094751925:2106]
2025-11-19T14:00:37.907239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436122094751925:2106] [node 5] ICS09 handshake done sender: [2:7574436122094751911:2117] self: [2:7574436122094751896:0] peer: [5:7574436120050619435:0] socket: 471 qp: -1
2025-11-19T14:00:37.907245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436122094751925:2106] [node 5] ICS10 traffic start
2025-11-19T14:00:37.907253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:37.905690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436122766051614:2109] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.905711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:37.905716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436122766051614:2109] [node 5] ICP22 created new session: [3:7574436122766051648:2109]
2025-11-19T14:00:37.905721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436122766051648:2109] [node 5] ICS09 handshake done sender: [3:7574436122766051625:2115] self: [3:7574436122766051624:0] peer: [5:7574436120050619432:0] socket: 467 qp: -1
2025-11-19T14:00:37.905725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436122766051648:2109] [node 5] ICS10 traffic start
2025-11-19T14:00:37.905747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:37.906065Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436122766051550:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7574436122766051640:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.908211Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436122766051640:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:37.912029Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436120413225784:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.912052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [1:7574436120413225784:2117] [node 5] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.913606Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122094751913:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.913662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436122094751891:2106] [node 5] ICP18 handshake (actor [2:7574436122094751913:2119]) is too old
2025-11-19T14:00:37.915219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7574436120050619341:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7574436120050619431:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.918537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7574436122766051550:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.918557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:37.918565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436122766051550:2049] [node 2] ICP22 created new session: [3:7574436122766051650:2049]
2025-11-19T14:00:37.918571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436122766051650:2049] [node 2] ICS09 handshake done sender: [3:7574436122766051640:2120] self: [3:7574436122766051645:0] peer: [2:7574436122094751895:0] socket: 454 qp: -1
2025-11-19T14:00:37.918577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436122766051650:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:37.918584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.916860Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [6:7574436120921961143:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(45451198ns) {HandleHandshakeStatus:329(45450168ns) {DropIncomingHandshake:441(253ns) DropOutgoingHandshake:456(2057ns) SwitchToState:178(40ns) ProcessPendingSessionEvents:495(17421719ns) {unordered ForwardSessionEventToSession:612 num# 60 sum# 17401982ns max# 17340781ns}}}
2025-11-19T14:00:37.917154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436120921961145:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.917172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:37.917180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436120921961145:2050] [node 3] ICP22 created new session: [6:7574436120921961234:2050]
2025-11-19T14:00:37.917186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436120921961234:2050] [node 3] ICS09 handshake done sender: [6:7574436120921961217:2113] self: [6:7574436120921961226:0] peer: [3:7574436122766051616:0] socket: 429 qp: -1
2025-11-19T14:00:37.917194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436120921961234:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:37.917204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:37.917797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436120921961144:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.917832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436120921961231:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17959
2025-11-19T14:00:37.919384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119898753270:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31250
2025-11-19T14:00:37.919428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119898753271:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31250
2025-11-19T14:00:37.919933Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436119898753261:2115] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:37.920612Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [4:7574436119898753245:2108] [node 5] ICP35 event processing took too much time PendingConnection:278(64450929ns) {HandleHandshakeStatus:329(64450196ns) {DropIncomingHandshake:441(638ns) DropOutgoingHandshake:456(381ns) SwitchToState:178(62ns) ProcessPendingSessionEvents:495(21337316ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 21320626ns max# 21274191ns}}}
2025-11-19T14:00:37.920979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436120050619438:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-19T14:00:37.921488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436120921961232:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17959
2025-11-19T14:00:37.920949Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436119898753270:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.921277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119898753272:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31250
2025-11-19T14:00:37.924090Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436120413225784:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.924545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436120413225773:2113] [node 5] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.924564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:37.924571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436120413225773:2113] [node 5] ICP22 created new session: [1:7574436120413225798:2113]
2025-11-19T14:00:37.924578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436120413225798:2113] [node 5] ICS09 handshake done sender: [1:7574436120413225784:2117] self: [1:7574436120413225774:0] peer: [5:7574436120050619434:0] socket: 458 qp: -1
2025-11-19T14:00:37.924584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436120413225798:2113] [node 5] ICS10 traffic start
2025-11-19T14:00:37.924591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:37.924253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436119898753184:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.924275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:37.924283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436119898753184:2049] [node 2] ICP22 created new session: [4:7574436119898753273:2049]
2025-11-19T14:00:37.924290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436119898753273:2049] [node 2] ICS09 handshake done sender: [4:7574436119898753261:2115] self: [4:7574436119898753268:0] peer: [2:7574436122094751901:0] socket: 448 qp: -1
2025-11-19T14:00:37.924300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436119898753273:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:37.924308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:37.924566Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436119898753183:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30736) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436119898753262:2116] held: no
2025-11-19T14:00:37.924932Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436119898753262:2116] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:37.925315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7574436119898753185:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.925342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436119898753183:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.925351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:37.925357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436119898753183:2048] [node 1] ICP22 created new session: [4:7574436119898753276:2048]
2025-11-19T14:00:37.925363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436119898753276:2048] [node 1] ICS09 handshake done sender: [4:7574436119898753262:2116] self: [4:7574436119898753265:0] peer: [1:7574436120413225770:0] socket: 430 qp: -1
2025-11-19T14:00:37.925368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436119898753276:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:37.925389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:37.924040Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436120050619431:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:37.924186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436120050619341:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.924200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:37.924206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436120050619341:2049] [node 2] ICP22 created new session: [5:7574436120050619440:2049]
2025-11-19T14:00:37.924212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436120050619440:2049] [node 2] ICS09 handshake done sender: [5:7574436120050619431:2120] self: [5:7574436120050619435:0] peer: [2:7574436122094751896:0] socket: 473 qp: -1
2025-11-19T14:00:37.924223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436120050619440:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:37.924232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:37.925665Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [3:7574436122766051623:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.925677Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [3:7574436122766051623:2114] [node 4] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.926511Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [3:7574436122766051623:2114] [node 4] ICH04 handshake succeeded
2025-11-19T14:00:37.926595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7574436122766051613:2108] [node 4] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.926612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:37.926617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7574436122766051613:2108] [node 4] ICP22 created new session: [3:7574436122766051653:2108]
2025-11-19T14:00:37.926622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [3:7574436122766051653:2108] [node 4] ICS09 handshake done sender: [3:7574436122766051623:2114] self: [3:7574436122766051622:0] peer: [4:7574436119898753275:0] socket: 433 qp: -1
2025-11-19T14:00:37.926627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [3:7574436122766051653:2108] [node 4] ICS10 traffic start
2025-11-19T14:00:37.926634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:37.928312Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [6:7574436120921961232:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.928363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7574436120921961223:2117] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:37.928674Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122094751900:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.928694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [2:7574436122094751900:2113] [node 6] ICRDMA Non success qp response from remote side
2025-11-19T14:00:37.928846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [5:7574436120050619439:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27488
2025-11-19T14:00:37.929139Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436120921961218:2114] [node 5] ICH04 handshake succeeded
2025-11-19T14:00:37.932328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436120921961200:2102] [node 5] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.932355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-11-19T14:00:37.932366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436120921961200:2102] [node 5] ICP22 created new session: [6:7574436120921961240:2102]
2025-11-19T14:00:37.932375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436120921961240:2102] [node 5] ICS09 handshake done sender: [6:7574436120921961218:2114] self: [6:7574436120921961221:0] peer: [5:7574436120050619399:0] socket: 447 qp: -1
2025-11-19T14:00:37.932385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436120921961240:2102] [node 5] ICS10 traffic start
2025-11-19T14:00:37.932413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [5] connected
2025-11-19T14:00:37.932896Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [5:7574436120050619425:2118] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:37.932711Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [4:7574436119898753260:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.932728Z :INTERCONNECT ERROR: interconnect_handshake.cpp:954: Handshake [4:7574436119898753260:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.933206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [2:7574436122094751927:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4361
2025-11-19T14:00:37.934884Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436122094751900:2113] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:37.935202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7574436120050619340:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.935229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:37.935236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7574436120050619340:2048] [node 1] ICP22 created new session: [5:7574436120050619442:2048]
2025-11-19T14:00:37.935245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [5:7574436120050619442:2048] [node 1] ICS09 handshake done sender: [5:7574436120050619425:2118] self: [5:7574436120050619434:0] peer: [1:7574436120413225774:0] socket: 462 qp: -1
2025-11-19T14:00:37.935251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [5:7574436120050619442:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:37.935260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:37.936394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436122094751894:2109] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.936418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:37.936424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436122094751894:2109] [node 6] ICP22 created new session: [2:7574436122094751929:2109]
2025-11-19T14:00:37.936429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436122094751929:2109] [node 6] ICS09 handshake done sender: [2:7574436122094751900:2113] self: [2:7574436122094751899:0] peer: [6:7574436120921961237:0] socket: 445 qp: -1
2025-11-19T14:00:37.936435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436122094751929:2109] [node 6] ICS10 traffic start
2025-11-19T14:00:37.936441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.937720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436120921961242:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17959
2025-11-19T14:00:37.937820Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436122094751927:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:37.937856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7574436122094751894:2109] [node 6] ICP18 handshake (actor [2:7574436122094751927:2120]) is too old
2025-11-19T14:00:37.938901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436120921961144:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2321) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7574436120921961219:2115] held: no
2025-11-19T14:00:37.939011Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436120921961219:2115] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:37.939146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436120921961144:2049] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.939158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:37.939163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436120921961144:2049] [node 2] ICP22 created new session: [6:7574436120921961243:2049]
2025-11-19T14:00:37.939168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436120921961243:2049] [node 2] ICS09 handshake done sender: [6:7574436120921961219:2115] self: [6:7574436120921961237:0] peer: [2:7574436122094751899:0] socket: 452 qp: -1
2025-11-19T14:00:37.939173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436120921961243:2049] [node 2] ICS10 traffic start
2025-11-19T14:00:37.939179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:37.942501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7574436119898753185:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7574436119898753270:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.02025-11-19T14:00:37.943975Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436119898753260:2114] [node 6] ICH04 handshake succeeded
2025-11-19T14:00:37.944105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [6:7574436120921961245:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17959
2025-11-19T14:00:37.944223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7574436119898753251:2110] [node 6] ICP20 outgoing handshake succeeded
2025-11-19T14:00:37.944237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-11-19T14:00:37.944243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436119898753251:2110] [node 6] ICP22 created new session: [4:7574436119898753281:2110]
2025-11-19T14:00:37.944252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436119898753281:2110] [node 6] ICS09 handshake done sender: [4:7574436119898753260:2114] self: [4:7574436119898753252:0] peer: [6:7574436120921961239:0] socket: 438 qp: -1
2025-11-19T14:00:37.944260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436119898753281:2110] [node 6] ICS10 traffic start
2025-11-19T14:00:37.944269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.948064Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [6:7574436120921961232:2121] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-11-19T14:00:37.948331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7574436120921961223:2117] [node 4] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.948352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-11-19T14:00:37.948361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7574436120921961223:2117] [node 4] ICP22 created new session: [6:7574436120921961246:2117]
2025-11-19T14:00:37.948369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [6:7574436120921961246:2117] [node 4] ICS09 handshake done sender: [6:7574436120921961232:2121] self: [6:7574436120921961239:0] peer: [4:7574436119898753252:0] socket: 439 qp: -1
2025-11-19T14:00:37.948379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [6:7574436120921961246:2117] [node 4] ICS10 traffic start
2025-11-19T14:00:37.948393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [4] connected
2025-11-19T14:00:37.948507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119898753279:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31250
2025-11-19T14:00:37.949256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1389: Handshake [4:7574436119898753280:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31250
2025-11-19T14:00:37.949758Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [4:7574436119898753270:2117] [node 3] ICH04 handshake succeeded
2025-11-19T14:00:37.951873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7574436119898753185:2050] [node 3] ICP19 incoming handshake succeeded
2025-11-19T14:00:37.951897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-11-19T14:00:37.951907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7574436119898753185:2050] [node 3] ICP22 created new session: [4:7574436119898753283:2050]
2025-11-19T14:00:37.951915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [4:7574436119898753283:2050] [node 3] ICS09 handshake done sender: [4:7574436119898753270:2117] self: [4:7574436119898753275:0] peer: [3:7574436122766051622:0] socket: 466 qp: -1
2025-11-19T14:00:37.951924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [4:7574436119898753283:2050] [node 3] ICS10 traffic start
2025-11-19T14:00:37.951937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [3] connected
2025-11-19T14:00:37.953796Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7574436119898753251:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:37.985065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:37.985197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436122094751918:2048] [node 1] ICS01 socket: 421 reason# EndOfStream
2025-11-19T14:00:37.985209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436122094751827:2048] [node 1] ICP30 unregister session Session# [2:7574436122094751918:2048] VirtualId# [2:7574436122094751910:0]
2025-11-19T14:00:37.985220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436122094751918:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.985263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:37.985288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:37.985794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-11-19T14:00:37.985874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [3:7574436122766051637:2048] [node 1] ICS07 socket disconnect 419 reason# EndOfStream
2025-11-19T14:00:37.985880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436122766051637:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:37.985925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:37.985945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436122766051637:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:37.999355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436122766051549:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30736) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-11-19T14:00:37.999379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [3:7574436122766051637:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:38.004141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-11-19T14:00:38.004256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436120050619442:2048] [node 1] ICS01 socket: 462 reason# EndOfStream
2025-11-19T14:00:38.004268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436120050619340:2048] [node 1] ICP30 unregister session Session# [5:7574436120050619442:2048] VirtualId# [5:7574436120050619434:0]
2025-11-19T14:00:38.004277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436120050619442:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:38.004335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:38.004373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:38.006195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-11-19T14:00:38.006283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436119898753276:2048] [node 1] ICS01 socket: 430 reason# EndOfStream
2025-11-19T14:00:38.006291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436119898753183:2048] [node 1] ICP30 unregister session Session# [4:7574436119898753276:2048] VirtualId# [4:7574436119898753265:0]
2025-11-19T14:00:38.006298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436119898753276:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:38.006349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:38.006375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:38.006615Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-11-19T14:00:38.006678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [6:7574436120921961229:2048] [node 1] ICS07 socket disconnect 416 reason# EndOfStream
2025-11-19T14:00:38.006684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436120921961229:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:38.006713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:38.006734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [6:7574436120921961229:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:38.010981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7574436122766051549:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30736) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:38.011008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436122766051637:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:38.011014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436122766051549:2048] [node 1] ICP30 unregister session Session# [3:7574436122766051637:2048] VirtualId# [3:7574436122766051630:0]
2025-11-19T14:00:38.011081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:38.011090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7574436122766051549:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30736) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:38.011094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30736) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:38.011378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7574436120921961143:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30736) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:00:38.011393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436120921961229:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:00:38.011397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436120921961143:2048] [node 1] ICP30 unregister session Session# [6:7574436120921961229:2048] VirtualId# [6:7574436120921961214:0]
2025-11-19T14:00:38.011446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:38.011451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7574436120921961143:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30736) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:38.011453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30736) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:00:38.012743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-11-19T14:00:38.012822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [3:7574436122766051650:2049] [node 2] ICS01 socket: 454 reason# EndOfStream
2025-11-19T14:00:38.012828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7574436122766051550:2049] [node 2] ICP30 unregister session Session# [3:7574436122766051650:2049] VirtualId# [3:7574436122766051645:0]
2025-11-19T14:00:38.012834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [3:7574436122766051650:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:38.012874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:38.012894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:38.022295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7574436122766051549:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:38.022356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7574436120921961143:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:00:38.022536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-11-19T14:00:38.022670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436120050619440:2049] [node 2] ICS01 socket: 473 reason# EndOfStream
2025-11-19T14:00:38.022683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436120050619341:2049] [node 2] ICP30 unregister session Session# [5:7574436120050619440:2049] VirtualId# [5:7574436120050619435:0]
2025-11-19T14:00:38.022693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436120050619440:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:38.022744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:38.022770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:38.023881Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-11-19T14:00:38.023963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [4:7574436119898753273:2049] [node 2] ICS01 socket: 448 reason# EndOfStream
2025-11-19T14:00:38.023971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7574436119898753184:2049] [node 2] ICP30 unregister session Session# [4:7574436119898753273:2049] VirtualId# [4:7574436119898753268:0]
2025-11-19T14:00:38.023978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [4:7574436119898753273:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:38.024026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:38.024048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:38.024079Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-11-19T14:00:38.024135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436120921961243:2049] [node 2] ICS01 socket: 452 reason# EndOfStream
2025-11-19T14:00:38.024141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436120921961144:2049] [node 2] ICP30 unregister session Session# [6:7574436120921961243:2049] VirtualId# [6:7574436120921961237:0]
2025-11-19T14:00:38.024147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436120921961243:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:38.024171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:38.024190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:38.045579Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-11-19T14:00:38.045702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [5:7574436120050619436:2050] [node 3] ICS01 socket: 444 reason# EndOfStream
2025-11-19T14:00:38.045716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7574436120050619342:2050] [node 3] ICP30 unregister session Session# [5:7574436120050619436:2050] VirtualId# [5:7574436120050619432:0]
2025-11-19T14:00:38.045725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [5:7574436120050619436:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:38.045784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:38.045812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:38.050222Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-11-19T14:00:38.050333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436120921961234:2050] [node 3] ICS01 socket: 429 reason# EndOfStream
2025-11-19T14:00:38.050345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436120921961145:2050] [node 3] ICP30 unregister session Session# [6:7574436120921961234:2050] VirtualId# [6:7574436120921961226:0]
2025-11-19T14:00:38.050354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436120921961234:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:38.050415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [3] disconnected
2025-11-19T14:00:38.050441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [3] session destroyed
2025-11-19T14:00:38.078688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-11-19T14:00:38.078808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [6:7574436120921961246:2117] [node 4] ICS01 socket: 439 reason# EndOfStream
2025-11-19T14:00:38.078821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7574436120921961223:2117] [node 4] ICP30 unregister session Session# [6:7574436120921961246:2117] VirtualId# [6:7574436120921961239:0]
2025-11-19T14:00:38.078832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [6:7574436120921961246:2117] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:38.078896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [4] disconnected
2025-11-19T14:00:38.078926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
VERIFY failed (2025-11-19T14:00:38.165020Z): 
  ydb/library/actors/interconnect/ut/dynamic_proxy_ut.cpp:53
  Check(): requirement index == expected failed
NPrivate::InternalPanicImpl(int, char const*, char const*, int, int, int, TBasicStringBuf<char, std::__y1::char_traits<char>>, char const*, unsigned long)+438 (0xBDBBE6)
NPrivate::Panic(NPrivate::TStaticBuf const&, int, char const*, char const*, char const*, ...)+263 (0xBD2427)
TArriveQueue::Check()+771 (0xAB40A3)
RaceTestIter(unsigned int, unsigned int)+3022 (0xAB389E)
NTestSuiteDynamicProxy::TTestCaseRaceCheck10::Execute_(NUnitTest::TTestContext&)+67 (0xAB42C3)
NTestSuiteDynamicProxy::TCurrentTest::Execute()::'lambda'()::operator()() const+71 (0xAB5E47)
NUnitTest::TTestBase::Run(std::__y1::function<void ()>, TBasicString<char, std::__y1::char_traits<char>> const&, char const*, bool)+120 (0xC69E18)
NTestSuiteDynamicProxy::TCurrentTest::Execute()+429 (0xAB57ED)
NUnitTest::TTestFactory::Execute()+817 (0xC6A561)
NUnitTest::RunMain(int, char**)+3229 (0xC7C72D)
??+0 (0x7FBAA6D42D90)
__libc_start_main+128 (0x7FBAA6D42E40)
_start+41 (0xA3C029)
2025-11-19T14:00:44.604008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436150609062629:2048] [node 2] ICP01 ready to work
nodeId# 2 peerNodeId# 1 TEvPoisonSession
Subscribe
2025-11-19T14:00:44.627018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436151321205409:2048] [node 1] ICP01 ready to work
2025-11-19T14:00:44.696483Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436150609062638:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:44.696904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:44.697086Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436151321205419:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:44.787873Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436150609062638:2056] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:44.788173Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436151321205419:2056] [node 1] ICH04 handshake succeeded
TEvNodeConnected
2025-11-19T14:00:44.788646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:44.788680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:44.788688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436151321205422:2048]
2025-11-19T14:00:44.788695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436151321205422:2048] [node 1] ICS09 handshake done sender: [2:7574436151321205419:2056] self: [2:7574436151321205418:0] peer: [1:7574436150609062639:0] socket: 16 qp: 126161
2025-11-19T14:00:44.788711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436151321205422:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:44.788733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:44.790770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:44.790795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:44.790800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436150609062641:2048]
2025-11-19T14:00:44.790807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436150609062641:2048] [node 2] ICS09 handshake done sender: [1:7574436150609062638:2056] self: [1:7574436150609062639:0] peer: [2:7574436151321205418:0] socket: 17 qp: 126157
2025-11-19T14:00:44.790813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436150609062641:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:44.790832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:00:44.884043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436151321205422:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-19T14:00:44.884620Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:44.884712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436150609062641:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:44.884716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436150609062641:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:44.884747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:44.884761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436150609062641:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:44.884122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436151321205422:2048] VirtualId# [2:7574436151321205418:0]
2025-11-19T14:00:44.884131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436151321205422:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:44.884164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:44.884194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:00:44.891319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436151321205424:2057]) peer tries to resume nonexistent session Self# [1:7574436150609062639:0] Peer# [2:7574436151321205418:0]
2025-11-19T14:00:44.891350Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436151321205424:2057] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:44.891838Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436150609062643:2058] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:44.892378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:44.892411Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:44.892416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436150609062641:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:44.892463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436150609062641:2048] VirtualId# [1:7574436150609062639:0]
2025-11-19T14:00:44.892483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:44.915956Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436151321205428:2059] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:44.917055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:44.917334Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436150609062646:2059] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:44.917402Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436150609062648:2060] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:44.927862Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436151321205428:2059] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:44.927982Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436150609062646:2059] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:44.928088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:44.928110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:44.928118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436150609062651:2048]
2025-11-19T14:00:44.928124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436150609062651:2048] [node 2] ICS09 handshake done sender: [1:7574436150609062646:2059] self: [1:7574436150609062645:0] peer: [2:7574436151321205429:0] socket: 17 qp: 126295
2025-11-19T14:00:44.928131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436150609062651:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:44.928144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:44.932095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:44.932122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:44.932130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436151321205431:2048]
2025-11-19T14:00:44.932138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436151321205431:2048] [node 1] ICS09 handshake done sender: [2:7574436151321205428:2059] self: [2:7574436151321205429:0] peer: [1:7574436150609062645:0] socket: 18 qp: 126298
2025-11-19T14:00:44.932146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436151321205431:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:44.932157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:44.932242Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436151321205409:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:00:45.108514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:00:45.108574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [1 <-> 2] socket: Broken pipe
2025-11-19T14:00:45.108583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436150609062651:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:00:45.108588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436150609062651:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:00:45.108595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:45.108807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:45.108914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436150609062651:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:45.108917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436150609062651:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:45.110305Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:45.110376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436151321205431:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-19T14:00:45.110382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436151321205431:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:45.110411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:45.110426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436151321205431:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:45.113966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436151321205431:2048] [node 1] ICS08 incoming handshake Self# [1:7574436150609062645:0] Peer# [2:7574436151321205429:0] Counter# 680 LastInputSerial# 16
2025-11-19T14:00:45.116991Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436154904029949:2061] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:45.117101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:45.117118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436150609062651:2048] [node 2] ICS09 handshake done sender: [1:7574436154904029949:2061] self: [1:7574436150609062645:0] peer: [2:7574436151321205429:0] socket: 16 qp: 126391
2025-11-19T14:00:45.117425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436150609062651:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:45.117440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:45.137233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436150609062651:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:00:45.137247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436150609062651:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:00:45.137276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:45.137292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436150609062651:2048] [node 2] ICS08 incoming handshake Self# [2:7574436151321205429:0] Peer# [1:7574436150609062645:0] Counter# 16 LastInputSerial# 680
2025-11-19T14:00:45.137419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:45.137506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436150609062651:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:45.140800Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436155616172730:2062] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:45.140964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:45.140978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436151321205431:2048] [node 1] ICS09 handshake done sender: [2:7574436155616172730:2062] self: [2:7574436151321205429:0] peer: [1:7574436150609062645:0] socket: 19 qp: 126392
2025-11-19T14:00:45.141381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436151321205431:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:45.141394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:45.141493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:45.141541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436151321205431:2048] [node 1] ICS01 socket: 19 reason# EndOfStream
2025-11-19T14:00:45.141565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436151321205431:2048] VirtualId# [2:7574436151321205429:0]
2025-11-19T14:00:45.141570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436151321205431:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:45.141589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:45.141607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:00:45.151188Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436154904029955:2064] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:45.151223Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:45.151228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436150609062651:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:45.151254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436150609062651:2048] VirtualId# [1:7574436150609062645:0]
2025-11-19T14:00:45.151273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:45.151280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:45.152675Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436155616172736:2064] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:45.154659Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436154904029955:2064] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:45.154725Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436155616172736:2064] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:45.154794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:45.154808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:45.154813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436155616172739:2048]
2025-11-19T14:00:45.154819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436155616172739:2048] [node 1] ICS09 handshake done sender: [2:7574436155616172736:2064] self: [2:7574436155616172735:0] peer: [1:7574436154904029956:0] socket: 17 qp: 126423
2025-11-19T14:00:45.154825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436155616172739:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:45.154832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:45.155313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:45.155328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:45.155333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436154904029958:2048]
2025-11-19T14:00:45.155339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436154904029958:2048] [node 2] ICS09 handshake done sender: [1:7574436154904029955:2064] self: [1:7574436154904029956:0] peer: [2:7574436155616172735:0] socket: 18 qp: 126422
2025-11-19T14:00:45.155343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436154904029958:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:45.155349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:00:45.341109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436154904029958:2048] [node 2] ICS01 socket: 18 reason# Debug
2025-11-19T14:00:45.341224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436154904029958:2048] VirtualId# [1:7574436154904029956:0]
2025-11-19T14:00:45.341234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436154904029958:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:45.341263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:45.341279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:45.352209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:45.352300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436155616172739:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:45.352308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436155616172739:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:45.352337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:45.352353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436155616172739:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:45.369909Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436155616172741:2065] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:00:45.369427Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436154904029960:2066]) peer tries to resume nonexistent session Self# [2:7574436155616172735:0] Peer# [1:7574436154904029956:0]
2025-11-19T14:00:45.369453Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436154904029960:2066] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:45.375950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:45.375995Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:45.376000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436155616172739:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:45.376078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436155616172739:2048] VirtualId# [2:7574436155616172735:0]
2025-11-19T14:00:45.376125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 378
TEvUndelivered Cookie# 379
TEvUndelivered Cookie# 380
TEvUndelivered Cookie# 381
TEvUndelivered Cookie# 382
TEvUndelivered Cookie# 383
TEvUndelivered Cookie# 384
TEvUndelivered Cookie# 385
TEvUndelivered Cookie# 386
TEvUndelivered Cookie# 387
2025-11-19T14:00:45.383920Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436154904029961:2067] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:45.383974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:45.384168Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436155616172744:2066] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:45.401251Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436154904029961:2067] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:45.401839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:45.401856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:45.401862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436154904029964:2048]
2025-11-19T14:00:45.401870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436154904029964:2048] [node 2] ICS09 handshake done sender: [1:7574436154904029961:2067] self: [1:7574436154904029962:0] peer: [2:7574436155616172743:0] socket: 17 qp: 126510
2025-11-19T14:00:45.401876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436154904029964:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:45.401885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:45.401404Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436155616172744:2066] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:45.401517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:45.401534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:45.401546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436155616172747:2048]
2025-11-19T14:00:45.401555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436155616172747:2048] [node 1] ICS09 handshake done sender: [2:7574436155616172744:2066] self: [2:7574436155616172743:0] peer: [1:7574436154904029962:0] socket: 16 qp: 126512
2025-11-19T14:00:45.401563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436155616172747:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:45.401574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:00:45.527942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436155616172747:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-19T14:00:45.528234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436155616172747:2048] VirtualId# [2:7574436155616172743:0]
2025-11-19T14:00:45.528245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436155616172747:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:45.528275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:45.528298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:45.528846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:45.528930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436154904029964:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:45.528936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436154904029964:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:45.528969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:45.528983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436154904029964:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-19T14:00:45.544102Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436155616172749:2067]) peer tries to resume nonexistent session Self# [1:7574436154904029962:0] Peer# [2:7574436155616172743:0]
2025-11-19T14:00:45.545300Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436155616172749:2067] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:45.546424Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436154904029966:2069] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:45.547213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:45.547259Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:45.547265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436154904029964:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:45.547331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436154904029964:2048] VirtualId# [1:7574436154904029962:0]
2025-11-19T14:00:45.547356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:45.551234Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436154904029968:2070] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:45.559094Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436155616172753:2069] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:45.559140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:45.559314Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436154904029970:2071] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:45.575891Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436155616172753:2069] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:45.576399Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436154904029970:2071] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:45.576542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:45.576563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:45.576571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436154904029974:2048]
2025-11-19T14:00:45.576580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436154904029974:2048] [node 2] ICS09 handshake done sender: [1:7574436154904029970:2071] self: [1:7574436154904029969:0] peer: [2:7574436155616172754:0] socket: 17 qp: 126587
2025-11-19T14:00:45.576588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436154904029974:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:45.576597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:45.577370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:45.577384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:45.577390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436155616172756:2048]
2025-11-19T14:00:45.577395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436155616172756:2048] [node 1] ICS09 handshake done sender: [2:7574436155616172753:2069] self: [2:7574436155616172754:0] peer: [1:7574436154904029969:0] socket: 16 qp: 126588
2025-11-19T14:00:45.577399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436155616172756:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:45.577405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:00:45.860177Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:00:45.860260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436155616172756:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-19T14:00:45.860266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436155616172756:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:45.860289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:45.860301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436155616172756:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:45.862563Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:45.862659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436154904029974:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:45.862663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436154904029974:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:45.862691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:45.862707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436154904029974:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:45.878944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436155616172756:2048] [node 1] ICS08 incoming handshake Self# [1:7574436154904029969:0] Peer# [2:7574436155616172754:0] Counter# 811 LastInputSerial# 23
2025-11-19T14:00:45.879579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436154904029974:2048] [node 2] ICS08 incoming handshake Self# [2:7574436155616172754:0] Peer# [1:7574436154904029969:0] Counter# 23 LastInputSerial# 811
2025-11-19T14:00:45.879808Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436154904029976:2072] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:45.879909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:45.879928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436154904029974:2048] [node 2] ICS09 handshake done sender: [1:7574436154904029976:2072] self: [1:7574436154904029969:0] peer: [2:7574436155616172754:0] socket: 22 qp: 126680
2025-11-19T14:00:45.880283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436154904029974:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:45.880297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:45.902396Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436155616172760:2072] held: no
2025-11-19T14:00:45.916560Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436155616172760:2072] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:45.916677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:45.916693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436155616172756:2048] [node 1] ICS09 handshake done sender: [2:7574436155616172760:2072] self: [2:7574436155616172754:0] peer: [1:7574436154904029969:0] socket: 23 qp: 126685
2025-11-19T14:00:45.917086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436155616172756:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:45.917100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:00:46.001137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436155616172756:2048] [node 1] ICS01 socket: 23 reason# Debug
2025-11-19T14:00:46.001179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436155616172756:2048] VirtualId# [2:7574436155616172754:0]
2025-11-19T14:00:46.001185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436155616172756:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:46.001221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:46.001245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:46.001727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:46.001812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436154904029974:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-19T14:00:46.001816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436154904029974:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:46.001855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:46.001871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436154904029974:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-19T14:00:46.006540Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436159911140059:2074]) peer tries to resume nonexistent session Self# [1:7574436154904029969:0] Peer# [2:7574436155616172754:0]
2025-11-19T14:00:46.006572Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436159911140059:2074] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:46.007042Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436159198997277:2074] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:46.016168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:46.016217Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:46.016221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436154904029974:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:46.016247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436154904029974:2048] VirtualId# [1:7574436154904029969:0]
2025-11-19T14:00:46.016269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:46.040221Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436159911140063:2076] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:46.041487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:46.049210Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436159198997280:2075] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:46.049717Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436159198997282:2076] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:46.066789Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436159911140063:2076] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:46.066951Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436159198997280:2075] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:46.066935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:46.066953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:46.066961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436159911140066:2048]
2025-11-19T14:00:46.066968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436159911140066:2048] [node 1] ICS09 handshake done sender: [2:7574436159911140063:2076] self: [2:7574436159911140064:0] peer: [1:7574436159198997279:0] socket: 18 qp: 126750
2025-11-19T14:00:46.066976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436159911140066:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:46.066986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:46.067151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:46.067187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:46.067197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436159198997285:2048]
2025-11-19T14:00:46.067206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436159198997285:2048] [node 2] ICS09 handshake done sender: [1:7574436159198997280:2075] self: [1:7574436159198997279:0] peer: [2:7574436159911140064:0] socket: 16 qp: 126749
2025-11-19T14:00:46.067215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436159198997285:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:46.067228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:00:46.487518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436159911140066:2048] [node 1] ICS01 socket: 18 reason# Debug
2025-11-19T14:00:46.487604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436159911140066:2048] VirtualId# [2:7574436159911140064:0]
2025-11-19T14:00:46.487611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436159911140066:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:46.487651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:46.487672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:46.488055Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:46.488175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436159198997285:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:46.488179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436159198997285:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:46.488212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:46.488226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436159198997285:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-19T14:00:46.500242Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436159911140068:2078]) peer tries to resume nonexistent session Self# [1:7574436159198997279:0] Peer# [2:7574436159911140064:0]
2025-11-19T14:00:46.501440Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436159911140068:2078] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:46.504231Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436159198997287:2077] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:46.509610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:46.509646Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:46.509651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436159198997285:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:46.509712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436159198997285:2048] VirtualId# [1:7574436159198997279:0]
2025-11-19T14:00:46.509732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:46.524050Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436159198997289:2078] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:46.528767Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436159911140072:2080] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:46.528819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:46.529017Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436159198997291:2079] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:46.534496Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436159911140072:2080] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:46.534611Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436159198997291:2079] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:46.534709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:46.534731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:46.534739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436159198997295:2048]
2025-11-19T14:00:46.534746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436159198997295:2048] [node 2] ICS09 handshake done sender: [1:7574436159198997291:2079] self: [1:7574436159198997290:0] peer: [2:7574436159911140073:0] socket: 16 qp: 126915
2025-11-19T14:00:46.534754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436159198997295:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:46.534764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:46.536530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:46.536547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:46.536553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436159911140075:2048]
2025-11-19T14:00:46.536559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436159911140075:2048] [node 1] ICS09 handshake done sender: [2:7574436159911140072:2080] self: [2:7574436159911140073:0] peer: [1:7574436159198997290:0] socket: 17 qp: 126917
2025-11-19T14:00:46.536566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436159911140075:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:46.536573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:46.536642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436151321205409:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:00:46.819641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436159911140075:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-11-19T14:00:46.820871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436159911140075:2048] VirtualId# [2:7574436159911140073:0]
2025-11-19T14:00:46.820885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436159911140075:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:46.820908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:46.820935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:46.821693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:46.821792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436159198997295:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:46.821796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436159198997295:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:46.821830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:46.821845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436159198997295:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 1523
TEvUndelivered Cookie# 1524
2025-11-19T14:00:46.824815Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436159911140077:2082]) peer tries to resume nonexistent session Self# [1:7574436159198997290:0] Peer# [2:7574436159911140073:0]
2025-11-19T14:00:46.824839Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436159911140077:2082] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:46.825352Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436159198997297:2080] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:46.825956Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:46.825987Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:46.825992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436159198997295:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:46.826037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436159198997295:2048] VirtualId# [1:7574436159198997290:0]
2025-11-19T14:00:46.826054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:46.835229Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436159911140078:2083] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:46.835273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:46.835509Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436159198997300:2081] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:46.848791Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436159911140078:2083] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:46.849087Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436159198997300:2081] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:46.849156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:46.849173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:46.849180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436159911140081:2048]
2025-11-19T14:00:46.849188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436159911140081:2048] [node 1] ICS09 handshake done sender: [2:7574436159911140078:2083] self: [2:7574436159911140079:0] peer: [1:7574436159198997299:0] socket: 17 qp: 127042
2025-11-19T14:00:46.849195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436159911140081:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:46.849205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:46.849562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:46.849572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:46.849577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436159198997303:2048]
2025-11-19T14:00:46.849582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436159198997303:2048] [node 2] ICS09 handshake done sender: [1:7574436159198997300:2081] self: [1:7574436159198997299:0] peer: [2:7574436159911140079:0] socket: 16 qp: 127041
2025-11-19T14:00:46.849587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436159198997303:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:46.849592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:00:47.003318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436159911140081:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-11-19T14:00:47.004232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:47.004325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436159198997303:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:47.004330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436159198997303:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:47.004363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:47.004378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436159198997303:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:47.003382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436159911140081:2048] VirtualId# [2:7574436159911140079:0]
2025-11-19T14:00:47.003398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436159911140081:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:47.003423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:47.003445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:00:47.010400Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436164206107379:2085]) peer tries to resume nonexistent session Self# [1:7574436159198997299:0] Peer# [2:7574436159911140079:0]
2025-11-19T14:00:47.010438Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436164206107379:2085] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:47.011568Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436163493964601:2082] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:47.012599Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:47.012636Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:47.012641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436159198997303:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:47.012673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436159198997303:2048] VirtualId# [1:7574436159198997299:0]
2025-11-19T14:00:47.012693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:47.051949Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436164206107383:2087] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:47.052070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:47.052094Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436163493964606:2084] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:47.053959Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436163493964604:2083] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:47.073179Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436164206107383:2087] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:47.073304Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436163493964604:2083] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:47.073422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:47.073443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:47.073451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436163493964609:2048]
2025-11-19T14:00:47.073458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436163493964609:2048] [node 2] ICS09 handshake done sender: [1:7574436163493964604:2083] self: [1:7574436163493964603:0] peer: [2:7574436164206107384:0] socket: 16 qp: 127115
2025-11-19T14:00:47.073465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436163493964609:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:47.073475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:47.074188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:47.074204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:47.074209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436164206107386:2048]
2025-11-19T14:00:47.074215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436164206107386:2048] [node 1] ICS09 handshake done sender: [2:7574436164206107383:2087] self: [2:7574436164206107384:0] peer: [1:7574436163493964603:0] socket: 18 qp: 127127
2025-11-19T14:00:47.074220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436164206107386:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:47.074227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:00:47.270676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:00:47.271268Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:47.271351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436163493964609:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:47.271357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436163493964609:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:47.271368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:47.271388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436163493964609:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:47.274074Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:47.274154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436164206107386:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-19T14:00:47.274161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436164206107386:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:47.274198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:47.274217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436164206107386:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:47.275499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436164206107386:2048] [node 1] ICS08 incoming handshake Self# [1:7574436163493964603:0] Peer# [2:7574436164206107384:0] Counter# 788 LastInputSerial# 19
2025-11-19T14:00:47.280525Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436163493964611:2085] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:47.280684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:47.280697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436163493964609:2048] [node 2] ICS09 handshake done sender: [1:7574436163493964611:2085] self: [1:7574436163493964603:0] peer: [2:7574436164206107384:0] socket: 17 qp: 127236
2025-11-19T14:00:47.281313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436163493964609:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:47.281328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:47.281430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436163493964609:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:00:47.281434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436163493964609:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:00:47.281447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:47.281458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436163493964609:2048] [node 2] ICS08 incoming handshake Self# [2:7574436164206107384:0] Peer# [1:7574436163493964603:0] Counter# 19 LastInputSerial# 788
2025-11-19T14:00:47.281478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:47.281539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436163493964609:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:47.282564Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436164206107389:2090] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:47.283601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:47.283617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436164206107386:2048] [node 1] ICS09 handshake done sender: [2:7574436164206107389:2090] self: [2:7574436164206107384:0] peer: [1:7574436163493964603:0] socket: 19 qp: 127238
2025-11-19T14:00:47.284093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436164206107386:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:47.284106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:47.284209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:47.284255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436164206107386:2048] [node 1] ICS01 socket: 19 reason# EndOfStream
2025-11-19T14:00:47.284283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436164206107386:2048] VirtualId# [2:7574436164206107384:0]
2025-11-19T14:00:47.284289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436164206107386:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:47.284317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:47.284338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:00:47.308545Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436163493964617:2088] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:47.308593Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:47.308600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436163493964609:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:47.308644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436163493964609:2048] VirtualId# [1:7574436163493964603:0]
2025-11-19T14:00:47.308671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:47.308685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:47.310542Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436164206107395:2092] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:47.327860Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436163493964617:2088] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:47.328226Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436164206107395:2092] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:47.328325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:47.328341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:47.328351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436164206107398:2048]
2025-11-19T14:00:47.328358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436164206107398:2048] [node 1] ICS09 handshake done sender: [2:7574436164206107395:2092] self: [2:7574436164206107394:0] peer: [1:7574436163493964618:0] socket: 17 qp: 127245
2025-11-19T14:00:47.328368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436164206107398:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:47.328379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:47.328855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:47.328869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:47.328874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436163493964620:2048]
2025-11-19T14:00:47.328881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436163493964620:2048] [node 2] ICS09 handshake done sender: [1:7574436163493964617:2088] self: [1:7574436163493964618:0] peer: [2:7574436164206107394:0] socket: 18 qp: 127243
2025-11-19T14:00:47.328886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436163493964620:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:47.328892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-19T14:00:47.435873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7574436151321205409:2048] [node 1] ICP34 closed connection by debug command
2025-11-19T14:00:47.435954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:47.436032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436164206107398:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:47.436038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436164206107398:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:47.436047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:47.436059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436164206107398:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:47.438342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:47.438439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436163493964620:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-19T14:00:47.438444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436163493964620:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:47.438495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:47.438512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436163493964620:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:47.446740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436164206107398:2048] [node 1] ICS08 incoming handshake Self# [1:7574436163493964618:0] Peer# [2:7574436164206107394:0] Counter# 259 LastInputSerial# 6
2025-11-19T14:00:47.447266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436163493964620:2048] [node 2] ICS08 incoming handshake Self# [2:7574436164206107394:0] Peer# [1:7574436163493964618:0] Counter# 6 LastInputSerial# 259
2025-11-19T14:00:47.447488Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436163493964622:2090] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:47.447608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:47.447619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436163493964620:2048] [node 2] ICS09 handshake done sender: [1:7574436163493964622:2090] self: [1:7574436163493964618:0] peer: [2:7574436164206107394:0] socket: 22 qp: 127285
2025-11-19T14:00:47.448084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436163493964620:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:47.448097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:47.451237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436164206107402:2094] held: no
2025-11-19T14:00:47.453272Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436164206107402:2094] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:47.453348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:47.453367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436164206107398:2048] [node 1] ICS09 handshake done sender: [2:7574436164206107402:2094] self: [2:7574436164206107394:0] peer: [1:7574436163493964618:0] socket: 23 qp: 127287
2025-11-19T14:00:47.454145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436164206107398:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:47.454161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:00:47.981396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436164206107398:2048] [node 1] ICS01 socket: 23 reason# Debug
2025-11-19T14:00:47.981461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436164206107398:2048] VirtualId# [2:7574436164206107394:0]
2025-11-19T14:00:47.981473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436164206107398:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:47.981512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:47.981539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:47.984021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:47.984137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436163493964620:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-19T14:00:47.984159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436163493964620:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:47.984202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:47.984219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436163493964620:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 2605
TEvUndelivered Cookie# 2606
TEvUndelivered Cookie# 2607
TEvUndelivered Cookie# 2608
2025-11-19T14:00:47.988527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436164206107405:2096]) peer tries to resume nonexistent session Self# [1:7574436163493964618:0] Peer# [2:7574436164206107394:0]
2025-11-19T14:00:47.988557Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436164206107405:2096] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:47.989352Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436163493964627:2092] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:47.990208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:47.990243Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:47.990251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436163493964620:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:47.990276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436163493964620:2048] VirtualId# [1:7574436163493964618:0]
2025-11-19T14:00:47.990303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:48.006706Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436163493964629:2093] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:48.006818Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436164206107409:2098] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:48.006842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:48.007009Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436163493964631:2094] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:48.013461Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436164206107409:2098] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:48.013592Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436163493964631:2094] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:48.013711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:48.013729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:48.013736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436167788931931:2048]
2025-11-19T14:00:48.013756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436167788931931:2048] [node 2] ICS09 handshake done sender: [1:7574436163493964631:2094] self: [1:7574436163493964630:0] peer: [2:7574436168501074706:0] socket: 16 qp: 127431
2025-11-19T14:00:48.013763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436167788931931:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:48.013772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:48.017862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:48.017884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:48.017892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436168501074708:2048]
2025-11-19T14:00:48.017898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436168501074708:2048] [node 1] ICS09 handshake done sender: [2:7574436164206107409:2098] self: [2:7574436168501074706:0] peer: [1:7574436163493964630:0] socket: 17 qp: 127430
2025-11-19T14:00:48.017914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436168501074708:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:48.017922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:48.017989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436151321205409:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:00:48.342481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436168501074708:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-11-19T14:00:48.342599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436168501074708:2048] VirtualId# [2:7574436168501074706:0]
2025-11-19T14:00:48.342611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436168501074708:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:48.342646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:48.342683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:48.343626Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:48.343738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436167788931931:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:48.343786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436167788931931:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:48.343821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:48.343837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436167788931931:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-19T14:00:48.348284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436168501074710:2100]) peer tries to resume nonexistent session Self# [1:7574436163493964630:0] Peer# [2:7574436168501074706:0]
2025-11-19T14:00:48.348312Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436168501074710:2100] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:48.350713Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436167788931933:2095] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:48.361541Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:48.361601Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:48.361608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436167788931931:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:48.361657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436167788931931:2048] VirtualId# [1:7574436163493964630:0]
2025-11-19T14:00:48.361681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:48.367072Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436168501074711:2101] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:48.367124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:48.368652Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436167788931936:2096] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:48.376048Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436168501074711:2101] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:48.376486Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436167788931936:2096] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:48.376665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:48.376684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:48.376693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436167788931939:2048]
2025-11-19T14:00:48.376700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436167788931939:2048] [node 2] ICS09 handshake done sender: [1:7574436167788931936:2096] self: [1:7574436167788931935:0] peer: [2:7574436168501074712:0] socket: 16 qp: 127546
2025-11-19T14:00:48.376710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436167788931939:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:48.376731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:48.376937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:48.376953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:48.376959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436168501074714:2048]
2025-11-19T14:00:48.376964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436168501074714:2048] [node 1] ICS09 handshake done sender: [2:7574436168501074711:2101] self: [2:7574436168501074712:0] peer: [1:7574436167788931935:0] socket: 17 qp: 127547
2025-11-19T14:00:48.376970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436168501074714:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:48.376978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:00:48.832165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436168501074714:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-11-19T14:00:48.833039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:48.833134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436167788931939:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:48.833139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436167788931939:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:48.833173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:48.833187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436167788931939:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:48.832253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436168501074714:2048] VirtualId# [2:7574436168501074712:0]
2025-11-19T14:00:48.832262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436168501074714:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:48.832287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:48.832309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:00:48.838359Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436168501074716:2103]) peer tries to resume nonexistent session Self# [1:7574436167788931935:0] Peer# [2:7574436168501074712:0]
2025-11-19T14:00:48.838391Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436168501074716:2103] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:48.843569Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436167788931941:2097] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:48.850310Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:48.850361Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:48.850366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436167788931939:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:48.850854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436167788931939:2048] VirtualId# [1:7574436167788931935:0]
2025-11-19T14:00:48.850883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:48.857905Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436168501074720:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:48.859542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:48.859966Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436167788931944:2098] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:48.860047Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436167788931946:2099] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:48.872427Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436168501074720:2105] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:48.872492Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436167788931944:2098] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:48.873669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:48.873692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:48.873701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436168501074723:2048]
2025-11-19T14:00:48.873708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436168501074723:2048] [node 1] ICS09 handshake done sender: [2:7574436168501074720:2105] self: [2:7574436168501074721:0] peer: [1:7574436167788931943:0] socket: 18 qp: 127682
2025-11-19T14:00:48.873717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436168501074723:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:48.873727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:48.875329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:48.875351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:48.875358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436167788931949:2048]
2025-11-19T14:00:48.875365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436167788931949:2048] [node 2] ICS09 handshake done sender: [1:7574436167788931944:2098] self: [1:7574436167788931943:0] peer: [2:7574436168501074721:0] socket: 16 qp: 127681
2025-11-19T14:00:48.875371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436167788931949:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:48.875380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:00:49.164097Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:00:49.164224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436168501074723:2048] [node 1] ICS07 socket disconnect 18 reason# Debug
2025-11-19T14:00:49.164231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436168501074723:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:49.164264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:49.164277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436168501074723:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:49.166201Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:49.166292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436167788931949:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:49.166298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436167788931949:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:49.166334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:49.166349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436167788931949:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:49.175610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436168501074723:2048] [node 1] ICS08 incoming handshake Self# [1:7574436167788931943:0] Peer# [2:7574436168501074721:0] Counter# 1811 LastInputSerial# 44
2025-11-19T14:00:49.175818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436167788931949:2048] [node 2] ICS08 incoming handshake Self# [2:7574436168501074721:0] Peer# [1:7574436167788931943:0] Counter# 44 LastInputSerial# 1811
2025-11-19T14:00:49.176133Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436172083899247:2100] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:49.176241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:49.176253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436167788931949:2048] [node 2] ICS09 handshake done sender: [1:7574436172083899247:2100] self: [1:7574436167788931943:0] peer: [2:7574436168501074721:0] socket: 22 qp: 127783
2025-11-19T14:00:49.176598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436167788931949:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:49.176613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:49.184299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436172796042023:2108] held: no
2025-11-19T14:00:49.187866Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436172796042023:2108] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:49.188378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:49.188396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436168501074723:2048] [node 1] ICS09 handshake done sender: [2:7574436172796042023:2108] self: [2:7574436168501074721:0] peer: [1:7574436167788931943:0] socket: 23 qp: 127787
2025-11-19T14:00:49.188952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436168501074723:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:49.188969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:00:49.475319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:00:49.475439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:49.475521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436167788931949:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-19T14:00:49.475529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436167788931949:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:49.475537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:49.475561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436167788931949:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:49.477114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:49.477183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436168501074723:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-19T14:00:49.477195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436168501074723:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:49.477218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:49.477233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436168501074723:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:49.479964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436168501074723:2048] [node 1] ICS08 incoming handshake Self# [1:7574436167788931943:0] Peer# [2:7574436168501074721:0] Counter# 3379 LastInputSerial# 82
2025-11-19T14:00:49.481757Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436172083899252:2102] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:49.481849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:49.481859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436167788931949:2048] [node 2] ICS09 handshake done sender: [1:7574436172083899252:2102] self: [1:7574436167788931943:0] peer: [2:7574436168501074721:0] socket: 17 qp: 127885
2025-11-19T14:00:49.482200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436167788931949:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:49.482210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:49.484927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436167788931949:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:00:49.484939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436167788931949:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:00:49.484965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:49.484977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436167788931949:2048] [node 2] ICS08 incoming handshake Self# [2:7574436168501074721:0] Peer# [1:7574436167788931943:0] Counter# 82 LastInputSerial# 3379
2025-11-19T14:00:49.485204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:49.485285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436167788931949:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:00:49.485309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436167788931949:2048] VirtualId# [1:7574436167788931943:0]
2025-11-19T14:00:49.485320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:49.500762Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436172796042027:2111] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:49.501112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:49.501129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436168501074723:2048] [node 1] ICS09 handshake done sender: [2:7574436172796042027:2111] self: [2:7574436168501074721:0] peer: [1:7574436167788931943:0] socket: 18 qp: 127888
2025-11-19T14:00:49.501532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436168501074723:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:49.501544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:49.501701Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:00:49.501705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436168501074723:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:00:49.501715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436168501074723:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:00:49.501722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:49.501753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:49.501801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436168501074723:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:49.501803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436168501074723:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:49.513658Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436172083899258:2105]) peer tries to resume nonexistent session Self# [2:7574436168501074721:0] Peer# [1:7574436167788931943:0]
2025-11-19T14:00:49.513679Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436172083899258:2105] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:49.518135Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436172796042032:2113] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:00:49.519069Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:49.519104Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:49.519108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436168501074723:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:49.519135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436168501074723:2048] VirtualId# [2:7574436168501074721:0]
2025-11-19T14:00:49.519176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 4485
TEvUndelivered Cookie# 4486
TEvUndelivered Cookie# 4487
TEvUndelivered Cookie# 4488
TEvUndelivered Cookie# 4489
TEvUndelivered Cookie# 4490
TEvUndelivered Cookie# 4491
TEvUndelivered Cookie# 4492
TEvUndelivered Cookie# 4493
2025-11-19T14:00:49.537658Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436172796042035:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:49.537024Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436172083899259:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:49.537067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:49.544306Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436172083899259:2106] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:49.544412Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436172796042035:2114] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:49.544526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:49.544543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:49.544550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436172796042038:2048]
2025-11-19T14:00:49.544557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436172796042038:2048] [node 1] ICS09 handshake done sender: [2:7574436172796042035:2114] self: [2:7574436172796042034:0] peer: [1:7574436172083899260:0] socket: 16 qp: 127902
2025-11-19T14:00:49.544564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436172796042038:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:49.544573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:49.544735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:49.544747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:49.544762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436172083899262:2048]
2025-11-19T14:00:49.544767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436172083899262:2048] [node 2] ICS09 handshake done sender: [1:7574436172083899259:2106] self: [1:7574436172083899260:0] peer: [2:7574436172796042034:0] socket: 17 qp: 127901
2025-11-19T14:00:49.544772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436172083899262:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:49.544777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:00:49.602974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436172083899262:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-19T14:00:49.604470Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:49.604556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436172796042038:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-19T14:00:49.603898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436172083899262:2048] VirtualId# [1:7574436172083899260:0]
2025-11-19T14:00:49.603940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436172083899262:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:49.603993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:49.604010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:49.604710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436172796042038:2048] VirtualId# [2:7574436172796042034:0]
2025-11-19T14:00:49.604719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436172796042038:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:49.604758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:49.604775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
Subscribe
2025-11-19T14:00:49.627660Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436172083899264:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:49.627717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:49.628499Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436172796042041:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:49.648766Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436172083899264:2108] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:49.649633Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436172796042041:2115] [node 1] ICH04 handshake succeeded
TEvNodeConnected
2025-11-19T14:00:49.649780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:49.649799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:49.649808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436172796042044:2048]
2025-11-19T14:00:49.649815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436172796042044:2048] [node 1] ICS09 handshake done sender: [2:7574436172796042041:2115] self: [2:7574436172796042040:0] peer: [1:7574436172083899265:0] socket: 16 qp: 127939
2025-11-19T14:00:49.649823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436172796042044:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:49.649832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:49.652329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:49.652348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:49.652354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436172083899267:2048]
2025-11-19T14:00:49.652361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436172083899267:2048] [node 2] ICS09 handshake done sender: [1:7574436172083899264:2108] self: [1:7574436172083899265:0] peer: [2:7574436172796042040:0] socket: 17 qp: 127937
2025-11-19T14:00:49.652369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436172083899267:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:49.652376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:00:50.008679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:00:50.008778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436172796042044:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-19T14:00:50.008783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436172796042044:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:50.008808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:50.008821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436172796042044:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:50.014432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:50.014532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436172083899267:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:50.014538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436172083899267:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:50.014588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:50.014606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436172083899267:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:50.036166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436172083899267:2048] [node 2] ICS08 incoming handshake Self# [2:7574436172796042040:0] Peer# [1:7574436172083899265:0] Counter# 59 LastInputSerial# 2292
2025-11-19T14:00:50.038210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436172796042044:2048] [node 1] ICS08 incoming handshake Self# [1:7574436172083899265:0] Peer# [2:7574436172796042040:0] Counter# 2292 LastInputSerial# 59
2025-11-19T14:00:50.038542Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436176378866565:2110] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:50.038653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:50.038670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436172083899267:2048] [node 2] ICS09 handshake done sender: [1:7574436176378866565:2110] self: [1:7574436172083899265:0] peer: [2:7574436172796042040:0] socket: 22 qp: 128088
2025-11-19T14:00:50.039021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436172083899267:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:50.039035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:50.041380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436177091009344:2117] held: no
2025-11-19T14:00:50.042803Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436177091009344:2117] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:50.042862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:50.042874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436172796042044:2048] [node 1] ICS09 handshake done sender: [2:7574436177091009344:2117] self: [2:7574436172796042040:0] peer: [1:7574436172083899265:0] socket: 23 qp: 128091
2025-11-19T14:00:50.043253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436172796042044:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:50.043265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:00:50.383977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436172796042044:2048] [node 1] ICS01 socket: 23 reason# Debug
2025-11-19T14:00:50.384029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436172796042044:2048] VirtualId# [2:7574436172796042040:0]
2025-11-19T14:00:50.384037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436172796042044:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:50.384075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:50.384104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:50.387628Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:50.387720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436172083899267:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-19T14:00:50.387726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436172083899267:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
TEvNodeDisconnected
2025-11-19T14:00:50.388212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:50.388246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436172083899267:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:50.390903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436177091009347:2119]) peer tries to resume nonexistent session Self# [1:7574436172083899265:0] Peer# [2:7574436172796042040:0]
2025-11-19T14:00:50.390927Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436177091009347:2119] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:50.394266Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436176378866570:2112] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:50.394961Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:50.394997Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:50.395001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436172083899267:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:50.395025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436172083899267:2048] VirtualId# [1:7574436172083899265:0]
2025-11-19T14:00:50.395044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:50.408126Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436177091009348:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:50.408191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:50.408401Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436176378866573:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:50.411891Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436177091009348:2120] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:50.412138Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436176378866573:2113] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:50.412248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:50.412264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:50.412272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436176378866576:2048]
2025-11-19T14:00:50.412279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436176378866576:2048] [node 2] ICS09 handshake done sender: [1:7574436176378866573:2113] self: [1:7574436176378866572:0] peer: [2:7574436177091009349:0] socket: 16 qp: 128236
2025-11-19T14:00:50.412286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436176378866576:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:50.412296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:50.412451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:50.412464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:50.412468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436177091009351:2048]
2025-11-19T14:00:50.412473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436177091009351:2048] [node 1] ICS09 handshake done sender: [2:7574436177091009348:2120] self: [2:7574436177091009349:0] peer: [1:7574436176378866572:0] socket: 17 qp: 128234
2025-11-19T14:00:50.412477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436177091009351:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:50.412482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:00:50.707259Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:00:50.707350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436176378866576:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-19T14:00:50.707356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436176378866576:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:50.707391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:50.707404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436176378866576:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:50.709744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:50.709831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436177091009351:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:50.709837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436177091009351:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:50.709874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:50.709892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436177091009351:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:50.712769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436177091009351:2048] [node 1] ICS08 incoming handshake Self# [1:7574436176378866572:0] Peer# [2:7574436177091009349:0] Counter# 1113 LastInputSerial# 27
2025-11-19T14:00:50.717864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436176378866576:2048] [node 2] ICS08 incoming handshake Self# [2:7574436177091009349:0] Peer# [1:7574436176378866572:0] Counter# 27 LastInputSerial# 1113
2025-11-19T14:00:50.718444Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436176378866578:2114] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:50.718563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:50.718578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436176378866576:2048] [node 2] ICS09 handshake done sender: [1:7574436176378866578:2114] self: [1:7574436176378866572:0] peer: [2:7574436177091009349:0] socket: 20 qp: 128369
2025-11-19T14:00:50.718957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436176378866576:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:50.718968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:50.725146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436177091009354:2123] held: no
2025-11-19T14:00:50.726763Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436177091009354:2123] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:50.726856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:50.726870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436177091009351:2048] [node 1] ICS09 handshake done sender: [2:7574436177091009354:2123] self: [2:7574436177091009349:0] peer: [1:7574436176378866572:0] socket: 21 qp: 128374
2025-11-19T14:00:50.727323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436177091009351:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:50.727336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:00:50.944230Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:00:50.944320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436177091009351:2048] [node 1] ICS07 socket disconnect 21 reason# Debug
2025-11-19T14:00:50.944326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436177091009351:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:50.944348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:50.944361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436177091009351:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:50.945006Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:50.945084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436176378866576:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-19T14:00:50.945089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436176378866576:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:50.945129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:50.945147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436176378866576:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:50.957698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436177091009351:2048] [node 1] ICS08 incoming handshake Self# [1:7574436176378866572:0] Peer# [2:7574436177091009349:0] Counter# 2349 LastInputSerial# 56
2025-11-19T14:00:50.971903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436176378866576:2048] [node 2] ICS08 incoming handshake Self# [2:7574436177091009349:0] Peer# [1:7574436176378866572:0] Counter# 56 LastInputSerial# 2349
2025-11-19T14:00:50.972124Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436176378866583:2116] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:50.983555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:50.983581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436176378866576:2048] [node 2] ICS09 handshake done sender: [1:7574436176378866583:2116] self: [1:7574436176378866572:0] peer: [2:7574436177091009349:0] socket: 19 qp: 128485
2025-11-19T14:00:50.984013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436176378866576:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:50.984031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:51.011273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436177091009360:2126] held: no
2025-11-19T14:00:51.011305Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436177091009360:2126] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:51.011403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:51.011416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436177091009351:2048] [node 1] ICS09 handshake done sender: [2:7574436177091009360:2126] self: [2:7574436177091009349:0] peer: [1:7574436176378866572:0] socket: 22 qp: 128487
2025-11-19T14:00:51.016056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436177091009351:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:51.016092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:00:51.459951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:00:51.460074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436177091009351:2048] [node 1] ICS07 socket disconnect 22 reason# Debug
2025-11-19T14:00:51.460080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436177091009351:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:51.460101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:51.460115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436177091009351:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:51.463411Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:51.463499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436176378866576:2048] [node 2] ICS07 socket disconnect 19 reason# EndOfStream
2025-11-19T14:00:51.463505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436176378866576:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:51.463532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:51.463551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436176378866576:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:51.470863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436177091009351:2048] [node 1] ICS08 incoming handshake Self# [1:7574436176378866572:0] Peer# [2:7574436177091009349:0] Counter# 4957 LastInputSerial# 122
2025-11-19T14:00:51.476875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436176378866576:2048] [node 2] ICS08 incoming handshake Self# [2:7574436177091009349:0] Peer# [1:7574436176378866572:0] Counter# 122 LastInputSerial# 4957
2025-11-19T14:00:51.480826Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436180673833884:2118] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:51.480962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:51.480980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436176378866576:2048] [node 2] ICS09 handshake done sender: [1:7574436180673833884:2118] self: [1:7574436176378866572:0] peer: [2:7574436177091009349:0] socket: 18 qp: 128652
2025-11-19T14:00:51.481601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436176378866576:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:51.481614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:51.486337Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436181385976661:2129] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:51.496171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:51.496194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436177091009351:2048] [node 1] ICS09 handshake done sender: [2:7574436181385976661:2129] self: [2:7574436177091009349:0] peer: [1:7574436176378866572:0] socket: 20 qp: 128655
2025-11-19T14:00:51.496692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436177091009351:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:51.496706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:00:51.953486Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:00:51.953577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436176378866576:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-19T14:00:51.953583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436176378866576:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:51.953607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:51.953621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436176378866576:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:51.960117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:51.960203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436177091009351:2048] [node 1] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-19T14:00:51.960209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436177091009351:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:51.960252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:51.960271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436177091009351:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:51.961500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436177091009351:2048] [node 1] ICS08 incoming handshake Self# [1:7574436176378866572:0] Peer# [2:7574436177091009349:0] Counter# 8004 LastInputSerial# 195
2025-11-19T14:00:51.964793Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436180673833889:2120] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:51.964976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:51.964991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436176378866576:2048] [node 2] ICS09 handshake done sender: [1:7574436180673833889:2120] self: [1:7574436176378866572:0] peer: [2:7574436177091009349:0] socket: 16 qp: 128818
2025-11-19T14:00:51.965393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436176378866576:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:51.965405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:51.969438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436176378866576:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:00:51.969450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436176378866576:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:00:51.969478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:51.969492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436176378866576:2048] [node 2] ICS08 incoming handshake Self# [2:7574436177091009349:0] Peer# [1:7574436176378866572:0] Counter# 195 LastInputSerial# 8004
2025-11-19T14:00:51.969690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:51.969770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436176378866576:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:51.970263Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436181385976665:2132] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:51.970428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:51.970444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436177091009351:2048] [node 1] ICS09 handshake done sender: [2:7574436181385976665:2132] self: [2:7574436177091009349:0] peer: [1:7574436176378866572:0] socket: 17 qp: 128819
2025-11-19T14:00:51.970902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436177091009351:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:51.970913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:51.971009Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:51.971054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436177091009351:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-11-19T14:00:51.971074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436177091009351:2048] VirtualId# [2:7574436177091009349:0]
2025-11-19T14:00:51.971079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436177091009351:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:51.971101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:51.971118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:00:51.981520Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436180673833895:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:51.981553Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:51.981560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436176378866576:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:51.981586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436176378866576:2048] VirtualId# [1:7574436176378866572:0]
2025-11-19T14:00:51.981606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:51.981614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:51.984400Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436181385976671:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:51.994095Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436180673833895:2123] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:51.994615Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436181385976671:2134] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:51.994687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:51.994703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:51.994713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436180673833898:2048]
2025-11-19T14:00:51.994727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436180673833898:2048] [node 2] ICS09 handshake done sender: [1:7574436180673833895:2123] self: [1:7574436180673833896:0] peer: [2:7574436181385976670:0] socket: 18 qp: 128831
2025-11-19T14:00:51.994734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436180673833898:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:51.994746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:51.995003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:51.995022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:51.995026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436181385976674:2048]
2025-11-19T14:00:51.995032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436181385976674:2048] [node 1] ICS09 handshake done sender: [2:7574436181385976671:2134] self: [2:7574436181385976670:0] peer: [1:7574436180673833896:0] socket: 17 qp: 128832
2025-11-19T14:00:51.995036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436181385976674:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:51.995042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:00:52.087968Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:00:52.088046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436181385976674:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-11-19T14:00:52.088051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436181385976674:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:52.088077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:52.088088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436181385976674:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:52.090725Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:52.090800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436180673833898:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-19T14:00:52.090804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436180673833898:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:52.090844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:52.090859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436180673833898:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:52.096064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436181385976674:2048] [node 1] ICS08 incoming handshake Self# [1:7574436180673833896:0] Peer# [2:7574436181385976670:0] Counter# 143 LastInputSerial# 3
2025-11-19T14:00:52.096352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436180673833898:2048] [node 2] ICS08 incoming handshake Self# [2:7574436181385976670:0] Peer# [1:7574436180673833896:0] Counter# 3 LastInputSerial# 143
2025-11-19T14:00:52.096600Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436184968801196:2125] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:52.096755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:52.096783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436180673833898:2048] [node 2] ICS09 handshake done sender: [1:7574436184968801196:2125] self: [1:7574436180673833896:0] peer: [2:7574436181385976670:0] socket: 22 qp: 128876
2025-11-19T14:00:52.097137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436180673833898:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:52.097152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:52.099355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436185680943974:2136] held: no
2025-11-19T14:00:52.101152Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436185680943974:2136] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:52.101337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:52.101351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436181385976674:2048] [node 1] ICS09 handshake done sender: [2:7574436185680943974:2136] self: [2:7574436181385976670:0] peer: [1:7574436180673833896:0] socket: 23 qp: 128879
2025-11-19T14:00:52.101863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436181385976674:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:52.101878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:00:52.456215Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:00:52.456294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436181385976674:2048] [node 1] ICS07 socket disconnect 23 reason# Debug
2025-11-19T14:00:52.456300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436181385976674:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:52.456324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:52.456337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436181385976674:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:52.458246Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:52.458316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436180673833898:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-19T14:00:52.458322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436180673833898:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:52.458357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:52.458373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436180673833898:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:52.464677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436181385976674:2048] [node 1] ICS08 incoming handshake Self# [1:7574436180673833896:0] Peer# [2:7574436181385976670:0] Counter# 2848 LastInputSerial# 69
2025-11-19T14:00:52.464909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436180673833898:2048] [node 2] ICS08 incoming handshake Self# [2:7574436181385976670:0] Peer# [1:7574436180673833896:0] Counter# 69 LastInputSerial# 2848
2025-11-19T14:00:52.465094Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436184968801201:2127] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:52.465180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:52.465194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436180673833898:2048] [node 2] ICS09 handshake done sender: [1:7574436184968801201:2127] self: [1:7574436180673833896:0] peer: [2:7574436181385976670:0] socket: 19 qp: 128995
2025-11-19T14:00:52.465542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436180673833898:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:52.465553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:52.473305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436185680943979:2139] held: no
2025-11-19T14:00:52.475048Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436185680943979:2139] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:52.475154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:52.475167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436181385976674:2048] [node 1] ICS09 handshake done sender: [2:7574436185680943979:2139] self: [2:7574436181385976670:0] peer: [1:7574436180673833896:0] socket: 20 qp: 128997
2025-11-19T14:00:52.475599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436181385976674:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:52.475615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:00:52.673115Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:00:52.673209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436180673833898:2048] [node 2] ICS07 socket disconnect 19 reason# Debug
2025-11-19T14:00:52.673215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436180673833898:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:52.673248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:52.673266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436180673833898:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:52.675125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:52.675210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436181385976674:2048] [node 1] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-19T14:00:52.675216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436181385976674:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:52.675243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:52.675261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436181385976674:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:52.677955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436181385976674:2048] [node 1] ICS08 incoming handshake Self# [1:7574436180673833896:0] Peer# [2:7574436181385976670:0] Counter# 4015 LastInputSerial# 96
2025-11-19T14:00:52.684723Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436184968801206:2129] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:52.684842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:52.684855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436180673833898:2048] [node 2] ICS09 handshake done sender: [1:7574436184968801206:2129] self: [1:7574436180673833896:0] peer: [2:7574436181385976670:0] socket: 16 qp: 129082
2025-11-19T14:00:52.685835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436180673833898:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:52.685854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:52.693704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436180673833898:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:00:52.693720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436180673833898:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:00:52.693740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:52.693761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436180673833898:2048] [node 2] ICS08 incoming handshake Self# [2:7574436181385976670:0] Peer# [1:7574436180673833896:0] Counter# 96 LastInputSerial# 4015
2025-11-19T14:00:52.693831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:52.693905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436180673833898:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:00:52.693928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436180673833898:2048] VirtualId# [1:7574436180673833896:0]
2025-11-19T14:00:52.693937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:52.696556Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436185680943983:2142] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:52.700483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:52.700503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436181385976674:2048] [node 1] ICS09 handshake done sender: [2:7574436185680943983:2142] self: [2:7574436181385976670:0] peer: [1:7574436180673833896:0] socket: 18 qp: 129086
2025-11-19T14:00:52.700937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436181385976674:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:52.700950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:52.701081Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:00:52.701086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436181385976674:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:00:52.701090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436181385976674:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:00:52.701096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:52.701127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:52.701177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436181385976674:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:52.701179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436181385976674:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:52.716589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436184968801212:2132]) peer tries to resume nonexistent session Self# [2:7574436181385976670:0] Peer# [1:7574436180673833896:0]
2025-11-19T14:00:52.716617Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436184968801212:2132] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:52.717248Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436185680943988:2144] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:00:52.718045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:52.718090Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:52.718100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436181385976674:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:52.718134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436181385976674:2048] VirtualId# [2:7574436181385976670:0]
2025-11-19T14:00:52.718185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 8464
TEvUndelivered Cookie# 8465
TEvUndelivered Cookie# 8466
TEvUndelivered Cookie# 8467
TEvUndelivered Cookie# 8468
TEvUndelivered Cookie# 8469
TEvUndelivered Cookie# 8470
2025-11-19T14:00:52.722106Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436184968801213:2133] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:52.722140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:52.723145Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436185680943991:2145] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:52.729016Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436184968801213:2133] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:52.729134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:52.729151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:52.729159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436184968801216:2048]
2025-11-19T14:00:52.729167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436184968801216:2048] [node 2] ICS09 handshake done sender: [1:7574436184968801213:2133] self: [1:7574436184968801214:0] peer: [2:7574436185680943990:0] socket: 17 qp: 129108
2025-11-19T14:00:52.729177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436184968801216:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:52.729186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:52.732023Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436185680943991:2145] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:52.736252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:52.736275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:52.736284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436185680943994:2048]
2025-11-19T14:00:52.736293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436185680943994:2048] [node 1] ICS09 handshake done sender: [2:7574436185680943991:2145] self: [2:7574436185680943990:0] peer: [1:7574436184968801214:0] socket: 16 qp: 129109
2025-11-19T14:00:52.736302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436185680943994:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:52.736312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:00:53.228979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:00:53.229109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:53.229195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436184968801216:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:53.229203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436184968801216:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:53.229213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:53.229229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436184968801216:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:53.231252Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:53.231334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436185680943994:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:53.231339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436185680943994:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:53.231365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:53.231383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436185680943994:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:53.244510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436185680943994:2048] [node 1] ICS08 incoming handshake Self# [1:7574436184968801214:0] Peer# [2:7574436185680943990:0] Counter# 2614 LastInputSerial# 67
2025-11-19T14:00:53.245028Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436189263768514:2135] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:53.245134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:53.245144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436184968801216:2048] [node 2] ICS09 handshake done sender: [1:7574436189263768514:2135] self: [1:7574436184968801214:0] peer: [2:7574436185680943990:0] socket: 20 qp: 129308
2025-11-19T14:00:53.245579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436184968801216:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:53.245590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:53.249340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436184968801216:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:00:53.249353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436184968801216:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:00:53.249384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:53.249398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436184968801216:2048] [node 2] ICS08 incoming handshake Self# [2:7574436185680943990:0] Peer# [1:7574436184968801214:0] Counter# 67 LastInputSerial# 2614
2025-11-19T14:00:53.249471Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:53.249561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436184968801216:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:00:53.249597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436184968801216:2048] VirtualId# [1:7574436184968801214:0]
2025-11-19T14:00:53.249608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:53.260860Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436189975911293:2147] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:53.261397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:53.261413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436185680943994:2048] [node 1] ICS09 handshake done sender: [2:7574436189975911293:2147] self: [2:7574436185680943990:0] peer: [1:7574436184968801214:0] socket: 21 qp: 129309
2025-11-19T14:00:53.262002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436185680943994:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:53.262016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:53.262171Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:00:53.262176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436185680943994:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:00:53.262180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436185680943994:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:00:53.262187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:53.262220Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:53.262278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436185680943994:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:53.262281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436185680943994:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:53.292119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436189263768520:2138]) peer tries to resume nonexistent session Self# [2:7574436185680943990:0] Peer# [1:7574436184968801214:0]
2025-11-19T14:00:53.292149Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436189263768520:2138] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:53.292749Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436189975911298:2149] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:00:53.293668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:53.293709Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:53.293715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436185680943994:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:53.293742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436185680943994:2048] VirtualId# [2:7574436185680943990:0]
2025-11-19T14:00:53.293778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 9120
TEvUndelivered Cookie# 9121
TEvUndelivered Cookie# 9122
TEvUndelivered Cookie# 9123
TEvUndelivered Cookie# 9124
TEvUndelivered Cookie# 9125
TEvUndelivered Cookie# 9126
TEvUndelivered Cookie# 9127
2025-11-19T14:00:53.300006Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436189263768521:2139] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:53.300047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:53.307849Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436189263768521:2139] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:53.308201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:53.308208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:53.308212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436189263768524:2048]
2025-11-19T14:00:53.308218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436189263768524:2048] [node 2] ICS09 handshake done sender: [1:7574436189263768521:2139] self: [1:7574436189263768522:0] peer: [2:7574436189975911300:0] socket: 17 qp: 129326
2025-11-19T14:00:53.308222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436189263768524:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:53.308228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:53.304389Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436189975911301:2150] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:53.307931Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436189975911301:2150] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:53.308012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:53.308026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:53.308035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436189975911304:2048]
2025-11-19T14:00:53.308042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436189975911304:2048] [node 1] ICS09 handshake done sender: [2:7574436189975911301:2150] self: [2:7574436189975911300:0] peer: [1:7574436189263768522:0] socket: 16 qp: 129327
2025-11-19T14:00:53.308051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436189975911304:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:53.308060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:00:53.361631Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:00:53.361726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436189263768524:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-11-19T14:00:53.361731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436189263768524:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:53.361775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:53.361787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436189263768524:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:53.362964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:53.363039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436189975911304:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-19T14:00:53.363779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436189975911304:2048] VirtualId# [2:7574436189975911300:0]
2025-11-19T14:00:53.363791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436189975911304:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:53.363820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:53.363837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:53.367681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436189975911306:2151]) peer tries to resume nonexistent session Self# [1:7574436189263768522:0] Peer# [2:7574436189975911300:0]
2025-11-19T14:00:53.367703Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436189975911306:2151] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:53.367929Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436189263768526:2141] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:53.368489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:53.368532Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:53.368537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436189263768524:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:53.368622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436189263768524:2048] VirtualId# [1:7574436189263768522:0]
2025-11-19T14:00:53.368635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:53.384457Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436189975911307:2152] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:53.384512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:53.384705Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436189263768529:2142] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
Subscribe
2025-11-19T14:00:53.432125Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436189975911307:2152] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:53.432592Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436189263768529:2142] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:53.432730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:53.432746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:53.432754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436189263768532:2048]
2025-11-19T14:00:53.432761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436189263768532:2048] [node 2] ICS09 handshake done sender: [1:7574436189263768529:2142] self: [1:7574436189263768528:0] peer: [2:7574436189975911308:0] socket: 16 qp: 129353
2025-11-19T14:00:53.432769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436189263768532:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:53.432778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
TEvNodeConnected
2025-11-19T14:00:53.433178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:53.433192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:53.433197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436189975911310:2048]
2025-11-19T14:00:53.433205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436189975911310:2048] [node 1] ICS09 handshake done sender: [2:7574436189975911307:2152] self: [2:7574436189975911308:0] peer: [1:7574436189263768528:0] socket: 17 qp: 129352
2025-11-19T14:00:53.433209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436189975911310:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:53.433216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:00:53.801396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:00:53.801560Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:53.801645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436189263768532:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:53.801650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436189263768532:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:53.801661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:53.801678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436189263768532:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:53.802395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:53.802466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436189975911310:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:53.802470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436189975911310:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:53.802499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:53.802515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436189975911310:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:53.805089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436189975911310:2048] [node 1] ICS08 incoming handshake Self# [1:7574436189263768528:0] Peer# [2:7574436189975911308:0] Counter# 1716 LastInputSerial# 43
2025-11-19T14:00:53.809963Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436189263768534:2143] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:53.810112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:53.810125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436189263768532:2048] [node 2] ICS09 handshake done sender: [1:7574436189263768534:2143] self: [1:7574436189263768528:0] peer: [2:7574436189975911308:0] socket: 20 qp: 129518
2025-11-19T14:00:53.810597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436189263768532:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:53.810614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:53.810647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436189263768532:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:00:53.810651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436189263768532:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:00:53.810691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:53.810706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436189263768532:2048] [node 2] ICS08 incoming handshake Self# [2:7574436189975911308:0] Peer# [1:7574436189263768528:0] Counter# 43 LastInputSerial# 1716
2025-11-19T14:00:53.810727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:53.810790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436189263768532:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:00:53.810816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436189263768532:2048] VirtualId# [1:7574436189263768528:0]
2025-11-19T14:00:53.810822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:53.821013Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436189975911313:2155] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:53.828707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:53.828733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436189975911310:2048] [node 1] ICS09 handshake done sender: [2:7574436189975911313:2155] self: [2:7574436189975911308:0] peer: [1:7574436189263768528:0] socket: 21 qp: 129519
2025-11-19T14:00:53.829460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436189975911310:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:53.829479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:53.829628Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:00:53.829634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436189975911310:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:00:53.829638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436189975911310:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:00:53.829646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:53.829688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:53.829754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436189975911310:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:53.829756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436189975911310:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:53.833585Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436189263768539:2145]) peer tries to resume nonexistent session Self# [2:7574436189975911308:0] Peer# [1:7574436189263768528:0]
2025-11-19T14:00:53.833613Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436189263768539:2145] [node 2] ICH08 Continuation request rejected by proxy
TEvNodeDisconnected
TEvUndelivered Cookie# 9554
TEvUndelivered Cookie# 9555
TEvUndelivered Cookie# 9556
TEvUndelivered Cookie# 9557
TEvUndelivered Cookie# 9558
TEvUndelivered Cookie# 9559
TEvUndelivered Cookie# 9560
TEvUndelivered Cookie# 9561
TEvUndelivered Cookie# 9562
2025-11-19T14:00:53.835888Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436189975911317:2157] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:00:53.836557Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:53.836591Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:53.836598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436189975911310:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:53.836635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436189975911310:2048] VirtualId# [2:7574436189975911308:0]
2025-11-19T14:00:53.836676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:53.849657Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436189263768540:2146] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:53.849706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:53.852057Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436189975911321:2158] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:53.863840Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436189263768540:2146] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:53.868089Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436189975911321:2158] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:53.868164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:53.868183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:53.868194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436189263768543:2048]
2025-11-19T14:00:53.868201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436189263768543:2048] [node 2] ICS09 handshake done sender: [1:7574436189263768540:2146] self: [1:7574436189263768541:0] peer: [2:7574436189975911320:0] socket: 17 qp: 129539
2025-11-19T14:00:53.868209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436189263768543:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:53.868219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:53.868483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:53.868495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:53.868499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436189975911324:2048]
2025-11-19T14:00:53.868505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436189975911324:2048] [node 1] ICS09 handshake done sender: [2:7574436189975911321:2158] self: [2:7574436189975911320:0] peer: [1:7574436189263768541:0] socket: 16 qp: 129541
2025-11-19T14:00:53.868509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436189975911324:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:53.868514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:00:54.317169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436189975911324:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-19T14:00:54.317255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436189975911324:2048] VirtualId# [2:7574436189975911320:0]
2025-11-19T14:00:54.317265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436189975911324:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:54.317293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:54.317317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:54.318483Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:54.318575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436189263768543:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:54.318580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436189263768543:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:54.318626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:54.318649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436189263768543:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-19T14:00:54.333911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436194270878622:2159]) peer tries to resume nonexistent session Self# [1:7574436189263768541:0] Peer# [2:7574436189975911320:0]
2025-11-19T14:00:54.333956Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436194270878622:2159] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:54.334719Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436193558735841:2148] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:54.340377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:54.340422Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:54.340426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436189263768543:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:54.340519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436189263768543:2048] VirtualId# [1:7574436189263768541:0]
2025-11-19T14:00:54.340541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:54.353676Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436194270878626:2161] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:54.355190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:54.359001Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436193558735844:2149] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:54.359115Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436193558735846:2150] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:54.364053Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436194270878626:2161] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:54.366872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:54.366891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:54.366899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436194270878629:2048]
2025-11-19T14:00:54.366906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436194270878629:2048] [node 1] ICS09 handshake done sender: [2:7574436194270878626:2161] self: [2:7574436194270878627:0] peer: [1:7574436193558735843:0] socket: 18 qp: 129831
2025-11-19T14:00:54.366916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436194270878629:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:54.366923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:54.367005Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436151321205409:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-11-19T14:00:54.364165Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436193558735844:2149] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:54.364272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:54.364289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:54.364297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436193558735849:2048]
2025-11-19T14:00:54.364305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436193558735849:2048] [node 2] ICS09 handshake done sender: [1:7574436193558735844:2149] self: [1:7574436193558735843:0] peer: [2:7574436194270878627:0] socket: 17 qp: 129829
2025-11-19T14:00:54.364313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436193558735849:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:54.364323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:00:54.672659Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:00:54.672685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436194270878629:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:00:54.672692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436194270878629:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:00:54.672702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:54.672084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436193558735849:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-19T14:00:54.672163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436193558735849:2048] VirtualId# [1:7574436193558735843:0]
2025-11-19T14:00:54.672172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436193558735849:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:54.672208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:54.672229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:54.680958Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:54.681058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436194270878629:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:54.681061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436194270878629:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:54.685843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436193558735851:2151]) peer tries to resume nonexistent session Self# [2:7574436194270878627:0] Peer# [1:7574436193558735843:0]
2025-11-19T14:00:54.685867Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436193558735851:2151] [node 2] ICH08 Continuation request rejected by proxy
TEvNodeDisconnected
TEvUndelivered Cookie# 10590
TEvUndelivered Cookie# 10591
TEvUndelivered Cookie# 10592
TEvUndelivered Cookie# 10593
TEvUndelivered Cookie# 10594
TEvUndelivered Cookie# 10595
2025-11-19T14:00:54.686593Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436194270878631:2163] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:00:54.687421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:54.687451Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:54.687455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436194270878629:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:54.687506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436194270878629:2048] VirtualId# [2:7574436194270878627:0]
2025-11-19T14:00:54.687541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:54.697040Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436193558735852:2152] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:54.697094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:54.697293Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436194270878634:2164] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:54.704251Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436193558735852:2152] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:54.705383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:54.705395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:54.705399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436193558735855:2048]
2025-11-19T14:00:54.705405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436193558735855:2048] [node 2] ICS09 handshake done sender: [1:7574436193558735852:2152] self: [1:7574436193558735853:0] peer: [2:7574436194270878633:0] socket: 17 qp: 129960
2025-11-19T14:00:54.705409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436193558735855:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:54.705416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:54.704979Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436194270878634:2164] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:54.705073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:54.705086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:54.705094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436194270878637:2048]
2025-11-19T14:00:54.705102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436194270878637:2048] [node 1] ICS09 handshake done sender: [2:7574436194270878634:2164] self: [2:7574436194270878633:0] peer: [1:7574436193558735853:0] socket: 16 qp: 129963
2025-11-19T14:00:54.705110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436194270878637:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:54.705119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:00:54.880120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436193558735855:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-19T14:00:54.880181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436193558735855:2048] VirtualId# [1:7574436193558735853:0]
2025-11-19T14:00:54.880189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436193558735855:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:54.880227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:54.880251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:54.881109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:54.881197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436194270878637:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:54.881208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436194270878637:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:54.881239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:54.881258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436194270878637:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:54.886723Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436193558735857:2154]) peer tries to resume nonexistent session Self# [2:7574436194270878633:0] Peer# [1:7574436193558735853:0]
2025-11-19T14:00:54.886747Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436193558735857:2154] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:54.892486Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436194270878639:2165] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 10706
TEvUndelivered Cookie# 10707
TEvUndelivered Cookie# 10708
TEvUndelivered Cookie# 10709
TEvUndelivered Cookie# 10710
TEvUndelivered Cookie# 10711
TEvUndelivered Cookie# 10712
TEvUndelivered Cookie# 10713
TEvUndelivered Cookie# 10714
TEvUndelivered Cookie# 10715
2025-11-19T14:00:54.900884Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:54.900933Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:54.900939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436194270878637:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:54.900995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436194270878637:2048] VirtualId# [2:7574436194270878633:0]
2025-11-19T14:00:54.901041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:54.906010Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436193558735858:2155] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:54.906061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:54.906289Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436194270878642:2166] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:54.913679Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436193558735858:2155] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:54.914050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:54.914061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:54.914065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436193558735861:2048]
2025-11-19T14:00:54.914069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436193558735861:2048] [node 2] ICS09 handshake done sender: [1:7574436193558735858:2155] self: [1:7574436193558735859:0] peer: [2:7574436194270878641:0] socket: 17 qp: 130053
2025-11-19T14:00:54.914073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436193558735861:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:54.914078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:54.913811Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436194270878642:2166] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:54.913882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:54.913894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:54.913901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436194270878645:2048]
2025-11-19T14:00:54.913908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436194270878645:2048] [node 1] ICS09 handshake done sender: [2:7574436194270878642:2166] self: [2:7574436194270878641:0] peer: [1:7574436193558735859:0] socket: 16 qp: 130054
2025-11-19T14:00:54.913915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436194270878645:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:54.913924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-19T14:00:55.428523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7574436151321205409:2048] [node 1] ICP34 closed connection by debug command
2025-11-19T14:00:55.430361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:55.430449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436193558735861:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:55.430452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436193558735861:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:55.430482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:55.430496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436193558735861:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:55.429984Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:00:55.429995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436194270878645:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:00:55.430000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436194270878645:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:00:55.430008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:55.432449Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:55.432524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436194270878645:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:55.432526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436194270878645:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:55.434121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436194270878645:2048] [node 1] ICS08 incoming handshake Self# [1:7574436193558735859:0] Peer# [2:7574436194270878641:0] Counter# 1831 LastInputSerial# 47
2025-11-19T14:00:55.436007Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436197853703159:2157] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:55.436248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:55.436262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436193558735861:2048] [node 2] ICS09 handshake done sender: [1:7574436197853703159:2157] self: [1:7574436193558735859:0] peer: [2:7574436194270878641:0] socket: 20 qp: 130200
2025-11-19T14:00:55.436758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436193558735861:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:55.436770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:55.445205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436193558735861:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:00:55.445223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436193558735861:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:00:55.445253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:55.445267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436193558735861:2048] [node 2] ICS08 incoming handshake Self# [2:7574436194270878641:0] Peer# [1:7574436193558735859:0] Counter# 47 LastInputSerial# 1831
2025-11-19T14:00:55.445606Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:55.445690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436193558735861:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:00:55.445721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436193558735861:2048] VirtualId# [1:7574436193558735859:0]
2025-11-19T14:00:55.445733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:55.455167Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436198565845944:2168] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:55.456008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:55.456023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436194270878645:2048] [node 1] ICS09 handshake done sender: [2:7574436198565845944:2168] self: [2:7574436194270878641:0] peer: [1:7574436193558735859:0] socket: 21 qp: 130203
2025-11-19T14:00:55.456656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436194270878645:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:55.456668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:55.456827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:00:55.456831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436194270878645:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:00:55.456835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436194270878645:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:00:55.456840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:55.456871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:55.456929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436194270878645:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:55.456931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436194270878645:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:55.470379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436197853703165:2160]) peer tries to resume nonexistent session Self# [2:7574436194270878641:0] Peer# [1:7574436193558735859:0]
2025-11-19T14:00:55.470397Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436197853703165:2160] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:55.471369Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436198565845949:2170] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:00:55.476652Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:55.476696Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:55.476701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436194270878645:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:55.476724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436194270878645:2048] VirtualId# [2:7574436194270878641:0]
2025-11-19T14:00:55.476755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 11181
TEvUndelivered Cookie# 11182
TEvUndelivered Cookie# 11183
TEvUndelivered Cookie# 11184
2025-11-19T14:00:55.483357Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436197853703166:2161] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:55.483392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:55.484453Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436198565845952:2171] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:55.496037Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436197853703166:2161] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:55.496207Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436198565845952:2171] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:55.496479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:55.496490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:55.496494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436197853703169:2048]
2025-11-19T14:00:55.496499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436197853703169:2048] [node 2] ICS09 handshake done sender: [1:7574436197853703166:2161] self: [1:7574436197853703167:0] peer: [2:7574436198565845951:0] socket: 17 qp: 130230
2025-11-19T14:00:55.496503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436197853703169:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:55.496508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:55.496302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:55.496320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:55.496327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436198565845955:2048]
2025-11-19T14:00:55.496333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436198565845955:2048] [node 1] ICS09 handshake done sender: [2:7574436198565845952:2171] self: [2:7574436198565845951:0] peer: [1:7574436197853703167:0] socket: 16 qp: 130233
2025-11-19T14:00:55.496341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436198565845955:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:55.496351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:00:55.592811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:55.592892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436197853703169:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:55.592900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436197853703169:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:55.592930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:55.592947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436197853703169:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:55.592269Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:00:55.592344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436198565845955:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-19T14:00:55.592350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436198565845955:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:55.592384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:55.592395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436198565845955:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:55.601552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436198565845955:2048] [node 1] ICS08 incoming handshake Self# [1:7574436197853703167:0] Peer# [2:7574436198565845951:0] Counter# 44 LastInputSerial# 1
2025-11-19T14:00:55.606357Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436197853703171:2163] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:55.617136Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436198565845958:2173] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:55.617242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:55.617264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436198565845955:2048] [node 1] ICS09 handshake done sender: [2:7574436198565845958:2173] self: [2:7574436198565845951:0] peer: [1:7574436197853703167:0] socket: 21 qp: 130286
2025-11-19T14:00:55.617856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436198565845955:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:55.617870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:55.619285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:55.619309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436197853703169:2048] [node 2] ICS09 handshake done sender: [1:7574436197853703171:2163] self: [1:7574436197853703167:0] peer: [2:7574436198565845951:0] socket: 20 qp: 130281
2025-11-19T14:00:55.619979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436197853703169:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:55.620011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:55.620045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436197853703169:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:00:55.620048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436197853703169:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:00:55.620080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:55.620093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436197853703169:2048] [node 2] ICS08 incoming handshake Self# [2:7574436198565845951:0] Peer# [1:7574436197853703167:0] Counter# 1 LastInputSerial# 44
2025-11-19T14:00:55.620185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436197853703169:2048] [node 2] ICS07 socket disconnect -1 reason# NewSession
2025-11-19T14:00:55.620521Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:55.620733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436198565845955:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-19T14:00:55.620738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436198565845955:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:55.620746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:55.620770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436198565845955:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:55.624764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436197853703169:2048] [node 2] ICS08 incoming handshake Self# [2:7574436198565845951:0] Peer# [1:7574436197853703167:0] Counter# 1 LastInputSerial# 44
2025-11-19T14:00:55.626924Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436198565845962:2175] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:55.627005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:55.627015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436198565845955:2048] [node 1] ICS09 handshake done sender: [2:7574436198565845962:2175] self: [2:7574436198565845951:0] peer: [1:7574436197853703167:0] socket: 16 qp: 130295
2025-11-19T14:00:55.627456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436198565845955:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:55.627467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:55.628465Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436197853703176:2165] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:55.632106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:55.632130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436197853703169:2048] [node 2] ICS09 handshake done sender: [1:7574436197853703176:2165] self: [1:7574436197853703167:0] peer: [2:7574436198565845951:0] socket: 18 qp: 130294
2025-11-19T14:00:55.632880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436197853703169:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:55.632896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:00:56.141633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:00:56.141712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436198565845955:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-19T14:00:56.141717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436198565845955:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:56.141747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:56.141759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436198565845955:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:56.143890Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:56.143984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436197853703169:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-19T14:00:56.143990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436197853703169:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:56.144048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:56.144065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436197853703169:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:56.150100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436197853703169:2048] [node 2] ICS08 incoming handshake Self# [2:7574436198565845951:0] Peer# [1:7574436197853703167:0] Counter# 72 LastInputSerial# 2885
2025-11-19T14:00:56.150525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436198565845955:2048] [node 1] ICS08 incoming handshake Self# [1:7574436197853703167:0] Peer# [2:7574436198565845951:0] Counter# 2885 LastInputSerial# 72
2025-11-19T14:00:56.157324Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436202148670475:2167] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:56.157853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:56.157872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436197853703169:2048] [node 2] ICS09 handshake done sender: [1:7574436202148670475:2167] self: [1:7574436197853703167:0] peer: [2:7574436198565845951:0] socket: 22 qp: 130427
2025-11-19T14:00:56.158392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436197853703169:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:56.158406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:56.166362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436202860813264:2177] held: no
2025-11-19T14:00:56.166392Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436202860813264:2177] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:56.166501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:56.166518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436198565845955:2048] [node 1] ICS09 handshake done sender: [2:7574436202860813264:2177] self: [2:7574436198565845951:0] peer: [1:7574436197853703167:0] socket: 23 qp: 130426
2025-11-19T14:00:56.167129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436198565845955:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:56.167157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:00:56.268057Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:00:56.268149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436197853703169:2048] [node 2] ICS07 socket disconnect 22 reason# Debug
2025-11-19T14:00:56.268154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436197853703169:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:56.268194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:56.268209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436197853703169:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:56.268519Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:56.268636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:00:56.268645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436198565845955:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:00:56.268649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436198565845955:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:00:56.268660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:56.268693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436198565845955:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:56.268695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436198565845955:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:56.274371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436198565845955:2048] [node 1] ICS08 incoming handshake Self# [1:7574436197853703167:0] Peer# [2:7574436198565845951:0] Counter# 3750 LastInputSerial# 92
2025-11-19T14:00:56.276755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436197853703169:2048] [node 2] ICS08 incoming handshake Self# [2:7574436198565845951:0] Peer# [1:7574436197853703167:0] Counter# 92 LastInputSerial# 3750
2025-11-19T14:00:56.277085Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436202148670480:2169] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:56.277176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:56.277191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436197853703169:2048] [node 2] ICS09 handshake done sender: [1:7574436202148670480:2169] self: [1:7574436197853703167:0] peer: [2:7574436198565845951:0] socket: 16 qp: 130470
2025-11-19T14:00:56.277908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436197853703169:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:56.277920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:56.282196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436202860813268:2180] held: no
2025-11-19T14:00:56.282231Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436202860813268:2180] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:56.282275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:56.282289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436198565845955:2048] [node 1] ICS09 handshake done sender: [2:7574436202860813268:2180] self: [2:7574436198565845951:0] peer: [1:7574436197853703167:0] socket: 17 qp: 130471
2025-11-19T14:00:56.282938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436198565845955:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:56.282951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:00:56.628097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436198565845955:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-11-19T14:00:56.628147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436198565845955:2048] VirtualId# [2:7574436198565845951:0]
2025-11-19T14:00:56.628155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436198565845955:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:56.628181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:56.628206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:00:56.629137Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:56.629232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436197853703169:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:56.629238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436197853703169:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:56.629275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:56.629299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436197853703169:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-19T14:00:56.634410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436202860813272:2182]) peer tries to resume nonexistent session Self# [1:7574436197853703167:0] Peer# [2:7574436198565845951:0]
2025-11-19T14:00:56.634443Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436202860813272:2182] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:56.634996Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436202148670485:2171] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:56.635913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:56.635972Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:56.635977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436197853703169:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:56.636018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436197853703169:2048] VirtualId# [1:7574436197853703167:0]
2025-11-19T14:00:56.636042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:56.647605Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436202860813276:2184] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:56.648913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:56.649235Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436202148670488:2172] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:56.649302Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436202148670490:2173] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:56.660193Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436202860813276:2184] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:56.663910Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436202148670488:2172] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:56.664147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:56.664170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:56.664177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436202148670493:2048]
2025-11-19T14:00:56.664184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436202148670493:2048] [node 2] ICS09 handshake done sender: [1:7574436202148670488:2172] self: [1:7574436202148670487:0] peer: [2:7574436202860813277:0] socket: 16 qp: 130641
2025-11-19T14:00:56.664192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436202148670493:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:56.664201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:56.668452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:56.668476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:56.668486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436202860813279:2048]
2025-11-19T14:00:56.668493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436202860813279:2048] [node 1] ICS09 handshake done sender: [2:7574436202860813276:2184] self: [2:7574436202860813277:0] peer: [1:7574436202148670487:0] socket: 18 qp: 130642
2025-11-19T14:00:56.668501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436202860813279:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:56.668510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:56.668598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436151321205409:2048] [node 1] ICP27 obsolete handshake fail ignored
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:00:57.024692Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:00:57.024804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436202860813279:2048] [node 1] ICS07 socket disconnect 18 reason# Debug
2025-11-19T14:00:57.024812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436202860813279:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:57.024834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:57.024847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436202860813279:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:57.026538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:57.026611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436202148670493:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:57.026616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436202148670493:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:57.026642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:57.026657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436202148670493:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:57.031957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436202148670493:2048] [node 2] ICS08 incoming handshake Self# [2:7574436202860813277:0] Peer# [1:7574436202148670487:0] Counter# 53 LastInputSerial# 2155
2025-11-19T14:00:57.032764Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436206443637791:2174] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:57.032878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:57.032892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436202148670493:2048] [node 2] ICS09 handshake done sender: [1:7574436206443637791:2174] self: [1:7574436202148670487:0] peer: [2:7574436202860813277:0] socket: 22 qp: 130739
2025-11-19T14:00:57.033348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436202148670493:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:57.033359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:57.032486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436202860813279:2048] [node 1] ICS08 incoming handshake Self# [1:7574436202148670487:0] Peer# [2:7574436202860813277:0] Counter# 2155 LastInputSerial# 53
2025-11-19T14:00:57.035337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436207155780579:2187] held: no
2025-11-19T14:00:57.040552Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436207155780579:2187] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:57.040645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:57.040657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436202860813279:2048] [node 1] ICS09 handshake done sender: [2:7574436207155780579:2187] self: [2:7574436202860813277:0] peer: [1:7574436202148670487:0] socket: 23 qp: 130741
2025-11-19T14:00:57.041298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436202860813279:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:57.041315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:00:57.302520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:57.302624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436202148670493:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-19T14:00:57.302629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436202148670493:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:57.302675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:57.302691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436202148670493:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:57.301242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436202860813279:2048] [node 1] ICS01 socket: 23 reason# Debug
2025-11-19T14:00:57.301299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436202860813279:2048] VirtualId# [2:7574436202860813277:0]
2025-11-19T14:00:57.301307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436202860813279:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:57.301343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:57.301366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 13774
TEvUndelivered Cookie# 13775
2025-11-19T14:00:57.318539Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436207155780582:2189]) peer tries to resume nonexistent session Self# [1:7574436202148670487:0] Peer# [2:7574436202860813277:0]
2025-11-19T14:00:57.318568Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436207155780582:2189] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:57.322779Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436206443637796:2176] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:00:57.324390Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:57.324431Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:57.324436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436202148670493:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:57.324463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436202148670493:2048] VirtualId# [1:7574436202148670487:0]
2025-11-19T14:00:57.324486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:57.328830Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436206443637798:2177] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:57.330426Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436207155780586:2191] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:57.330454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:57.330863Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436206443637800:2178] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:57.343534Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436206443637800:2178] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:57.343664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:57.343686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:57.343695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436206443637804:2048]
2025-11-19T14:00:57.343702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436206443637804:2048] [node 2] ICS09 handshake done sender: [1:7574436206443637800:2178] self: [1:7574436206443637799:0] peer: [2:7574436207155780587:0] socket: 16 qp: 130838
2025-11-19T14:00:57.343709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436206443637804:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:57.343718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:57.343320Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436207155780586:2191] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:57.343840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:57.343850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:57.343855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436207155780589:2048]
2025-11-19T14:00:57.343860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436207155780589:2048] [node 1] ICS09 handshake done sender: [2:7574436207155780586:2191] self: [2:7574436207155780587:0] peer: [1:7574436206443637799:0] socket: 17 qp: 130837
2025-11-19T14:00:57.343866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436207155780589:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:57.343872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:00:57.856634Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:00:57.856726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436206443637804:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-11-19T14:00:57.856731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436206443637804:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:57.856770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:57.856785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436206443637804:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:57.857402Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:57.857556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436207155780589:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:57.857561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436207155780589:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:57.857567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:57.857587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436207155780589:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:57.863579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436207155780589:2048] [node 1] ICS08 incoming handshake Self# [1:7574436206443637799:0] Peer# [2:7574436207155780587:0] Counter# 4482 LastInputSerial# 110
2025-11-19T14:00:57.866017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436206443637804:2048] [node 2] ICS08 incoming handshake Self# [2:7574436207155780587:0] Peer# [1:7574436206443637799:0] Counter# 110 LastInputSerial# 4482
2025-11-19T14:00:57.866348Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436206443637806:2179] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:57.866521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:57.866537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436206443637804:2048] [node 2] ICS09 handshake done sender: [1:7574436206443637806:2179] self: [1:7574436206443637799:0] peer: [2:7574436207155780587:0] socket: 18 qp: 131030
2025-11-19T14:00:57.867001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436206443637804:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:57.867013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:57.870503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436207155780592:2194] held: no
2025-11-19T14:00:57.870565Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436207155780592:2194] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:57.870621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:57.870630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436207155780589:2048] [node 1] ICS09 handshake done sender: [2:7574436207155780592:2194] self: [2:7574436207155780587:0] peer: [1:7574436206443637799:0] socket: 19 qp: 131032
2025-11-19T14:00:57.871128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436207155780589:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:57.871144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:00:58.311230Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:00:58.311317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436206443637804:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-19T14:00:58.311323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436206443637804:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:58.311358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:58.311371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436206443637804:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:58.313117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:58.313195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436207155780589:2048] [node 1] ICS07 socket disconnect 19 reason# EndOfStream
2025-11-19T14:00:58.313201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436207155780589:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:58.313229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:58.313245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436207155780589:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:58.315852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436207155780589:2048] [node 1] ICS08 incoming handshake Self# [1:7574436206443637799:0] Peer# [2:7574436207155780587:0] Counter# 8389 LastInputSerial# 209
2025-11-19T14:00:58.317487Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436210738605107:2181] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:58.317612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:58.317628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436206443637804:2048] [node 2] ICS09 handshake done sender: [1:7574436210738605107:2181] self: [1:7574436206443637799:0] peer: [2:7574436207155780587:0] socket: 16 qp: 127
2025-11-19T14:00:58.317969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436206443637804:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:58.317983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:58.320327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436206443637804:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:00:58.320339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436206443637804:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:00:58.320367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:58.320380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436206443637804:2048] [node 2] ICS08 incoming handshake Self# [2:7574436207155780587:0] Peer# [1:7574436206443637799:0] Counter# 209 LastInputSerial# 8389
2025-11-19T14:00:58.320466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:58.320539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436206443637804:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:00:58.320562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436206443637804:2048] VirtualId# [1:7574436206443637799:0]
2025-11-19T14:00:58.320573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:58.323553Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436211450747893:2197] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:58.323711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:58.323725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436207155780589:2048] [node 1] ICS09 handshake done sender: [2:7574436211450747893:2197] self: [2:7574436207155780587:0] peer: [1:7574436206443637799:0] socket: 17 qp: 129
2025-11-19T14:00:58.325577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436207155780589:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:58.325600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:58.325742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:00:58.325746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436207155780589:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:00:58.325750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436207155780589:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:00:58.325759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:58.325792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:58.325848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436207155780589:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:58.325850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436207155780589:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:58.330647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436210738605113:2184]) peer tries to resume nonexistent session Self# [2:7574436207155780587:0] Peer# [1:7574436206443637799:0]
2025-11-19T14:00:58.330666Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436210738605113:2184] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:58.335962Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436211450747898:2199] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:00:58.336886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:58.336933Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:58.336938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436207155780589:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:58.336965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436207155780589:2048] VirtualId# [2:7574436207155780587:0]
2025-11-19T14:00:58.337001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 15799
TEvUndelivered Cookie# 15800
TEvUndelivered Cookie# 15801
TEvUndelivered Cookie# 15802
TEvUndelivered Cookie# 15803
TEvUndelivered Cookie# 15804
TEvUndelivered Cookie# 15805
TEvUndelivered Cookie# 15806
2025-11-19T14:00:58.341397Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436210738605114:2185] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:58.341434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:58.342138Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436211450747901:2200] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:58.352618Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436210738605114:2185] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:58.353970Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436211450747901:2200] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:58.354041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:58.354057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:58.354065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436210738605117:2048]
2025-11-19T14:00:58.354072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436210738605117:2048] [node 2] ICS09 handshake done sender: [1:7574436210738605114:2185] self: [1:7574436210738605115:0] peer: [2:7574436211450747900:0] socket: 17 qp: 143
2025-11-19T14:00:58.354079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436210738605117:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:58.354089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:58.354457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:58.354468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:58.354474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436211450747904:2048]
2025-11-19T14:00:58.354478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436211450747904:2048] [node 1] ICS09 handshake done sender: [2:7574436211450747901:2200] self: [2:7574436211450747900:0] peer: [1:7574436210738605115:0] socket: 16 qp: 144
2025-11-19T14:00:58.354483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436211450747904:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:58.354489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:00:58.637922Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:00:58.638024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436211450747904:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-19T14:00:58.638029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436211450747904:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:58.638054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:58.638065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436211450747904:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:58.639658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:58.639757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436210738605117:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:58.639763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436210738605117:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:58.639805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:58.639823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436210738605117:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:58.644954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436211450747904:2048] [node 1] ICS08 incoming handshake Self# [1:7574436210738605115:0] Peer# [2:7574436211450747900:0] Counter# 1770 LastInputSerial# 41
2025-11-19T14:00:58.645167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436210738605117:2048] [node 2] ICS08 incoming handshake Self# [2:7574436211450747900:0] Peer# [1:7574436210738605115:0] Counter# 41 LastInputSerial# 1770
2025-11-19T14:00:58.645331Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436210738605119:2187] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:58.645424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:58.645436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436210738605117:2048] [node 2] ICS09 handshake done sender: [1:7574436210738605119:2187] self: [1:7574436210738605115:0] peer: [2:7574436211450747900:0] socket: 22 qp: 240
2025-11-19T14:00:58.645745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436210738605117:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:58.645756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:58.647228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436211450747908:2202] held: no
2025-11-19T14:00:58.649006Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436211450747908:2202] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:58.649088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:58.649101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436211450747904:2048] [node 1] ICS09 handshake done sender: [2:7574436211450747908:2202] self: [2:7574436211450747900:0] peer: [1:7574436210738605115:0] socket: 23 qp: 246
2025-11-19T14:00:58.649641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436211450747904:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:58.649659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-19T14:00:59.149163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7574436151321205409:2048] [node 1] ICP34 closed connection by debug command
2025-11-19T14:00:59.150620Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:00:59.150631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436211450747904:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:00:59.150636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436211450747904:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:00:59.150644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:59.150950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:59.151042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436210738605117:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-19T14:00:59.151046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436210738605117:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:59.151080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:59.151095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436210738605117:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:59.153437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:59.153537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436211450747904:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:59.153540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436211450747904:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:59.155166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436211450747904:2048] [node 1] ICS08 incoming handshake Self# [1:7574436210738605115:0] Peer# [2:7574436211450747900:0] Counter# 5510 LastInputSerial# 130
2025-11-19T14:00:59.157298Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436215033572420:2189] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:59.157399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:59.157410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436210738605117:2048] [node 2] ICS09 handshake done sender: [1:7574436215033572420:2189] self: [1:7574436210738605115:0] peer: [2:7574436211450747900:0] socket: 16 qp: 457
2025-11-19T14:00:59.157713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436210738605117:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:59.157738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:59.165345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436210738605117:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:00:59.165361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436210738605117:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:00:59.165392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:59.165407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436210738605117:2048] [node 2] ICS08 incoming handshake Self# [2:7574436211450747900:0] Peer# [1:7574436210738605115:0] Counter# 130 LastInputSerial# 5510
2025-11-19T14:00:59.165610Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:59.165696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436210738605117:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:00:59.165723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436210738605117:2048] VirtualId# [1:7574436210738605115:0]
2025-11-19T14:00:59.165736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:59.167028Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436215745715208:2205] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:59.176953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:59.176979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436211450747904:2048] [node 1] ICS09 handshake done sender: [2:7574436215745715208:2205] self: [2:7574436211450747900:0] peer: [1:7574436210738605115:0] socket: 18 qp: 458
2025-11-19T14:00:59.177419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436211450747904:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:59.177434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:59.177611Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:00:59.177617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436211450747904:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:00:59.177621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436211450747904:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:00:59.177627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:59.177666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:59.177752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436211450747904:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:00:59.177754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436211450747904:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:59.190791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436215033572426:2192]) peer tries to resume nonexistent session Self# [2:7574436211450747900:0] Peer# [1:7574436210738605115:0]
2025-11-19T14:00:59.190818Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436215033572426:2192] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:59.192207Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436215745715213:2207] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:00:59.193497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:59.193538Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:59.193543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436211450747904:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:59.193575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436211450747904:2048] VirtualId# [2:7574436211450747900:0]
2025-11-19T14:00:59.193611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 17105
TEvUndelivered Cookie# 17106
TEvUndelivered Cookie# 17107
TEvUndelivered Cookie# 17108
TEvUndelivered Cookie# 17109
TEvUndelivered Cookie# 17110
2025-11-19T14:00:59.204694Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436215033572427:2193] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:59.204750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:59.209813Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436215745715216:2208] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:59.227984Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436215033572427:2193] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:59.228319Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436215745715216:2208] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:59.228432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:59.228448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:59.228457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436215745715219:2048]
2025-11-19T14:00:59.228464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436215745715219:2048] [node 1] ICS09 handshake done sender: [2:7574436215745715216:2208] self: [2:7574436215745715215:0] peer: [1:7574436215033572428:0] socket: 16 qp: 478
2025-11-19T14:00:59.228473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436215745715219:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:59.228482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:00:59.232034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:59.232062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:59.232067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436215033572430:2048]
2025-11-19T14:00:59.232074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436215033572430:2048] [node 2] ICS09 handshake done sender: [1:7574436215033572427:2193] self: [1:7574436215033572428:0] peer: [2:7574436215745715215:0] socket: 17 qp: 479
2025-11-19T14:00:59.232082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436215033572430:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:59.232089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:00:59.412463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436215033572430:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-19T14:00:59.412826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:59.412912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436215745715219:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:59.412918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436215745715219:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:59.412945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:59.412963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436215745715219:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:59.412593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436215033572430:2048] VirtualId# [1:7574436215033572428:0]
2025-11-19T14:00:59.412604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436215033572430:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:00:59.412648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:59.412671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:00:59.428588Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436215033572432:2195]) peer tries to resume nonexistent session Self# [2:7574436215745715215:0] Peer# [1:7574436215033572428:0]
2025-11-19T14:00:59.428613Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436215033572432:2195] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:00:59.430070Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436215745715221:2209] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:00:59.431003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:00:59.431038Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:00:59.431043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436215745715219:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:00:59.431105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436215745715219:2048] VirtualId# [2:7574436215745715215:0]
2025-11-19T14:00:59.431155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 17250
TEvUndelivered Cookie# 17251
TEvUndelivered Cookie# 17252
TEvUndelivered Cookie# 17253
TEvUndelivered Cookie# 17254
TEvUndelivered Cookie# 17255
TEvUndelivered Cookie# 17256
TEvUndelivered Cookie# 17257
TEvUndelivered Cookie# 17258
TEvUndelivered Cookie# 17259
2025-11-19T14:00:59.440036Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436215033572433:2196] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:59.440104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:00:59.440582Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436215745715224:2210] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:00:59.452624Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436215033572433:2196] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:59.455967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:00:59.455987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:00:59.455995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436215033572436:2048]
2025-11-19T14:00:59.456001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436215033572436:2048] [node 2] ICS09 handshake done sender: [1:7574436215033572433:2196] self: [1:7574436215033572434:0] peer: [2:7574436215745715223:0] socket: 17 qp: 547
2025-11-19T14:00:59.456010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436215033572436:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:59.456021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:59.456230Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436215745715224:2210] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:59.456738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:00:59.456753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:00:59.456759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436215745715227:2048]
2025-11-19T14:00:59.456765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436215745715227:2048] [node 1] ICS09 handshake done sender: [2:7574436215745715224:2210] self: [2:7574436215745715223:0] peer: [1:7574436215033572434:0] socket: 16 qp: 546
2025-11-19T14:00:59.456770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436215745715227:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:59.456778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-19T14:00:59.562578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7574436151321205409:2048] [node 1] ICP34 closed connection by debug command
2025-11-19T14:00:59.562681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:00:59.562779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436215033572436:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:00:59.562784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436215033572436:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:59.562827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:00:59.562840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436215033572436:2048] [node 2] ICS15 start handshake
2025-11-19T14:00:59.564179Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:00:59.564253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436215745715227:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:00:59.564260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436215745715227:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:00:59.564273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:00:59.564310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436215745715227:2048] [node 1] ICS15 start handshake
2025-11-19T14:00:59.569618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436215745715227:2048] [node 1] ICS08 incoming handshake Self# [1:7574436215033572434:0] Peer# [2:7574436215745715223:0] Counter# 135 LastInputSerial# 3
2025-11-19T14:00:59.571284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436215033572436:2048] [node 2] ICS08 incoming handshake Self# [2:7574436215745715223:0] Peer# [1:7574436215033572434:0] Counter# 3 LastInputSerial# 135
2025-11-19T14:00:59.572872Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436215033572438:2198] [node 2] ICH04 handshake succeeded
2025-11-19T14:00:59.573417Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436215745715230:2212] [node 1] ICH04 handshake succeeded
2025-11-19T14:00:59.573506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:00:59.573525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436215033572436:2048] [node 2] ICS09 handshake done sender: [1:7574436215033572438:2198] self: [1:7574436215033572434:0] peer: [2:7574436215745715223:0] socket: 20 qp: 583
2025-11-19T14:00:59.574136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436215033572436:2048] [node 2] ICS10 traffic start
2025-11-19T14:00:59.574151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:00:59.575871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:00:59.575903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436215745715227:2048] [node 1] ICS09 handshake done sender: [2:7574436215745715230:2212] self: [2:7574436215745715223:0] peer: [1:7574436215033572434:0] socket: 21 qp: 582
2025-11-19T14:00:59.576607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436215745715227:2048] [node 1] ICS10 traffic start
2025-11-19T14:00:59.576622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:01:00.100034Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:01:00.100135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436215033572436:2048] [node 2] ICS07 socket disconnect 20 reason# Debug
2025-11-19T14:01:00.100141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436215033572436:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:00.100183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:00.100196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436215033572436:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:00.103724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:00.103824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436215745715227:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-19T14:01:00.103830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436215745715227:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:00.103860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:00.103877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436215745715227:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:00.106901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436215745715227:2048] [node 1] ICS08 incoming handshake Self# [1:7574436215033572434:0] Peer# [2:7574436215745715223:0] Counter# 3498 LastInputSerial# 85
2025-11-19T14:01:00.109142Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436219328539739:2200] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:00.109274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:00.109289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436215033572436:2048] [node 2] ICS09 handshake done sender: [1:7574436219328539739:2200] self: [1:7574436215033572434:0] peer: [2:7574436215745715223:0] socket: 16 qp: 778
2025-11-19T14:01:00.109846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436215033572436:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:00.109859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:00.114082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436215033572436:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:00.114097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436215033572436:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:00.114122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:00.114135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436215033572436:2048] [node 2] ICS08 incoming handshake Self# [2:7574436215745715223:0] Peer# [1:7574436215033572434:0] Counter# 85 LastInputSerial# 3498
2025-11-19T14:01:00.114258Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:00.114340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436215033572436:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:00.114372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436215033572436:2048] VirtualId# [1:7574436215033572434:0]
2025-11-19T14:01:00.114385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:00.126569Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436220040682531:2215] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:00.127168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:00.127185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436215745715227:2048] [node 1] ICS09 handshake done sender: [2:7574436220040682531:2215] self: [2:7574436215745715223:0] peer: [1:7574436215033572434:0] socket: 17 qp: 779
2025-11-19T14:01:00.127768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436215745715227:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:00.127781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:00.127937Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:00.127942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436215745715227:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:00.127949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436215745715227:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:00.127955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:00.127988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:00.128046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436215745715227:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:00.128063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436215745715227:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:00.142317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436219328539745:2203]) peer tries to resume nonexistent session Self# [2:7574436215745715223:0] Peer# [1:7574436215033572434:0]
2025-11-19T14:01:00.142338Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436219328539745:2203] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:00.142922Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436220040682536:2217] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:00.143628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:00.143669Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:00.143673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436215745715227:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:00.143699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436215745715227:2048] VirtualId# [2:7574436215745715223:0]
2025-11-19T14:01:00.143733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 18097
TEvUndelivered Cookie# 18098
TEvUndelivered Cookie# 18099
TEvUndelivered Cookie# 18100
TEvUndelivered Cookie# 18101
TEvUndelivered Cookie# 18102
TEvUndelivered Cookie# 18103
2025-11-19T14:01:00.154705Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436219328539746:2204] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:00.154748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:00.155952Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436220040682539:2218] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:00.164116Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436219328539746:2204] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:00.164264Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436220040682539:2218] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:00.164574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:00.164585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:00.164592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436219328539749:2048]
2025-11-19T14:01:00.164597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436219328539749:2048] [node 2] ICS09 handshake done sender: [1:7574436219328539746:2204] self: [1:7574436219328539747:0] peer: [2:7574436220040682538:0] socket: 17 qp: 799
2025-11-19T14:01:00.164601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436219328539749:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:00.164606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:00.164374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:00.164394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:00.164403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436220040682542:2048]
2025-11-19T14:01:00.164410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436220040682542:2048] [node 1] ICS09 handshake done sender: [2:7574436220040682539:2218] self: [2:7574436220040682538:0] peer: [1:7574436219328539747:0] socket: 16 qp: 801
2025-11-19T14:01:00.164418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436220040682542:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:00.164428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:01:00.265136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436219328539749:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-19T14:01:00.265228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436219328539749:2048] VirtualId# [1:7574436219328539747:0]
2025-11-19T14:01:00.265238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436219328539749:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:00.265263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:00.265281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:00.266111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:00.266205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436220040682542:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:00.266210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436220040682542:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:00.266217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:00.266240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436220040682542:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:00.269538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436219328539751:2206]) peer tries to resume nonexistent session Self# [2:7574436220040682538:0] Peer# [1:7574436219328539747:0]
2025-11-19T14:01:00.269559Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436219328539751:2206] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:00.270097Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436220040682544:2219] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:00.270875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:00.270909Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:00.270914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436220040682542:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:00.270960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436220040682542:2048] VirtualId# [2:7574436220040682538:0]
2025-11-19T14:01:00.270979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:01:00.275268Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436219328539752:2207] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:00.275315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:00.275470Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436220040682547:2220] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:00.283497Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436219328539752:2207] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:00.283608Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436220040682547:2220] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:00.283730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:00.283783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:00.283792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436220040682550:2048]
2025-11-19T14:01:00.283798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436220040682550:2048] [node 1] ICS09 handshake done sender: [2:7574436220040682547:2220] self: [2:7574436220040682546:0] peer: [1:7574436219328539753:0] socket: 16 qp: 836
2025-11-19T14:01:00.283816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436220040682550:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:00.283826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:00.284145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:00.284159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:00.284164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436219328539755:2048]
2025-11-19T14:01:00.284171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436219328539755:2048] [node 2] ICS09 handshake done sender: [1:7574436219328539752:2207] self: [1:7574436219328539753:0] peer: [2:7574436220040682546:0] socket: 17 qp: 835
2025-11-19T14:01:00.284176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436219328539755:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:00.284182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:01:00.495978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:01:00.496172Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:00.496280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436219328539755:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:00.496285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436219328539755:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:00.496297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:00.496314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436219328539755:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:00.498340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:00.498410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436220040682550:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:00.498415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436220040682550:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:00.498441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:00.498459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436220040682550:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:00.502074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436220040682550:2048] [node 1] ICS08 incoming handshake Self# [1:7574436219328539753:0] Peer# [2:7574436220040682546:0] Counter# 767 LastInputSerial# 20
2025-11-19T14:01:00.511055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436219328539755:2048] [node 2] ICS08 incoming handshake Self# [2:7574436220040682546:0] Peer# [1:7574436219328539753:0] Counter# 20 LastInputSerial# 767
2025-11-19T14:01:00.511253Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436219328539757:2209] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:00.511344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:00.511360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436219328539755:2048] [node 2] ICS09 handshake done sender: [1:7574436219328539757:2209] self: [1:7574436219328539753:0] peer: [2:7574436220040682546:0] socket: 20 qp: 887
2025-11-19T14:01:00.511861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436219328539755:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:00.511875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:00.517381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436220040682553:2222] held: no
2025-11-19T14:01:00.527642Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436220040682553:2222] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:00.531939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:00.531965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436220040682550:2048] [node 1] ICS09 handshake done sender: [2:7574436220040682553:2222] self: [2:7574436220040682546:0] peer: [1:7574436219328539753:0] socket: 21 qp: 889
2025-11-19T14:01:00.532538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436220040682550:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:00.532555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:01:00.844986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:01:00.845081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436219328539755:2048] [node 2] ICS07 socket disconnect 20 reason# Debug
2025-11-19T14:01:00.845086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436219328539755:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:00.845122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:00.845137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436219328539755:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:00.845776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:00.845865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436220040682550:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-19T14:01:00.845870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436220040682550:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:00.845907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:00.845927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436220040682550:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:00.849419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436220040682550:2048] [node 1] ICS08 incoming handshake Self# [1:7574436219328539753:0] Peer# [2:7574436220040682546:0] Counter# 2959 LastInputSerial# 74
2025-11-19T14:01:00.855296Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436219328539762:2211] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:00.855575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:00.855590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436219328539755:2048] [node 2] ICS09 handshake done sender: [1:7574436219328539762:2211] self: [1:7574436219328539753:0] peer: [2:7574436220040682546:0] socket: 16 qp: 995
2025-11-19T14:01:00.856345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436219328539755:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:00.856359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:00.860271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436219328539755:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:00.860283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436219328539755:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:00.860310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:00.860330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436219328539755:2048] [node 2] ICS08 incoming handshake Self# [2:7574436220040682546:0] Peer# [1:7574436219328539753:0] Counter# 74 LastInputSerial# 2959
2025-11-19T14:01:00.860454Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:00.860532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436219328539755:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:00.860559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436219328539755:2048] VirtualId# [1:7574436219328539753:0]
2025-11-19T14:01:00.860568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:00.861984Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436220040682558:2225] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:00.862420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:00.862434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436220040682550:2048] [node 1] ICS09 handshake done sender: [2:7574436220040682558:2225] self: [2:7574436220040682546:0] peer: [1:7574436219328539753:0] socket: 18 qp: 998
2025-11-19T14:01:00.862860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436220040682550:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:00.862869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:00.862993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:00.862997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436220040682550:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:00.863000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436220040682550:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:00.863006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:00.863033Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:00.863076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436220040682550:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:00.863078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436220040682550:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:00.877793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436219328539768:2214]) peer tries to resume nonexistent session Self# [2:7574436220040682546:0] Peer# [1:7574436219328539753:0]
2025-11-19T14:01:00.877815Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436219328539768:2214] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:00.878476Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436220040682563:2227] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:00.879364Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:00.879399Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:00.879414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436220040682550:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:00.879439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436220040682550:2048] VirtualId# [2:7574436220040682546:0]
2025-11-19T14:01:00.879472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 18871
TEvUndelivered Cookie# 18872
TEvUndelivered Cookie# 18873
TEvUndelivered Cookie# 18874
TEvUndelivered Cookie# 18875
TEvUndelivered Cookie# 18876
2025-11-19T14:01:00.897275Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436219328539769:2215] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:00.897317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:00.919875Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436219328539769:2215] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:00.924001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:00.924027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:00.924044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436219328539772:2048]
2025-11-19T14:01:00.924054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436219328539772:2048] [node 2] ICS09 handshake done sender: [1:7574436219328539769:2215] self: [1:7574436219328539770:0] peer: [2:7574436220040682565:0] socket: 17 qp: 1026
2025-11-19T14:01:00.924081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436219328539772:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:00.924095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:00.899537Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436220040682566:2228] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:00.928307Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436220040682566:2228] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:00.928718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:00.928737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:00.928749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436220040682569:2048]
2025-11-19T14:01:00.928758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436220040682569:2048] [node 1] ICS09 handshake done sender: [2:7574436220040682566:2228] self: [2:7574436220040682565:0] peer: [1:7574436219328539770:0] socket: 16 qp: 1027
2025-11-19T14:01:00.928781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436220040682569:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:00.928796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:01:00.972290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:01:00.975518Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:00.975619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436219328539772:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-11-19T14:01:00.975900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:00.976031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436220040682569:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-19T14:01:00.976217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436220040682569:2048] VirtualId# [2:7574436220040682565:0]
2025-11-19T14:01:00.976236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436220040682569:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:00.976286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:00.976320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:01:00.976100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436219328539772:2048] VirtualId# [1:7574436219328539770:0]
2025-11-19T14:01:00.976120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436219328539772:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:00.976139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:00.976169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
Subscribe
2025-11-19T14:01:00.994501Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436219328539774:2217] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:00.994550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:00.995047Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436220040682572:2229] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:01.004089Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436219328539774:2217] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:01.008732Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436220040682572:2229] [node 1] ICH04 handshake succeeded
TEvNodeConnected
2025-11-19T14:01:01.009188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:01.009210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:01.009221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436224335649871:2048]
2025-11-19T14:01:01.009230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436224335649871:2048] [node 1] ICS09 handshake done sender: [2:7574436220040682572:2229] self: [2:7574436220040682571:0] peer: [1:7574436219328539775:0] socket: 16 qp: 1058
2025-11-19T14:01:01.009241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436224335649871:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:01.009255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:01.011561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:01.011586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:01.011593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436223623507073:2048]
2025-11-19T14:01:01.011604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436223623507073:2048] [node 2] ICS09 handshake done sender: [1:7574436219328539774:2217] self: [1:7574436219328539775:0] peer: [2:7574436220040682571:0] socket: 17 qp: 1059
2025-11-19T14:01:01.011612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436223623507073:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:01.011621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:01:01.392693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436224335649871:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-19T14:01:01.393011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436224335649871:2048] VirtualId# [2:7574436220040682571:0]
2025-11-19T14:01:01.393025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436224335649871:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:01.393063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:01.393093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:01:01.394741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:01.394844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436223623507073:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:01.394850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436223623507073:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:01.394897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:01.394915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436223623507073:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 19570
TEvUndelivered Cookie# 19571
TEvUndelivered Cookie# 19572
TEvUndelivered Cookie# 19573
2025-11-19T14:01:01.409498Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436224335649873:2230]) peer tries to resume nonexistent session Self# [1:7574436219328539775:0] Peer# [2:7574436220040682571:0]
2025-11-19T14:01:01.409532Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436224335649873:2230] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:01.415712Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436223623507075:2219] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:01:01.418727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:01.418776Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:01.418781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436223623507073:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:01.418857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436223623507073:2048] VirtualId# [1:7574436219328539775:0]
2025-11-19T14:01:01.418885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:01.438183Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436223623507077:2220] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:01.438353Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436224335649877:2232] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:01.438379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:01.439052Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436223623507079:2221] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:01.461105Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436224335649877:2232] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:01.461360Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436223623507079:2221] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:01.462873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:01.462898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:01.462906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436223623507083:2048]
2025-11-19T14:01:01.462913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436223623507083:2048] [node 2] ICS09 handshake done sender: [1:7574436223623507079:2221] self: [1:7574436223623507078:0] peer: [2:7574436224335649878:0] socket: 17 qp: 1234
2025-11-19T14:01:01.462923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436223623507083:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:01.462932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:01.464023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:01.464041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:01.464046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436224335649880:2048]
2025-11-19T14:01:01.464054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436224335649880:2048] [node 1] ICS09 handshake done sender: [2:7574436224335649877:2232] self: [2:7574436224335649878:0] peer: [1:7574436223623507078:0] socket: 16 qp: 1233
2025-11-19T14:01:01.464061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436224335649880:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:01.464080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:01:01.792083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436223623507083:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-19T14:01:01.792214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436223623507083:2048] VirtualId# [1:7574436223623507078:0]
2025-11-19T14:01:01.792224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436223623507083:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:01.792264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:01.792283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:01.794654Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:01.794744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436224335649880:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:01.794750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436224335649880:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:01.794762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:01.794797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436224335649880:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:01.797980Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436223623507085:2222]) peer tries to resume nonexistent session Self# [2:7574436224335649878:0] Peer# [1:7574436223623507078:0]
2025-11-19T14:01:01.798000Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436223623507085:2222] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:01.798569Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436224335649882:2234] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:01.799298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:01.799330Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:01.799336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436224335649880:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:01.799398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436224335649880:2048] VirtualId# [2:7574436224335649878:0]
2025-11-19T14:01:01.799419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:01:01.803991Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436223623507086:2223] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:01.804041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:01.808005Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436224335649885:2235] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:01.824002Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436223623507086:2223] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:01.824418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:01.824441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:01.824454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436223623507089:2048]
2025-11-19T14:01:01.824463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436223623507089:2048] [node 2] ICS09 handshake done sender: [1:7574436223623507086:2223] self: [1:7574436223623507087:0] peer: [2:7574436224335649884:0] socket: 17 qp: 1395
2025-11-19T14:01:01.824476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436223623507089:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:01.824489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:01.827900Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436224335649885:2235] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:01.828229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:01.828252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:01.828263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436224335649888:2048]
2025-11-19T14:01:01.828274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436224335649888:2048] [node 1] ICS09 handshake done sender: [2:7574436224335649885:2235] self: [2:7574436224335649884:0] peer: [1:7574436223623507087:0] socket: 16 qp: 1396
2025-11-19T14:01:01.828286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436224335649888:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:01.828299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:01:02.059863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436224335649888:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-19T14:01:02.059971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436224335649888:2048] VirtualId# [2:7574436224335649884:0]
2025-11-19T14:01:02.059981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436224335649888:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:02.060007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:02.060040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:01:02.060782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:02.060868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436223623507089:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:02.060872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436223623507089:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:02.060906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:02.060920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436223623507089:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-19T14:01:02.071960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436228630617186:2236]) peer tries to resume nonexistent session Self# [1:7574436223623507087:0] Peer# [2:7574436224335649884:0]
2025-11-19T14:01:02.087995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:278: Proxy [2:7574436151321205409:2048] [node 1] ICP35 event processing took too much time PendingConnection:278(16496800ns) {IncomingHandshake:229(16495555ns)}
2025-11-19T14:01:02.088050Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436228630617186:2236] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:02.088772Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436227918474387:2225] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:01:02.095621Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:02.095668Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:02.095674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436223623507089:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:02.096826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436223623507089:2048] VirtualId# [1:7574436223623507087:0]
2025-11-19T14:01:02.096856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:02.109775Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436227918474391:2227] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:02.113495Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436228630617190:2238] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:02.113528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:02.113710Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436227918474390:2226] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:02.138322Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436228630617190:2238] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:02.138756Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436227918474390:2226] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:02.138869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:02.138892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:02.138900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436227918474395:2048]
2025-11-19T14:01:02.138907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436227918474395:2048] [node 2] ICS09 handshake done sender: [1:7574436227918474390:2226] self: [1:7574436227918474389:0] peer: [2:7574436228630617191:0] socket: 18 qp: 1543
2025-11-19T14:01:02.138915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436227918474395:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:02.138924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:02.140181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:02.140199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:02.140205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436228630617193:2048]
2025-11-19T14:01:02.140211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436228630617193:2048] [node 1] ICS09 handshake done sender: [2:7574436228630617190:2238] self: [2:7574436228630617191:0] peer: [1:7574436227918474389:0] socket: 19 qp: 1545
2025-11-19T14:01:02.140218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436228630617193:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:02.140224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:01:02.296605Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:01:02.296690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436227918474395:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-19T14:01:02.296695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436227918474395:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:02.296721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:02.296733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436227918474395:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:02.319869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:02.319971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436228630617193:2048] [node 1] ICS07 socket disconnect 19 reason# EndOfStream
2025-11-19T14:01:02.319978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436228630617193:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:02.320021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:02.320041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436228630617193:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:02.323101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436228630617193:2048] [node 1] ICS08 incoming handshake Self# [1:7574436227918474389:0] Peer# [2:7574436228630617191:0] Counter# 482 LastInputSerial# 11
2025-11-19T14:01:02.325281Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436227918474397:2228] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:02.325540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:02.325555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436227918474395:2048] [node 2] ICS09 handshake done sender: [1:7574436227918474397:2228] self: [1:7574436227918474389:0] peer: [2:7574436228630617191:0] socket: 16 qp: 1669
2025-11-19T14:01:02.326044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436227918474395:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:02.326056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:02.338313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436227918474395:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:02.338327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436227918474395:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:02.338352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:02.338366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436227918474395:2048] [node 2] ICS08 incoming handshake Self# [2:7574436228630617191:0] Peer# [1:7574436227918474389:0] Counter# 11 LastInputSerial# 482
2025-11-19T14:01:02.338481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:02.338563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436227918474395:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:02.338592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436227918474395:2048] VirtualId# [1:7574436227918474389:0]
2025-11-19T14:01:02.338602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:02.362700Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436228630617196:2241] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:02.368066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:02.368101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436228630617193:2048] [node 1] ICS09 handshake done sender: [2:7574436228630617196:2241] self: [2:7574436228630617191:0] peer: [1:7574436227918474389:0] socket: 17 qp: 1680
2025-11-19T14:01:02.368956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436228630617193:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:02.368971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:02.369141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:02.369146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436228630617193:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:02.369151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436228630617193:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:02.369158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:02.369199Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:02.369263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436228630617193:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:02.369265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436228630617193:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:02.398578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436227918474403:2231]) peer tries to resume nonexistent session Self# [2:7574436228630617191:0] Peer# [1:7574436227918474389:0]
2025-11-19T14:01:02.398605Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436227918474403:2231] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:02.399311Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436228630617201:2243] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:02.404068Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:02.404135Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:02.404141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436228630617193:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:02.404174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436228630617193:2048] VirtualId# [2:7574436228630617191:0]
2025-11-19T14:01:02.404212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 20241
TEvUndelivered Cookie# 20242
TEvUndelivered Cookie# 20243
TEvUndelivered Cookie# 20244
TEvUndelivered Cookie# 20245
TEvUndelivered Cookie# 20246
2025-11-19T14:01:02.414157Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436227918474404:2232] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:02.414196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:02.415087Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436228630617204:2244] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:02.423832Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436227918474404:2232] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:02.423969Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436228630617204:2244] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:02.424234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:02.424252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:02.424260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436228630617207:2048]
2025-11-19T14:01:02.424267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436228630617207:2048] [node 1] ICS09 handshake done sender: [2:7574436228630617204:2244] self: [2:7574436228630617203:0] peer: [1:7574436227918474405:0] socket: 16 qp: 1720
2025-11-19T14:01:02.424275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436228630617207:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:02.424285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:02.424456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:02.424467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:02.424471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436227918474407:2048]
2025-11-19T14:01:02.424477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436227918474407:2048] [node 2] ICS09 handshake done sender: [1:7574436227918474404:2232] self: [1:7574436227918474405:0] peer: [2:7574436228630617203:0] socket: 17 qp: 1718
2025-11-19T14:01:02.424481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436227918474407:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:02.424486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:01:02.758747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:01:02.758843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436227918474407:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-11-19T14:01:02.758848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436227918474407:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:02.758885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:02.758899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436227918474407:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:02.763178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:02.763275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436228630617207:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:02.763286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436228630617207:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:02.763318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:02.763340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436228630617207:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:02.770403Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436227918474409:2234] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:02.770547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:02.770566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436227918474407:2048] [node 2] ICS09 handshake done sender: [1:7574436227918474409:2234] self: [1:7574436227918474405:0] peer: [2:7574436228630617203:0] socket: 20 qp: 1925
2025-11-19T14:01:02.771068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436227918474407:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:02.771086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:02.766167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436228630617207:2048] [node 1] ICS08 incoming handshake Self# [1:7574436227918474405:0] Peer# [2:7574436228630617203:0] Counter# 1099 LastInputSerial# 26
2025-11-19T14:01:02.775400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436227918474407:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:02.775413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436227918474407:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:02.775436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:02.775449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436227918474407:2048] [node 2] ICS08 incoming handshake Self# [2:7574436228630617203:0] Peer# [1:7574436227918474405:0] Counter# 26 LastInputSerial# 1099
2025-11-19T14:01:02.775548Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:02.775630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436227918474407:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:02.775655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436227918474407:2048] VirtualId# [1:7574436227918474405:0]
2025-11-19T14:01:02.775666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:02.777636Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436228630617210:2246] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:02.784691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:02.784712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436228630617207:2048] [node 1] ICS09 handshake done sender: [2:7574436228630617210:2246] self: [2:7574436228630617203:0] peer: [1:7574436227918474405:0] socket: 21 qp: 1927
2025-11-19T14:01:02.785501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436228630617207:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:02.785514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:02.785663Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:02.785667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436228630617207:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:02.785674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436228630617207:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:02.785680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:02.785712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:02.785771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436228630617207:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:02.785773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436228630617207:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:02.797514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436227918474415:2237]) peer tries to resume nonexistent session Self# [2:7574436228630617203:0] Peer# [1:7574436227918474405:0]
2025-11-19T14:01:02.797536Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436227918474415:2237] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:02.798220Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436228630617215:2248] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:02.799157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:02.799200Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:02.799205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436228630617207:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:02.799232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436228630617207:2048] VirtualId# [2:7574436228630617203:0]
2025-11-19T14:01:02.799280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 20509
TEvUndelivered Cookie# 20510
TEvUndelivered Cookie# 20511
TEvUndelivered Cookie# 20512
TEvUndelivered Cookie# 20513
TEvUndelivered Cookie# 20514
TEvUndelivered Cookie# 20515
TEvUndelivered Cookie# 20516
2025-11-19T14:01:02.812777Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436227918474416:2238] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:02.812816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:02.813892Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436228630617218:2249] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:02.824375Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436227918474416:2238] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:02.828218Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436228630617218:2249] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:02.828507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:02.828524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:02.828531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436228630617221:2048]
2025-11-19T14:01:02.828538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436228630617221:2048] [node 1] ICS09 handshake done sender: [2:7574436228630617218:2249] self: [2:7574436228630617217:0] peer: [1:7574436227918474417:0] socket: 16 qp: 1950
2025-11-19T14:01:02.828546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436228630617221:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:02.828556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:02.828739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:02.828752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:02.828756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436227918474419:2048]
2025-11-19T14:01:02.828761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436227918474419:2048] [node 2] ICS09 handshake done sender: [1:7574436227918474416:2238] self: [1:7574436227918474417:0] peer: [2:7574436228630617217:0] socket: 17 qp: 1949
2025-11-19T14:01:02.828765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436227918474419:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:02.828770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:01:02.992360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:01:02.992438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436228630617221:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-19T14:01:02.992445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436228630617221:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:02.992471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:02.992483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436228630617221:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:02.994257Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:02.994331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436227918474419:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:02.994336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436227918474419:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:02.994364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:02.994379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436227918474419:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:03.023315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436228630617221:2048] [node 1] ICS08 incoming handshake Self# [1:7574436227918474417:0] Peer# [2:7574436228630617217:0] Counter# 354 LastInputSerial# 8
2025-11-19T14:01:03.023556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436227918474419:2048] [node 2] ICS08 incoming handshake Self# [2:7574436228630617217:0] Peer# [1:7574436227918474417:0] Counter# 8 LastInputSerial# 354
2025-11-19T14:01:03.036342Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436227918474421:2240] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:03.036460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:03.036478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436227918474419:2048] [node 2] ICS09 handshake done sender: [1:7574436227918474421:2240] self: [1:7574436227918474417:0] peer: [2:7574436228630617217:0] socket: 22 qp: 2010
2025-11-19T14:01:03.036981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436227918474419:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:03.036994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:03.050413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436232925584521:2251] held: no
2025-11-19T14:01:03.050443Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436232925584521:2251] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:03.050543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:03.050556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436228630617221:2048] [node 1] ICS09 handshake done sender: [2:7574436232925584521:2251] self: [2:7574436228630617217:0] peer: [1:7574436227918474417:0] socket: 23 qp: 2019
2025-11-19T14:01:03.051327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436228630617221:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:03.051339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:01:03.120082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:01:03.120178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [1 <-> 2] socket: Broken pipe
2025-11-19T14:01:03.120183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436227918474419:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:03.120200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436227918474419:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:03.120208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:03.121700Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:03.121783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436227918474419:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:03.121786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436227918474419:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:03.123911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:03.124004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436228630617221:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-19T14:01:03.124010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436228630617221:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:03.124067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:03.124097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436228630617221:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:03.126123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436228630617221:2048] [node 1] ICS08 incoming handshake Self# [1:7574436227918474417:0] Peer# [2:7574436228630617217:0] Counter# 598 LastInputSerial# 13
2025-11-19T14:01:03.126552Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436232213441722:2242] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:03.126656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:03.126668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436227918474419:2048] [node 2] ICS09 handshake done sender: [1:7574436232213441722:2242] self: [1:7574436227918474417:0] peer: [2:7574436228630617217:0] socket: 16 qp: 2043
2025-11-19T14:01:03.127175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436227918474419:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:03.127185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:03.134504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436227918474419:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:03.134519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436227918474419:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:03.134546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:03.134559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436227918474419:2048] [node 2] ICS08 incoming handshake Self# [2:7574436228630617217:0] Peer# [1:7574436227918474417:0] Counter# 13 LastInputSerial# 598
2025-11-19T14:01:03.134621Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:03.134691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436227918474419:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:03.140573Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436232925584525:2254] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:03.149803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:03.149823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436228630617221:2048] [node 1] ICS09 handshake done sender: [2:7574436232925584525:2254] self: [2:7574436228630617217:0] peer: [1:7574436227918474417:0] socket: 17 qp: 2044
2025-11-19T14:01:03.150573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436228630617221:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:03.150586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:03.150698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:03.150759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436228630617221:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-11-19T14:01:03.150785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436228630617221:2048] VirtualId# [2:7574436228630617217:0]
2025-11-19T14:01:03.150791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436228630617221:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:03.150821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:03.150844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:01:03.177494Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436232213441728:2245] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:03.177532Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:03.177538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436227918474419:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:03.177565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436227918474419:2048] VirtualId# [1:7574436227918474417:0]
2025-11-19T14:01:03.177585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:03.177594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:03.179235Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436232925584531:2256] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:03.202128Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436232213441728:2245] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:03.202808Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436232925584531:2256] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:03.203059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:03.203089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:03.203098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436232925584534:2048]
2025-11-19T14:01:03.203105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436232925584534:2048] [node 1] ICS09 handshake done sender: [2:7574436232925584531:2256] self: [2:7574436232925584530:0] peer: [1:7574436232213441729:0] socket: 17 qp: 2065
2025-11-19T14:01:03.203114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436232925584534:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:03.203124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:03.203658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:03.203673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:03.203679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436232213441731:2048]
2025-11-19T14:01:03.203686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436232213441731:2048] [node 2] ICS09 handshake done sender: [1:7574436232213441728:2245] self: [1:7574436232213441729:0] peer: [2:7574436232925584530:0] socket: 18 qp: 2062
2025-11-19T14:01:03.203691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436232213441731:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:03.203698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:01:03.539474Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:01:03.539558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436232213441731:2048] [node 2] ICS07 socket disconnect 18 reason# Debug
2025-11-19T14:01:03.539564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436232213441731:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:03.539599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:03.539613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436232213441731:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:03.540058Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:03.540149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436232925584534:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:03.540154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436232925584534:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:03.540187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:03.540204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436232925584534:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:03.543127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436232925584534:2048] [node 1] ICS08 incoming handshake Self# [1:7574436232213441729:0] Peer# [2:7574436232925584530:0] Counter# 1674 LastInputSerial# 39
2025-11-19T14:01:03.545183Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436232213441733:2247] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:03.545324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:03.545338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436232213441731:2048] [node 2] ICS09 handshake done sender: [1:7574436232213441733:2247] self: [1:7574436232213441729:0] peer: [2:7574436232925584530:0] socket: 20 qp: 2202
2025-11-19T14:01:03.545766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436232213441731:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:03.545788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:03.551461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436232213441731:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:03.551474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436232213441731:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:03.551508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:03.551522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436232213441731:2048] [node 2] ICS08 incoming handshake Self# [2:7574436232925584530:0] Peer# [1:7574436232213441729:0] Counter# 39 LastInputSerial# 1674
2025-11-19T14:01:03.551668Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:03.551767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436232213441731:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:03.551795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436232213441731:2048] VirtualId# [1:7574436232213441729:0]
2025-11-19T14:01:03.551806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:03.557112Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436232925584537:2258] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:03.557518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:03.557534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436232925584534:2048] [node 1] ICS09 handshake done sender: [2:7574436232925584537:2258] self: [2:7574436232925584530:0] peer: [1:7574436232213441729:0] socket: 21 qp: 2203
2025-11-19T14:01:03.558189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436232925584534:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:03.558205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:03.558419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:03.558427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436232925584534:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:03.558435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436232925584534:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:03.558446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:03.558490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:03.558546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436232925584534:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:03.558548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436232925584534:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:03.586023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436232213441739:2250]) peer tries to resume nonexistent session Self# [2:7574436232925584530:0] Peer# [1:7574436232213441729:0]
2025-11-19T14:01:03.586047Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436232213441739:2250] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:03.586688Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436232925584542:2260] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:03.592139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:03.592197Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:03.592203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436232925584534:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:03.592234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436232925584534:2048] VirtualId# [2:7574436232925584530:0]
2025-11-19T14:01:03.592273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 21048
TEvUndelivered Cookie# 21049
TEvUndelivered Cookie# 21050
TEvUndelivered Cookie# 21051
TEvUndelivered Cookie# 21052
TEvUndelivered Cookie# 21053
TEvUndelivered Cookie# 21054
TEvUndelivered Cookie# 21055
2025-11-19T14:01:03.604816Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436232213441740:2251] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:03.604858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:03.614006Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436232925584545:2261] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:03.647991Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436232213441740:2251] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:03.652191Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436232925584545:2261] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:03.652849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:03.652876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:03.652887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436232925584548:2048]
2025-11-19T14:01:03.652897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436232925584548:2048] [node 1] ICS09 handshake done sender: [2:7574436232925584545:2261] self: [2:7574436232925584544:0] peer: [1:7574436232213441741:0] socket: 16 qp: 2257
2025-11-19T14:01:03.652906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436232925584548:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:03.652919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:03.653129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:03.653146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:03.653155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436232213441743:2048]
2025-11-19T14:01:03.653162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436232213441743:2048] [node 2] ICS09 handshake done sender: [1:7574436232213441740:2251] self: [1:7574436232213441741:0] peer: [2:7574436232925584544:0] socket: 17 qp: 2254
2025-11-19T14:01:03.653167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436232213441743:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:03.653174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:01:03.811880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436232213441743:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-19T14:01:03.811946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436232213441743:2048] VirtualId# [1:7574436232213441741:0]
2025-11-19T14:01:03.811959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436232213441743:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:03.812009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:03.812071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:03.816071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:03.816190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436232925584548:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:03.816196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436232925584548:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:03.816257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:03.816280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436232925584548:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:03.828488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436232213441745:2253]) peer tries to resume nonexistent session Self# [2:7574436232925584544:0] Peer# [1:7574436232213441741:0]
2025-11-19T14:01:03.828516Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436232213441745:2253] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:03.829237Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436232925584550:2262] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:03.830190Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:03.830238Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:03.830245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436232925584548:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:03.830280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436232925584548:2048] VirtualId# [2:7574436232925584544:0]
2025-11-19T14:01:03.830312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:01:03.852000Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436232213441746:2254] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:03.852068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:03.852378Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436232925584553:2263] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:03.868510Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436232213441746:2254] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:03.869374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:03.869391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:03.869396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436232213441749:2048]
2025-11-19T14:01:03.869401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436232213441749:2048] [node 2] ICS09 handshake done sender: [1:7574436232213441746:2254] self: [1:7574436232213441747:0] peer: [2:7574436232925584552:0] socket: 17 qp: 2346
2025-11-19T14:01:03.869408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436232213441749:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:03.869414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:03.868947Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436232925584553:2263] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:03.869173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:03.869188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:03.869198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436232925584556:2048]
2025-11-19T14:01:03.869205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436232925584556:2048] [node 1] ICS09 handshake done sender: [2:7574436232925584553:2263] self: [2:7574436232925584552:0] peer: [1:7574436232213441747:0] socket: 16 qp: 2343
2025-11-19T14:01:03.869214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436232925584556:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:03.869224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-19T14:01:04.124298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7574436151321205409:2048] [node 1] ICP34 closed connection by debug command
2025-11-19T14:01:04.126081Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:04.126088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436232925584556:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:04.126095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436232925584556:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:04.126104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:04.126383Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:04.126479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436232213441749:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:04.126491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436232213441749:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:04.126523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:04.126539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436232213441749:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:04.128657Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:04.128734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436232925584556:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:04.128736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436232925584556:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:04.130392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436232925584556:2048] [node 1] ICS08 incoming handshake Self# [1:7574436232213441747:0] Peer# [2:7574436232925584552:0] Counter# 1243 LastInputSerial# 32
2025-11-19T14:01:04.132497Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436236508409047:2256] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:04.132640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:04.132653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436232213441749:2048] [node 2] ICS09 handshake done sender: [1:7574436236508409047:2256] self: [1:7574436232213441747:0] peer: [2:7574436232925584552:0] socket: 20 qp: 2464
2025-11-19T14:01:04.133103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436232213441749:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:04.133114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:04.143195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436232213441749:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:04.143210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436232213441749:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:04.143236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:04.143248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436232213441749:2048] [node 2] ICS08 incoming handshake Self# [2:7574436232925584552:0] Peer# [1:7574436232213441747:0] Counter# 32 LastInputSerial# 1243
2025-11-19T14:01:04.143344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:04.143418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436232213441749:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:04.143443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436232213441749:2048] VirtualId# [1:7574436232213441747:0]
2025-11-19T14:01:04.143453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:04.154141Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436237220551855:2265] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:04.155226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:04.155244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436232925584556:2048] [node 1] ICS09 handshake done sender: [2:7574436237220551855:2265] self: [2:7574436232925584552:0] peer: [1:7574436232213441747:0] socket: 21 qp: 2466
2025-11-19T14:01:04.156361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436232925584556:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:04.156377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:04.156538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:04.156543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436232925584556:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:04.156548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436232925584556:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:04.156554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:04.156588Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:04.156646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436232925584556:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:04.156648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436232925584556:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:04.168842Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436236508409053:2259]) peer tries to resume nonexistent session Self# [2:7574436232925584552:0] Peer# [1:7574436232213441747:0]
2025-11-19T14:01:04.168861Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436236508409053:2259] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:04.172286Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436237220551860:2267] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:04.173221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:04.173263Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:04.173268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436232925584556:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:04.173297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436232925584556:2048] VirtualId# [2:7574436232925584552:0]
2025-11-19T14:01:04.173332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 21468
TEvUndelivered Cookie# 21469
TEvUndelivered Cookie# 21470
TEvUndelivered Cookie# 21471
TEvUndelivered Cookie# 21472
TEvUndelivered Cookie# 21473
2025-11-19T14:01:04.205914Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436236508409054:2260] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:04.205954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:04.207054Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436237220551863:2268] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:04.216282Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436236508409054:2260] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:04.216699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:04.216706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:04.216711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436236508409057:2048]
2025-11-19T14:01:04.216715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436236508409057:2048] [node 2] ICS09 handshake done sender: [1:7574436236508409054:2260] self: [1:7574436236508409055:0] peer: [2:7574436237220551862:0] socket: 17 qp: 2497
2025-11-19T14:01:04.216720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436236508409057:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:04.216726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:04.216403Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436237220551863:2268] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:04.216499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:04.216514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:04.216526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436237220551866:2048]
2025-11-19T14:01:04.216533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436237220551866:2048] [node 1] ICS09 handshake done sender: [2:7574436237220551863:2268] self: [2:7574436237220551862:0] peer: [1:7574436236508409055:0] socket: 16 qp: 2498
2025-11-19T14:01:04.216542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436237220551866:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:04.216552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:01:04.278923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:01:04.279193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:04.279275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436236508409057:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-11-19T14:01:04.279335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436236508409057:2048] VirtualId# [1:7574436236508409055:0]
2025-11-19T14:01:04.279343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436236508409057:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:04.279352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:04.279369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:04.281339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:04.281416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436237220551866:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-19T14:01:04.282036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436237220551866:2048] VirtualId# [2:7574436237220551862:0]
2025-11-19T14:01:04.282044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436237220551866:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:04.282090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:04.282111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
Subscribe
2025-11-19T14:01:04.306043Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436236508409059:2262] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:04.306094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:04.306265Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436237220551869:2269] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:04.312973Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436236508409059:2262] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:04.313087Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436237220551869:2269] [node 1] ICH04 handshake succeeded
TEvNodeConnected
2025-11-19T14:01:04.313220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:04.313236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:04.313243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436237220551872:2048]
2025-11-19T14:01:04.313253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436237220551872:2048] [node 1] ICS09 handshake done sender: [2:7574436237220551869:2269] self: [2:7574436237220551868:0] peer: [1:7574436236508409060:0] socket: 16 qp: 2528
2025-11-19T14:01:04.313260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436237220551872:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:04.313269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:04.315252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:04.315268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:04.315274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436236508409062:2048]
2025-11-19T14:01:04.315280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436236508409062:2048] [node 2] ICS09 handshake done sender: [1:7574436236508409059:2262] self: [1:7574436236508409060:0] peer: [2:7574436237220551868:0] socket: 17 qp: 2529
2025-11-19T14:01:04.315285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436236508409062:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:04.315291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:01:04.760439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:01:04.760522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436236508409062:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-11-19T14:01:04.760528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436236508409062:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:04.760570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:04.760584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436236508409062:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:04.766402Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:04.766507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436237220551872:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:04.766513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436237220551872:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:04.766547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:04.766564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436237220551872:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:04.770125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436237220551872:2048] [node 1] ICS08 incoming handshake Self# [1:7574436236508409060:0] Peer# [2:7574436237220551868:0] Counter# 2994 LastInputSerial# 74
2025-11-19T14:01:04.773463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436236508409062:2048] [node 2] ICS08 incoming handshake Self# [2:7574436237220551868:0] Peer# [1:7574436236508409060:0] Counter# 74 LastInputSerial# 2994
2025-11-19T14:01:04.779969Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436236508409064:2264] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:04.780808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:04.780844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436236508409062:2048] [node 2] ICS09 handshake done sender: [1:7574436236508409064:2264] self: [1:7574436236508409060:0] peer: [2:7574436237220551868:0] socket: 20 qp: 2582
2025-11-19T14:01:04.781341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436236508409062:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:04.781354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:04.783010Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436237220551875:2271] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:04.783132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:04.783142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436237220551872:2048] [node 1] ICS09 handshake done sender: [2:7574436237220551875:2271] self: [2:7574436237220551868:0] peer: [1:7574436236508409060:0] socket: 21 qp: 2583
2025-11-19T14:01:04.783810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436237220551872:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:04.783825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-19T14:01:05.130040Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:05.130104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436236508409062:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-19T14:01:05.130107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436236508409062:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:05.130134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:05.130147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436236508409062:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:05.128283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7574436151321205409:2048] [node 1] ICP34 closed connection by debug command
2025-11-19T14:01:05.128342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:05.128416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436237220551872:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-19T14:01:05.128422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436237220551872:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:05.128427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:05.128440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436237220551872:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:05.139084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436236508409062:2048] [node 2] ICS08 incoming handshake Self# [2:7574436237220551868:0] Peer# [1:7574436236508409060:0] Counter# 140 LastInputSerial# 5677
2025-11-19T14:01:05.142076Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436240803376365:2266] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:05.138942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436237220551872:2048] [node 1] ICS08 incoming handshake Self# [1:7574436236508409060:0] Peer# [2:7574436237220551868:0] Counter# 5677 LastInputSerial# 140
2025-11-19T14:01:05.142214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:05.142232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436236508409062:2048] [node 2] ICS09 handshake done sender: [1:7574436240803376365:2266] self: [1:7574436236508409060:0] peer: [2:7574436237220551868:0] socket: 18 qp: 2690
2025-11-19T14:01:05.142712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436236508409062:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:05.142725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:05.150687Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436241515519177:2274] held: no
2025-11-19T14:01:05.152304Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436241515519177:2274] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:05.153813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:05.153825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436237220551872:2048] [node 1] ICS09 handshake done sender: [2:7574436241515519177:2274] self: [2:7574436237220551868:0] peer: [1:7574436236508409060:0] socket: 19 qp: 2693
2025-11-19T14:01:05.154399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436237220551872:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:05.154411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:01:05.300043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:01:05.300202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:05.300286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436236508409062:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-19T14:01:05.300291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436236508409062:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:05.300300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:05.300316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436236508409062:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:05.302183Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:05.302252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436237220551872:2048] [node 1] ICS07 socket disconnect 19 reason# EndOfStream
2025-11-19T14:01:05.302257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436237220551872:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:05.302284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:05.302298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436237220551872:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:05.315086Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436240803376370:2268] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:05.313434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436237220551872:2048] [node 1] ICS08 incoming handshake Self# [1:7574436236508409060:0] Peer# [2:7574436237220551868:0] Counter# 6519 LastInputSerial# 160
2025-11-19T14:01:05.317811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:05.317831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436236508409062:2048] [node 2] ICS09 handshake done sender: [1:7574436240803376370:2268] self: [1:7574436236508409060:0] peer: [2:7574436237220551868:0] socket: 16 qp: 2765
2025-11-19T14:01:05.318221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436236508409062:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:05.318234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:05.318303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436236508409062:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:05.318309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436236508409062:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:05.318325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:05.318338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436236508409062:2048] [node 2] ICS08 incoming handshake Self# [2:7574436237220551868:0] Peer# [1:7574436236508409060:0] Counter# 160 LastInputSerial# 6519
2025-11-19T14:01:05.318357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:05.318421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436236508409062:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:05.318444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436236508409062:2048] VirtualId# [1:7574436236508409060:0]
2025-11-19T14:01:05.318450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:05.320963Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436241515519181:2277] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:05.327664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:05.327687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436237220551872:2048] [node 1] ICS09 handshake done sender: [2:7574436241515519181:2277] self: [2:7574436237220551868:0] peer: [1:7574436236508409060:0] socket: 20 qp: 2767
2025-11-19T14:01:05.328240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436237220551872:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:05.328255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:05.328395Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:05.328401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436237220551872:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:05.328406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436237220551872:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:05.328432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:05.328484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:05.328545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436237220551872:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:05.328547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436237220551872:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:05.340690Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436240803376376:2271]) peer tries to resume nonexistent session Self# [2:7574436237220551868:0] Peer# [1:7574436236508409060:0]
2025-11-19T14:01:05.340712Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436240803376376:2271] [node 2] ICH08 Continuation request rejected by proxy
TEvNodeDisconnected
TEvUndelivered Cookie# 23060
TEvUndelivered Cookie# 23061
TEvUndelivered Cookie# 23062
TEvUndelivered Cookie# 23063
TEvUndelivered Cookie# 23064
TEvUndelivered Cookie# 23065
TEvUndelivered Cookie# 23066
2025-11-19T14:01:05.341161Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436241515519186:2279] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:05.341766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:05.341796Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:05.341800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436237220551872:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:05.341822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436237220551872:2048] VirtualId# [2:7574436237220551868:0]
2025-11-19T14:01:05.341854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:01:05.345536Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436240803376377:2272] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:05.345567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:05.383006Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436240803376377:2272] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:05.384140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:05.384163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:05.384172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436240803376380:2048]
2025-11-19T14:01:05.384180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436240803376380:2048] [node 2] ICS09 handshake done sender: [1:7574436240803376377:2272] self: [1:7574436240803376378:0] peer: [2:7574436241515519188:0] socket: 17 qp: 2785
2025-11-19T14:01:05.384189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436240803376380:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:05.384199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:05.352500Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436241515519189:2280] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:05.384061Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436241515519189:2280] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:05.384421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:05.384431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:05.384436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436241515519192:2048]
2025-11-19T14:01:05.384442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436241515519192:2048] [node 1] ICS09 handshake done sender: [2:7574436241515519189:2280] self: [2:7574436241515519188:0] peer: [1:7574436240803376378:0] socket: 16 qp: 2786
2025-11-19T14:01:05.384447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436241515519192:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:05.384453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:01:05.896573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436241515519192:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-19T14:01:05.896664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436241515519192:2048] VirtualId# [2:7574436241515519188:0]
2025-11-19T14:01:05.896673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436241515519192:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:05.896699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:05.896720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:01:05.897869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:05.897960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436240803376380:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:05.897965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436240803376380:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:05.898005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:05.898019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436240803376380:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-19T14:01:05.901792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436241515519194:2281]) peer tries to resume nonexistent session Self# [1:7574436240803376378:0] Peer# [2:7574436241515519188:0]
2025-11-19T14:01:05.901817Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436241515519194:2281] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:05.902235Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436240803376382:2274] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:01:05.906867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:05.906907Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:05.906914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436240803376380:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:05.908561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436240803376380:2048] VirtualId# [1:7574436240803376378:0]
2025-11-19T14:01:05.908589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:05.927857Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436241515519195:2282] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:05.927916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:05.928176Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436240803376385:2275] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:05.950049Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436241515519195:2282] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:05.951532Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436240803376385:2275] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:05.951866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:05.951883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:05.951891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436240803376388:2048]
2025-11-19T14:01:05.951898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436240803376388:2048] [node 2] ICS09 handshake done sender: [1:7574436240803376385:2275] self: [1:7574436240803376384:0] peer: [2:7574436241515519196:0] socket: 17 qp: 2935
2025-11-19T14:01:05.951907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436240803376388:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:05.951916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:05.952166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:05.952178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:05.952183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436241515519198:2048]
2025-11-19T14:01:05.952188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436241515519198:2048] [node 1] ICS09 handshake done sender: [2:7574436241515519195:2282] self: [2:7574436241515519196:0] peer: [1:7574436240803376384:0] socket: 16 qp: 2936
2025-11-19T14:01:05.952192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436241515519198:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:05.952197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:01:06.173526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:01:06.173830Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:06.173912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436240803376388:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:06.173917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436240803376388:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:06.173926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:06.173942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436240803376388:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:06.184618Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:06.184719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436241515519198:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:06.184725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436241515519198:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:06.184769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:06.184787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436241515519198:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:06.186118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436241515519198:2048] [node 1] ICS08 incoming handshake Self# [1:7574436240803376384:0] Peer# [2:7574436241515519196:0] Counter# 1047 LastInputSerial# 23
2025-11-19T14:01:06.189278Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436245098343686:2276] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:06.189387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:06.189402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436240803376388:2048] [node 2] ICS09 handshake done sender: [1:7574436245098343686:2276] self: [1:7574436240803376384:0] peer: [2:7574436241515519196:0] socket: 20 qp: 2999
2025-11-19T14:01:06.189804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436240803376388:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:06.189815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:06.193360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436240803376388:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:06.193373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436240803376388:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:06.193395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:06.193417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436240803376388:2048] [node 2] ICS08 incoming handshake Self# [2:7574436241515519196:0] Peer# [1:7574436240803376384:0] Counter# 23 LastInputSerial# 1047
2025-11-19T14:01:06.193532Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:06.193606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436240803376388:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:06.198465Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436245810486497:2285] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:06.205178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:06.205202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436241515519198:2048] [node 1] ICS09 handshake done sender: [2:7574436245810486497:2285] self: [2:7574436241515519196:0] peer: [1:7574436240803376384:0] socket: 21 qp: 3000
2025-11-19T14:01:06.205749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436241515519198:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:06.205762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:06.205850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:06.205907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436241515519198:2048] [node 1] ICS01 socket: 21 reason# EndOfStream
2025-11-19T14:01:06.205933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436241515519198:2048] VirtualId# [2:7574436241515519196:0]
2025-11-19T14:01:06.205938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436241515519198:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:06.205964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:06.205986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:01:06.221857Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436245098343692:2279] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:06.221901Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:06.221906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436240803376388:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:06.221933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436240803376388:2048] VirtualId# [1:7574436240803376384:0]
2025-11-19T14:01:06.221952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:06.221961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:06.223189Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436245810486503:2287] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:06.237102Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436245098343692:2279] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:06.237219Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436245810486503:2287] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:06.237532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:06.237551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:06.237560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436245810486506:2048]
2025-11-19T14:01:06.237566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436245810486506:2048] [node 1] ICS09 handshake done sender: [2:7574436245810486503:2287] self: [2:7574436245810486502:0] peer: [1:7574436245098343693:0] socket: 16 qp: 3010
2025-11-19T14:01:06.237574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436245810486506:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:06.237582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:06.238201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:06.238217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:06.238223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436245098343695:2048]
2025-11-19T14:01:06.238230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436245098343695:2048] [node 2] ICS09 handshake done sender: [1:7574436245098343692:2279] self: [1:7574436245098343693:0] peer: [2:7574436245810486502:0] socket: 17 qp: 3009
2025-11-19T14:01:06.238236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436245098343695:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:06.238243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:01:06.541874Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:01:06.541959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436245098343695:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-11-19T14:01:06.541965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436245098343695:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:06.541986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:06.542000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436245098343695:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:06.545463Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:06.545550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436245810486506:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:06.545556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436245810486506:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:06.545592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:06.545607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436245810486506:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:06.546784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436245810486506:2048] [node 1] ICS08 incoming handshake Self# [1:7574436245098343693:0] Peer# [2:7574436245810486502:0] Counter# 1547 LastInputSerial# 37
2025-11-19T14:01:06.548680Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436245098343697:2281] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:06.550212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:06.550227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436245098343695:2048] [node 2] ICS09 handshake done sender: [1:7574436245098343697:2281] self: [1:7574436245098343693:0] peer: [2:7574436245810486502:0] socket: 20 qp: 3097
2025-11-19T14:01:06.550570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436245098343695:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:06.550581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:06.569734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436245098343695:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:06.569749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436245098343695:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:06.569777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:06.569792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436245098343695:2048] [node 2] ICS08 incoming handshake Self# [2:7574436245810486502:0] Peer# [1:7574436245098343693:0] Counter# 37 LastInputSerial# 1547
2025-11-19T14:01:06.569937Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:06.570016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436245098343695:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:06.570500Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436245810486509:2289] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:06.570665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:06.570679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436245810486506:2048] [node 1] ICS09 handshake done sender: [2:7574436245810486509:2289] self: [2:7574436245810486502:0] peer: [1:7574436245098343693:0] socket: 21 qp: 3098
2025-11-19T14:01:06.571200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436245810486506:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:06.571211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:06.571300Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:06.571345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436245810486506:2048] [node 1] ICS01 socket: 21 reason# EndOfStream
2025-11-19T14:01:06.571370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436245810486506:2048] VirtualId# [2:7574436245810486502:0]
2025-11-19T14:01:06.571374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436245810486506:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:06.571395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:06.571413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:01:06.599082Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436245098343703:2284] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:06.599119Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:06.599125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436245098343695:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:06.599159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436245098343695:2048] VirtualId# [1:7574436245098343693:0]
2025-11-19T14:01:06.599180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:06.599188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:06.601172Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436245810486515:2291] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:06.613345Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436245098343703:2284] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:06.613715Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436245810486515:2291] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:06.615037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:06.615061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:06.615075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436245810486518:2048]
2025-11-19T14:01:06.615083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436245810486518:2048] [node 1] ICS09 handshake done sender: [2:7574436245810486515:2291] self: [2:7574436245810486514:0] peer: [1:7574436245098343704:0] socket: 16 qp: 3137
2025-11-19T14:01:06.615093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436245810486518:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:06.615104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:06.615673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:06.615692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:06.615700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436245098343706:2048]
2025-11-19T14:01:06.615706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436245098343706:2048] [node 2] ICS09 handshake done sender: [1:7574436245098343703:2284] self: [1:7574436245098343704:0] peer: [2:7574436245810486514:0] socket: 17 qp: 3136
2025-11-19T14:01:06.615712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436245098343706:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:06.615720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:01:07.103904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:01:07.104007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436245810486518:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-19T14:01:07.104013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436245810486518:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:07.104038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:07.104051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436245810486518:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:07.108022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:07.108108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436245098343706:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:07.108114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436245098343706:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:07.108176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:07.108193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436245098343706:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:07.118325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436245810486518:2048] [node 1] ICS08 incoming handshake Self# [1:7574436245098343704:0] Peer# [2:7574436245810486514:0] Counter# 1999 LastInputSerial# 50
2025-11-19T14:01:07.118873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436245098343706:2048] [node 2] ICS08 incoming handshake Self# [2:7574436245810486514:0] Peer# [1:7574436245098343704:0] Counter# 50 LastInputSerial# 1999
2025-11-19T14:01:07.119219Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436249393311004:2286] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:07.119301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:07.119318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436245098343706:2048] [node 2] ICS09 handshake done sender: [1:7574436249393311004:2286] self: [1:7574436245098343704:0] peer: [2:7574436245810486514:0] socket: 22 qp: 3287
2025-11-19T14:01:07.119662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436245098343706:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:07.119675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:07.121208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436250105453818:2293] held: no
2025-11-19T14:01:07.128291Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436250105453818:2293] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:07.131976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:07.131992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436245810486518:2048] [node 1] ICS09 handshake done sender: [2:7574436250105453818:2293] self: [2:7574436245810486514:0] peer: [1:7574436245098343704:0] socket: 23 qp: 3292
2025-11-19T14:01:07.132459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436245810486518:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:07.132475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:01:07.348021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:01:07.351929Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:07.352013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436245098343706:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-11-19T14:01:07.352019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436245098343706:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:07.352027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:07.352055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436245098343706:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:07.353756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:07.353829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436245810486518:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-19T14:01:07.353835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436245810486518:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:07.353859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:07.353875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436245810486518:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:07.356340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436245810486518:2048] [node 1] ICS08 incoming handshake Self# [1:7574436245098343704:0] Peer# [2:7574436245810486514:0] Counter# 3128 LastInputSerial# 78
2025-11-19T14:01:07.358540Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436249393311009:2288] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:07.358646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:07.358660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436245098343706:2048] [node 2] ICS09 handshake done sender: [1:7574436249393311009:2288] self: [1:7574436245098343704:0] peer: [2:7574436245810486514:0] socket: 16 qp: 3345
2025-11-19T14:01:07.359075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436245098343706:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:07.359087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:07.361816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436245098343706:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:07.361828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436245098343706:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:07.361853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:07.361865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436245098343706:2048] [node 2] ICS08 incoming handshake Self# [2:7574436245810486514:0] Peer# [1:7574436245098343704:0] Counter# 78 LastInputSerial# 3128
2025-11-19T14:01:07.361954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:07.362029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436245098343706:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:07.362054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436245098343706:2048] VirtualId# [1:7574436245098343704:0]
2025-11-19T14:01:07.362063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:07.376692Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436250105453822:2296] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:07.376915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:07.376931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436245810486518:2048] [node 1] ICS09 handshake done sender: [2:7574436250105453822:2296] self: [2:7574436245810486514:0] peer: [1:7574436245098343704:0] socket: 18 qp: 3346
2025-11-19T14:01:07.377390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436245810486518:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:07.377402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:07.377525Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:07.377529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436245810486518:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:07.377535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436245810486518:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:07.377542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:07.377574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:07.377620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436245810486518:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:07.377622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436245810486518:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:07.384169Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436249393311015:2291]) peer tries to resume nonexistent session Self# [2:7574436245810486514:0] Peer# [1:7574436245098343704:0]
2025-11-19T14:01:07.384191Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436249393311015:2291] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:07.387781Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436250105453827:2298] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:07.388837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:07.388872Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:07.388877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436245810486518:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:07.388900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436245810486518:2048] VirtualId# [2:7574436245810486514:0]
2025-11-19T14:01:07.388943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 25076
TEvUndelivered Cookie# 25077
TEvUndelivered Cookie# 25078
TEvUndelivered Cookie# 25079
TEvUndelivered Cookie# 25080
TEvUndelivered Cookie# 25081
TEvUndelivered Cookie# 25082
TEvUndelivered Cookie# 25083
2025-11-19T14:01:07.408174Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436249393311016:2292] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:07.408224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:07.415050Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436250105453830:2299] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:07.433264Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436249393311016:2292] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:07.433781Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436250105453830:2299] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:07.433934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:07.433954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:07.433964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436250105453833:2048]
2025-11-19T14:01:07.433971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436250105453833:2048] [node 1] ICS09 handshake done sender: [2:7574436250105453830:2299] self: [2:7574436250105453829:0] peer: [1:7574436249393311017:0] socket: 16 qp: 3362
2025-11-19T14:01:07.433979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436250105453833:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:07.433991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:07.434168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:07.434184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:07.434190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436249393311019:2048]
2025-11-19T14:01:07.434196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436249393311019:2048] [node 2] ICS09 handshake done sender: [1:7574436249393311016:2292] self: [1:7574436249393311017:0] peer: [2:7574436250105453829:0] socket: 17 qp: 3360
2025-11-19T14:01:07.434201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436249393311019:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:07.434208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:01:07.845306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:01:07.846228Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:07.846313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436249393311019:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:07.846319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436249393311019:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:07.846332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:07.846354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436249393311019:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:07.851289Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:07.851459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436250105453833:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:07.851465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436250105453833:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:07.851475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:07.851497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436250105453833:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:07.852812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436250105453833:2048] [node 1] ICS08 incoming handshake Self# [1:7574436249393311017:0] Peer# [2:7574436250105453829:0] Counter# 1922 LastInputSerial# 47
2025-11-19T14:01:07.856361Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436249393311021:2294] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:07.856500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:07.856515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436249393311019:2048] [node 2] ICS09 handshake done sender: [1:7574436249393311021:2294] self: [1:7574436249393311017:0] peer: [2:7574436250105453829:0] socket: 20 qp: 3547
2025-11-19T14:01:07.856918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436249393311019:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:07.856931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:07.860101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436249393311019:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:07.860113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436249393311019:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:07.860140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:07.860166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436249393311019:2048] [node 2] ICS08 incoming handshake Self# [2:7574436250105453829:0] Peer# [1:7574436249393311017:0] Counter# 47 LastInputSerial# 1922
2025-11-19T14:01:07.860259Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:07.860332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436249393311019:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:07.865190Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436250105453836:2301] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:07.866461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:07.866479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436250105453833:2048] [node 1] ICS09 handshake done sender: [2:7574436250105453836:2301] self: [2:7574436250105453829:0] peer: [1:7574436249393311017:0] socket: 21 qp: 3549
2025-11-19T14:01:07.867073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436250105453833:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:07.867085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:07.867212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:07.867272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436250105453833:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-11-19T14:01:07.867277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436250105453833:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:07.867283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:07.867306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436250105453833:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:07.880034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436249393311019:2048] [node 2] ICS08 incoming handshake Self# [2:7574436250105453829:0] Peer# [1:7574436249393311017:0] Counter# 48 LastInputSerial# 1922
2025-11-19T14:01:07.882300Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436250105453841:2303] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:07.882769Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436249393311027:2297] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:07.882856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:07.882871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436250105453833:2048] [node 1] ICS09 handshake done sender: [2:7574436250105453841:2303] self: [2:7574436250105453829:0] peer: [1:7574436249393311017:0] socket: 18 qp: 3557
2025-11-19T14:01:07.883235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436250105453833:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:07.883249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:07.883798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:07.883815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436249393311019:2048] [node 2] ICS09 handshake done sender: [1:7574436249393311027:2297] self: [1:7574436249393311017:0] peer: [2:7574436250105453829:0] socket: 19 qp: 3556
2025-11-19T14:01:07.884261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436249393311019:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:07.884273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:01:08.435939Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:01:08.436062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436249393311019:2048] [node 2] ICS07 socket disconnect 19 reason# Debug
2025-11-19T14:01:08.436070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436249393311019:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:08.436124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:08.436139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436249393311019:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:08.438896Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:08.439012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436250105453833:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-19T14:01:08.439018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436250105453833:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:08.439024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:08.439045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436250105453833:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:08.442897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436250105453833:2048] [node 1] ICS08 incoming handshake Self# [1:7574436249393311017:0] Peer# [2:7574436250105453829:0] Counter# 5358 LastInputSerial# 130
2025-11-19T14:01:08.444356Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436253688278326:2299] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:08.444611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:08.444625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436249393311019:2048] [node 2] ICS09 handshake done sender: [1:7574436253688278326:2299] self: [1:7574436249393311017:0] peer: [2:7574436250105453829:0] socket: 16 qp: 3825
2025-11-19T14:01:08.445092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436249393311019:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:08.445103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:08.447710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436249393311019:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:08.447721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436249393311019:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:08.447768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:08.447781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436249393311019:2048] [node 2] ICS08 incoming handshake Self# [2:7574436250105453829:0] Peer# [1:7574436249393311017:0] Counter# 130 LastInputSerial# 5358
2025-11-19T14:01:08.447942Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:08.448017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436249393311019:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:08.448044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436249393311019:2048] VirtualId# [1:7574436249393311017:0]
2025-11-19T14:01:08.448056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:08.449271Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436254400421142:2305] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:08.449407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:08.449419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436250105453833:2048] [node 1] ICS09 handshake done sender: [2:7574436254400421142:2305] self: [2:7574436250105453829:0] peer: [1:7574436249393311017:0] socket: 17 qp: 3828
2025-11-19T14:01:08.449955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436250105453833:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:08.449965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:08.450090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:08.450095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436250105453833:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:08.450098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436250105453833:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:08.450103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:08.450131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:08.450173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436250105453833:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:08.450175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436250105453833:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:08.455668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436253688278332:2302]) peer tries to resume nonexistent session Self# [2:7574436250105453829:0] Peer# [1:7574436249393311017:0]
2025-11-19T14:01:08.455685Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436253688278332:2302] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:08.456196Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436254400421147:2307] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:08.456987Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:08.457017Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:08.457021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436250105453833:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:08.457049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436250105453833:2048] VirtualId# [2:7574436250105453829:0]
2025-11-19T14:01:08.457079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 26381
TEvUndelivered Cookie# 26382
TEvUndelivered Cookie# 26383
TEvUndelivered Cookie# 26384
TEvUndelivered Cookie# 26385
TEvUndelivered Cookie# 26386
TEvUndelivered Cookie# 26387
TEvUndelivered Cookie# 26388
2025-11-19T14:01:08.461155Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436253688278333:2303] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:08.461190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:08.461788Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436254400421150:2308] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:08.465170Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436253688278333:2303] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:08.465298Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436254400421150:2308] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:08.465385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:08.465401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:08.465408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436254400421153:2048]
2025-11-19T14:01:08.465414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436254400421153:2048] [node 1] ICS09 handshake done sender: [2:7574436254400421150:2308] self: [2:7574436254400421149:0] peer: [1:7574436253688278334:0] socket: 16 qp: 3842
2025-11-19T14:01:08.465421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436254400421153:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:08.465429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:08.465586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:08.465595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:08.465599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436253688278336:2048]
2025-11-19T14:01:08.465604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436253688278336:2048] [node 2] ICS09 handshake done sender: [1:7574436253688278333:2303] self: [1:7574436253688278334:0] peer: [2:7574436254400421149:0] socket: 17 qp: 3840
2025-11-19T14:01:08.465607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436253688278336:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:08.465612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-19T14:01:08.596786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7574436151321205409:2048] [node 1] ICP34 closed connection by debug command
2025-11-19T14:01:08.596921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:08.597003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436254400421153:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:08.597010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436254400421153:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:08.597019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:08.597036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436254400421153:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:08.599328Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:08.599408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436253688278336:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:08.599413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436253688278336:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:08.599454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:08.599471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436253688278336:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:08.611941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436254400421153:2048] [node 1] ICS08 incoming handshake Self# [1:7574436253688278334:0] Peer# [2:7574436254400421149:0] Counter# 134 LastInputSerial# 3
2025-11-19T14:01:08.612458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436253688278336:2048] [node 2] ICS08 incoming handshake Self# [2:7574436254400421149:0] Peer# [1:7574436253688278334:0] Counter# 3 LastInputSerial# 134
2025-11-19T14:01:08.612663Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436253688278338:2305] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:08.612753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:08.612768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436253688278336:2048] [node 2] ICS09 handshake done sender: [1:7574436253688278338:2305] self: [1:7574436253688278334:0] peer: [2:7574436254400421149:0] socket: 22 qp: 3890
2025-11-19T14:01:08.613162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436253688278336:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:08.613173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:08.614883Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436254400421157:2310] held: no
2025-11-19T14:01:08.617544Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436254400421157:2310] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:08.617669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:08.617684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436254400421153:2048] [node 1] ICS09 handshake done sender: [2:7574436254400421157:2310] self: [2:7574436254400421149:0] peer: [1:7574436253688278334:0] socket: 23 qp: 3900
2025-11-19T14:01:08.618227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436254400421153:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:08.618240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:01:08.875257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436253688278336:2048] [node 2] ICS01 socket: 22 reason# Debug
2025-11-19T14:01:08.875299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436253688278336:2048] VirtualId# [1:7574436253688278334:0]
2025-11-19T14:01:08.875307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436253688278336:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:08.875326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:08.875347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:08.876021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:08.876114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436254400421153:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-19T14:01:08.876119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436254400421153:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:08.876154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:08.876185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436254400421153:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:08.884401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436253688278343:2307]) peer tries to resume nonexistent session Self# [2:7574436254400421149:0] Peer# [1:7574436253688278334:0]
2025-11-19T14:01:08.884424Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436253688278343:2307] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:08.884965Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436254400421160:2312] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:08.897720Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:08.897763Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:08.897768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436254400421153:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:08.897807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436254400421153:2048] VirtualId# [2:7574436254400421149:0]
2025-11-19T14:01:08.897831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:01:08.918085Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436253688278344:2308] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:08.918153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:08.918523Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436254400421163:2313] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:08.935960Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436253688278344:2308] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:08.936225Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436254400421163:2313] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:08.936691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:08.936711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:08.936720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436254400421166:2048]
2025-11-19T14:01:08.936729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436254400421166:2048] [node 1] ICS09 handshake done sender: [2:7574436254400421163:2313] self: [2:7574436254400421162:0] peer: [1:7574436253688278345:0] socket: 16 qp: 4034
2025-11-19T14:01:08.936737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436254400421166:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:08.936748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:08.936930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:08.936943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:08.936949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436253688278347:2048]
2025-11-19T14:01:08.936954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436253688278347:2048] [node 2] ICS09 handshake done sender: [1:7574436253688278344:2308] self: [1:7574436253688278345:0] peer: [2:7574436254400421162:0] socket: 17 qp: 4036
2025-11-19T14:01:08.936958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436253688278347:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:08.936963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:01:09.241091Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:01:09.241236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436254400421166:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-19T14:01:09.241241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436254400421166:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:09.241261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:09.241271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436254400421166:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:09.241984Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:09.242058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436253688278347:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:09.242064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436253688278347:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:09.242099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:09.242117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436253688278347:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:09.252017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436253688278347:2048] [node 2] ICS08 incoming handshake Self# [2:7574436254400421162:0] Peer# [1:7574436253688278345:0] Counter# 24 LastInputSerial# 1016
2025-11-19T14:01:09.257255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436254400421166:2048] [node 1] ICS08 incoming handshake Self# [1:7574436253688278345:0] Peer# [2:7574436254400421162:0] Counter# 1016 LastInputSerial# 24
2025-11-19T14:01:09.257669Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436257983245645:2310] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:09.257790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:09.257808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436253688278347:2048] [node 2] ICS09 handshake done sender: [1:7574436257983245645:2310] self: [1:7574436253688278345:0] peer: [2:7574436254400421162:0] socket: 22 qp: 4164
2025-11-19T14:01:09.258129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436253688278347:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:09.258142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:09.259809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436258695388466:2315] held: no
2025-11-19T14:01:09.272966Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436258695388466:2315] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:09.273121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:09.273137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436254400421166:2048] [node 1] ICS09 handshake done sender: [2:7574436258695388466:2315] self: [2:7574436254400421162:0] peer: [1:7574436253688278345:0] socket: 23 qp: 4168
2025-11-19T14:01:09.273722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436254400421166:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:09.273743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:01:09.543039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:01:09.543127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436253688278347:2048] [node 2] ICS07 socket disconnect 22 reason# Debug
2025-11-19T14:01:09.543133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436253688278347:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:09.543167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:09.543182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436253688278347:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:09.543246Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:09.543260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436254400421166:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:09.543264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436254400421166:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:09.543270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:09.551887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:09.551993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436254400421166:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:09.552001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436254400421166:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:09.553383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436254400421166:2048] [node 1] ICS08 incoming handshake Self# [1:7574436253688278345:0] Peer# [2:7574436254400421162:0] Counter# 2282 LastInputSerial# 57
2025-11-19T14:01:09.557267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436253688278347:2048] [node 2] ICS08 incoming handshake Self# [2:7574436254400421162:0] Peer# [1:7574436253688278345:0] Counter# 57 LastInputSerial# 2282
2025-11-19T14:01:09.564227Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436257983245650:2312] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:09.564399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:09.564418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436253688278347:2048] [node 2] ICS09 handshake done sender: [1:7574436257983245650:2312] self: [1:7574436253688278345:0] peer: [2:7574436254400421162:0] socket: 16 qp: 4305
2025-11-19T14:01:09.564860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436253688278347:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:09.564876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:09.572703Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436258695388470:2318] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:09.572770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436258695388470:2318] held: no
2025-11-19T14:01:09.572780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:09.572788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436254400421166:2048] [node 1] ICS09 handshake done sender: [2:7574436258695388470:2318] self: [2:7574436254400421162:0] peer: [1:7574436253688278345:0] socket: 18 qp: 4313
2025-11-19T14:01:09.573240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436254400421166:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:09.573253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:01:09.920152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436253688278347:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-19T14:01:09.920213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436253688278347:2048] VirtualId# [1:7574436253688278345:0]
2025-11-19T14:01:09.920221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436253688278347:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:09.920259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:09.920283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:09.923974Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:09.924063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436254400421166:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-19T14:01:09.924070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436254400421166:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:09.924122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:09.924142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436254400421166:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:09.934002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436257983245655:2314]) peer tries to resume nonexistent session Self# [2:7574436254400421162:0] Peer# [1:7574436253688278345:0]
2025-11-19T14:01:09.934026Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436257983245655:2314] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:09.940308Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436258695388474:2320] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:09.940999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:09.941038Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:09.941046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436254400421166:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:09.941088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436254400421166:2048] VirtualId# [2:7574436254400421162:0]
2025-11-19T14:01:09.941133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 27794
TEvUndelivered Cookie# 27795
TEvUndelivered Cookie# 27796
TEvUndelivered Cookie# 27797
TEvUndelivered Cookie# 27798
TEvUndelivered Cookie# 27799
TEvUndelivered Cookie# 27800
TEvUndelivered Cookie# 27801
TEvUndelivered Cookie# 27802
TEvUndelivered Cookie# 27803
2025-11-19T14:01:09.948375Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436257983245656:2315] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:09.948440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:09.963396Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436258695388477:2321] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:09.975975Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436257983245656:2315] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:09.976378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:09.976389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:09.976394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436257983245659:2048]
2025-11-19T14:01:09.976400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436257983245659:2048] [node 2] ICS09 handshake done sender: [1:7574436257983245656:2315] self: [1:7574436257983245657:0] peer: [2:7574436258695388476:0] socket: 17 qp: 4494
2025-11-19T14:01:09.976405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436257983245659:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:09.976411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:09.976097Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436258695388477:2321] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:09.976197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:09.976210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:09.976218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436258695388480:2048]
2025-11-19T14:01:09.976226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436258695388480:2048] [node 1] ICS09 handshake done sender: [2:7574436258695388477:2321] self: [2:7574436258695388476:0] peer: [1:7574436257983245657:0] socket: 16 qp: 4496
2025-11-19T14:01:09.976234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436258695388480:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:09.976242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:01:10.255897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436258695388480:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-19T14:01:10.255999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436258695388480:2048] VirtualId# [2:7574436258695388476:0]
2025-11-19T14:01:10.256009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436258695388480:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:10.256036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:10.256063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:01:10.256679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:10.256763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436257983245659:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:10.256768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436257983245659:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:10.256802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:10.256817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436257983245659:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-11-19T14:01:10.260997Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436262990355778:2322]) peer tries to resume nonexistent session Self# [1:7574436257983245657:0] Peer# [2:7574436258695388476:0]
2025-11-19T14:01:10.261027Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436262990355778:2322] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:10.261605Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436262278212957:2317] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:01:10.262240Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:10.262271Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:10.262277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436257983245659:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:10.262326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436257983245659:2048] VirtualId# [1:7574436257983245657:0]
2025-11-19T14:01:10.262346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:10.272356Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436262990355782:2324] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:10.273559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:10.274296Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436262278212960:2318] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:10.274379Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436262278212962:2319] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:10.283026Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436262990355782:2324] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:10.284353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:10.284399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:10.284408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436262990355785:2048]
2025-11-19T14:01:10.284415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436262990355785:2048] [node 1] ICS09 handshake done sender: [2:7574436262990355782:2324] self: [2:7574436262990355783:0] peer: [1:7574436262278212959:0] socket: 18 qp: 4648
2025-11-19T14:01:10.284423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436262990355785:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:10.284434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:10.287982Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436262278212960:2318] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:10.288424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:10.288450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:10.288459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436262278212965:2048]
2025-11-19T14:01:10.288469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436262278212965:2048] [node 2] ICS09 handshake done sender: [1:7574436262278212960:2318] self: [1:7574436262278212959:0] peer: [2:7574436262990355783:0] socket: 17 qp: 4647
2025-11-19T14:01:10.288479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436262278212965:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:10.288492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:01:10.419938Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:01:10.420032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436262990355785:2048] [node 1] ICS07 socket disconnect 18 reason# Debug
2025-11-19T14:01:10.420037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436262990355785:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:10.420066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:10.420077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436262990355785:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:10.421667Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:10.421746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436262278212965:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:10.421750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436262278212965:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:10.421777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:10.421799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436262278212965:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:10.467350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436262990355785:2048] [node 1] ICS08 incoming handshake Self# [1:7574436262278212959:0] Peer# [2:7574436262990355783:0] Counter# 333 LastInputSerial# 8
2025-11-19T14:01:10.466552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436262278212965:2048] [node 2] ICS08 incoming handshake Self# [2:7574436262990355783:0] Peer# [1:7574436262278212959:0] Counter# 8 LastInputSerial# 333
2025-11-19T14:01:10.467703Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436262278212967:2320] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:10.469086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:10.469104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436262278212965:2048] [node 2] ICS09 handshake done sender: [1:7574436262278212967:2320] self: [1:7574436262278212959:0] peer: [2:7574436262990355783:0] socket: 22 qp: 4737
2025-11-19T14:01:10.469572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436262278212965:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:10.469590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:10.485905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436262990355789:2327] held: no
2025-11-19T14:01:10.485935Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436262990355789:2327] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:10.486025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:10.486038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436262990355785:2048] [node 1] ICS09 handshake done sender: [2:7574436262990355789:2327] self: [2:7574436262990355783:0] peer: [1:7574436262278212959:0] socket: 23 qp: 4748
2025-11-19T14:01:10.486499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436262990355785:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:10.486512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:01:10.772480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436262278212965:2048] [node 2] ICS01 socket: 22 reason# Debug
2025-11-19T14:01:10.772522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436262278212965:2048] VirtualId# [1:7574436262278212959:0]
2025-11-19T14:01:10.772529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436262278212965:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:10.772551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:10.772573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:10.776745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:10.776840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436262990355785:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-19T14:01:10.776847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436262990355785:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:10.776857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:10.776893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436262990355785:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:10.781583Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436262278212972:2322]) peer tries to resume nonexistent session Self# [2:7574436262990355783:0] Peer# [1:7574436262278212959:0]
2025-11-19T14:01:10.781605Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436262278212972:2322] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:10.786731Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436262990355792:2329] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:10.787466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:10.787501Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:10.787506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436262990355785:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:10.787532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436262990355785:2048] VirtualId# [2:7574436262990355783:0]
2025-11-19T14:01:10.787551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:01:10.791863Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436262278212973:2323] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:10.791915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:10.792326Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436262990355795:2330] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:10.807742Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436262278212973:2323] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:10.808441Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436262990355795:2330] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:10.808950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:10.808966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:10.808975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436262990355798:2048]
2025-11-19T14:01:10.808985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436262990355798:2048] [node 1] ICS09 handshake done sender: [2:7574436262990355795:2330] self: [2:7574436262990355794:0] peer: [1:7574436262278212974:0] socket: 16 qp: 4847
2025-11-19T14:01:10.808993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436262990355798:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:10.809003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:10.809079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:10.809097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:10.809106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436262278212976:2048]
2025-11-19T14:01:10.809113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436262278212976:2048] [node 2] ICS09 handshake done sender: [1:7574436262278212973:2323] self: [1:7574436262278212974:0] peer: [2:7574436262990355794:0] socket: 17 qp: 4846
2025-11-19T14:01:10.809122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436262278212976:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:10.809133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:01:11.227647Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:01:11.227732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436262278212976:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-11-19T14:01:11.227737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436262278212976:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:11.227770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:11.227786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436262278212976:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:11.236659Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:11.236759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436262990355798:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:11.236766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436262990355798:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:11.236794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:11.236811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436262990355798:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:11.239614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436262990355798:2048] [node 1] ICS08 incoming handshake Self# [1:7574436262278212974:0] Peer# [2:7574436262990355794:0] Counter# 2418 LastInputSerial# 57
2025-11-19T14:01:11.253276Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436266573180274:2325] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:11.253434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:11.253449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436262278212976:2048] [node 2] ICS09 handshake done sender: [1:7574436266573180274:2325] self: [1:7574436262278212974:0] peer: [2:7574436262990355794:0] socket: 20 qp: 4980
2025-11-19T14:01:11.254004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436262278212976:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:11.254023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:11.258759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436262278212976:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:11.258773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436262278212976:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:11.258799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:11.258813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436262278212976:2048] [node 2] ICS08 incoming handshake Self# [2:7574436262990355794:0] Peer# [1:7574436262278212974:0] Counter# 57 LastInputSerial# 2418
2025-11-19T14:01:11.258939Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:11.259017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436262278212976:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:11.259048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436262278212976:2048] VirtualId# [1:7574436262278212974:0]
2025-11-19T14:01:11.259062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:11.265082Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436267285323097:2332] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:11.265638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:11.265656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436262990355798:2048] [node 1] ICS09 handshake done sender: [2:7574436267285323097:2332] self: [2:7574436262990355794:0] peer: [1:7574436262278212974:0] socket: 21 qp: 4988
2025-11-19T14:01:11.266336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436262990355798:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:11.266351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:11.266501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:11.266506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436262990355798:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:11.266525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436262990355798:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:11.266533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:11.266565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:11.266620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436262990355798:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:11.266622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436262990355798:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:11.283423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436266573180280:2328]) peer tries to resume nonexistent session Self# [2:7574436262990355794:0] Peer# [1:7574436262278212974:0]
2025-11-19T14:01:11.283446Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436266573180280:2328] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:11.284429Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436267285323102:2334] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:11.285251Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:11.285294Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:11.285299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436262990355798:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:11.285325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436262990355798:2048] VirtualId# [2:7574436262990355794:0]
2025-11-19T14:01:11.285360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 29197
TEvUndelivered Cookie# 29198
TEvUndelivered Cookie# 29199
TEvUndelivered Cookie# 29200
TEvUndelivered Cookie# 29201
TEvUndelivered Cookie# 29202
TEvUndelivered Cookie# 29203
TEvUndelivered Cookie# 29204
TEvUndelivered Cookie# 29205
2025-11-19T14:01:11.294663Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436266573180281:2329] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:11.294708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:11.300140Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436267285323105:2335] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:11.308051Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436266573180281:2329] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:11.308173Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436267285323105:2335] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:11.308299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:11.308317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:11.308325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436267285323108:2048]
2025-11-19T14:01:11.308331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436267285323108:2048] [node 1] ICS09 handshake done sender: [2:7574436267285323105:2335] self: [2:7574436267285323104:0] peer: [1:7574436266573180282:0] socket: 16 qp: 5008
2025-11-19T14:01:11.308339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436267285323108:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:11.308349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:11.308508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:11.308518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:11.308523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436266573180284:2048]
2025-11-19T14:01:11.308529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436266573180284:2048] [node 2] ICS09 handshake done sender: [1:7574436266573180281:2329] self: [1:7574436266573180282:0] peer: [2:7574436267285323104:0] socket: 17 qp: 5007
2025-11-19T14:01:11.308534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436266573180284:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:11.308539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:01:11.408140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:01:11.408218Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [1 <-> 2] socket: Broken pipe
2025-11-19T14:01:11.408223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436266573180284:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:11.408228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436266573180284:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:11.408235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:11.408644Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:11.408726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436266573180284:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:11.408728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436266573180284:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:11.410210Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:11.410280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436267285323108:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:11.410286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436267285323108:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:11.410313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:11.410328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436267285323108:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:11.411532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436267285323108:2048] [node 1] ICS08 incoming handshake Self# [1:7574436266573180282:0] Peer# [2:7574436267285323104:0] Counter# 250 LastInputSerial# 5
2025-11-19T14:01:11.415845Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436266573180286:2331] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:11.416003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:11.416017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436266573180284:2048] [node 2] ICS09 handshake done sender: [1:7574436266573180286:2331] self: [1:7574436266573180282:0] peer: [2:7574436267285323104:0] socket: 20 qp: 5062
2025-11-19T14:01:11.416391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436266573180284:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:11.416403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:11.416448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436266573180284:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:11.416452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436266573180284:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:11.416464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:11.416474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436266573180284:2048] [node 2] ICS08 incoming handshake Self# [2:7574436267285323104:0] Peer# [1:7574436266573180282:0] Counter# 5 LastInputSerial# 250
2025-11-19T14:01:11.416493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:11.416549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436266573180284:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:11.417291Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436267285323111:2337] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:11.417496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:11.417507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436267285323108:2048] [node 1] ICS09 handshake done sender: [2:7574436267285323111:2337] self: [2:7574436267285323104:0] peer: [1:7574436266573180282:0] socket: 21 qp: 5064
2025-11-19T14:01:11.418065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436267285323108:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:11.418073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:11.418161Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:11.418205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436267285323108:2048] [node 1] ICS01 socket: 21 reason# EndOfStream
2025-11-19T14:01:11.418227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436267285323108:2048] VirtualId# [2:7574436267285323104:0]
2025-11-19T14:01:11.418232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436267285323108:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:11.418250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:11.418269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:01:11.427955Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436266573180292:2334] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:11.427988Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:11.427993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436266573180284:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:11.428020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436266573180284:2048] VirtualId# [1:7574436266573180282:0]
2025-11-19T14:01:11.428040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:11.428048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:11.429532Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436267285323117:2339] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:11.436009Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436266573180292:2334] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:11.436118Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436267285323117:2339] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:11.436240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:11.436261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:11.436268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436267285323120:2048]
2025-11-19T14:01:11.436277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436267285323120:2048] [node 1] ICS09 handshake done sender: [2:7574436267285323117:2339] self: [2:7574436267285323116:0] peer: [1:7574436266573180293:0] socket: 16 qp: 5083
2025-11-19T14:01:11.436285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436267285323120:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:11.436294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:11.436729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:11.436743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:11.436748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436266573180295:2048]
2025-11-19T14:01:11.436754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436266573180295:2048] [node 2] ICS09 handshake done sender: [1:7574436266573180292:2334] self: [1:7574436266573180293:0] peer: [2:7574436267285323116:0] socket: 18 qp: 5081
2025-11-19T14:01:11.436759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436266573180295:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:11.436766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-19T14:01:11.865309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7574436151321205409:2048] [node 1] ICP34 closed connection by debug command
2025-11-19T14:01:11.865377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:11.865383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436267285323120:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:11.865387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436267285323120:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:11.865395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:11.865719Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:11.865808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436266573180295:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-19T14:01:11.865816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436266573180295:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:11.865851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:11.865869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436266573180295:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:11.867473Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:11.867546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436267285323120:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:11.867548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436267285323120:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:11.868917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436267285323120:2048] [node 1] ICS08 incoming handshake Self# [1:7574436266573180293:0] Peer# [2:7574436267285323116:0] Counter# 2510 LastInputSerial# 61
2025-11-19T14:01:11.870648Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436266573180297:2336] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:11.870731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:11.870742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436266573180295:2048] [node 2] ICS09 handshake done sender: [1:7574436266573180297:2336] self: [1:7574436266573180293:0] peer: [2:7574436267285323116:0] socket: 20 qp: 5206
2025-11-19T14:01:11.871072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436266573180295:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:11.871082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:11.879101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436266573180295:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:11.879114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436266573180295:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:11.879144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:11.879158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436266573180295:2048] [node 2] ICS08 incoming handshake Self# [2:7574436267285323116:0] Peer# [1:7574436266573180293:0] Counter# 61 LastInputSerial# 2510
2025-11-19T14:01:11.879232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:11.879304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436266573180295:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:11.879333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436266573180295:2048] VirtualId# [1:7574436266573180293:0]
2025-11-19T14:01:11.879345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:11.880530Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436267285323123:2341] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:11.880723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:11.880734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436267285323120:2048] [node 1] ICS09 handshake done sender: [2:7574436267285323123:2341] self: [2:7574436267285323116:0] peer: [1:7574436266573180293:0] socket: 21 qp: 5208
2025-11-19T14:01:11.881311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436267285323120:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:11.881321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:11.881463Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:11.881466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436267285323120:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:11.881470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436267285323120:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:11.881474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:11.881503Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:11.881545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436267285323120:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:11.881547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436267285323120:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:11.891603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436266573180303:2339]) peer tries to resume nonexistent session Self# [2:7574436267285323116:0] Peer# [1:7574436266573180293:0]
2025-11-19T14:01:11.891631Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436266573180303:2339] [node 2] ICH08 Continuation request rejected by proxy
TEvNodeDisconnected
TEvUndelivered Cookie# 29877
TEvUndelivered Cookie# 29878
TEvUndelivered Cookie# 29879
TEvUndelivered Cookie# 29880
TEvUndelivered Cookie# 29881
TEvUndelivered Cookie# 29882
2025-11-19T14:01:11.892024Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436267285323128:2343] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:11.892611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:11.892638Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:11.892643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436267285323120:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:11.892666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436267285323120:2048] VirtualId# [2:7574436267285323116:0]
2025-11-19T14:01:11.892695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:01:11.907906Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436267285323131:2344] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:11.923681Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436267285323131:2344] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:11.924023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:11.924038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:11.924047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436267285323134:2048]
2025-11-19T14:01:11.924054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436267285323134:2048] [node 1] ICS09 handshake done sender: [2:7574436267285323131:2344] self: [2:7574436267285323130:0] peer: [1:7574436266573180305:0] socket: 16 qp: 5229
2025-11-19T14:01:11.924063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436267285323134:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:11.924084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:11.900284Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436266573180304:2340] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:11.900328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:11.923535Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436266573180304:2340] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:11.924971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:11.924983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:11.924990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436266573180307:2048]
2025-11-19T14:01:11.924997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436266573180307:2048] [node 2] ICS09 handshake done sender: [1:7574436266573180304:2340] self: [1:7574436266573180305:0] peer: [2:7574436267285323130:0] socket: 17 qp: 5228
2025-11-19T14:01:11.925005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436266573180307:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:11.925013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-19T14:01:12.385005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7574436151321205409:2048] [node 1] ICP34 closed connection by debug command
2025-11-19T14:01:12.386615Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:12.386702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436266573180307:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:12.386705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436266573180307:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:12.386734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:12.386747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436266573180307:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:12.386450Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:12.386467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436267285323134:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:12.386473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436267285323134:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:12.386481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:12.391822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:12.391921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436267285323134:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:12.391931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436267285323134:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:12.393582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436267285323134:2048] [node 1] ICS08 incoming handshake Self# [1:7574436266573180305:0] Peer# [2:7574436267285323130:0] Counter# 3376 LastInputSerial# 80
2025-11-19T14:01:12.395445Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436270868147605:2342] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:12.395645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:12.395662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436266573180307:2048] [node 2] ICS09 handshake done sender: [1:7574436270868147605:2342] self: [1:7574436266573180305:0] peer: [2:7574436267285323130:0] socket: 20 qp: 5417
2025-11-19T14:01:12.396010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436266573180307:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:12.396026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:12.399184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436266573180307:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:12.399196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436266573180307:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:12.399223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:12.399234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436266573180307:2048] [node 2] ICS08 incoming handshake Self# [2:7574436267285323130:0] Peer# [1:7574436266573180305:0] Counter# 80 LastInputSerial# 3376
2025-11-19T14:01:12.399324Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:12.399398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436266573180307:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:12.399420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436266573180307:2048] VirtualId# [1:7574436266573180305:0]
2025-11-19T14:01:12.399430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:12.411011Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436271580290433:2346] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:12.411204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:12.411219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436267285323134:2048] [node 1] ICS09 handshake done sender: [2:7574436271580290433:2346] self: [2:7574436267285323130:0] peer: [1:7574436266573180305:0] socket: 21 qp: 5419
2025-11-19T14:01:12.411803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436267285323134:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:12.411820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:12.411987Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:12.411994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436267285323134:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:12.411999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436267285323134:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:12.412005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:12.412046Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:12.412097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436267285323134:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:12.412099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436267285323134:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:12.427939Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436270868147611:2345]) peer tries to resume nonexistent session Self# [2:7574436267285323130:0] Peer# [1:7574436266573180305:0]
2025-11-19T14:01:12.427966Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436270868147611:2345] [node 2] ICH08 Continuation request rejected by proxy
TEvNodeDisconnected
TEvUndelivered Cookie# 30678
TEvUndelivered Cookie# 30679
TEvUndelivered Cookie# 30680
TEvUndelivered Cookie# 30681
TEvUndelivered Cookie# 30682
2025-11-19T14:01:12.432023Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436271580290438:2348] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:12.432801Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:12.432840Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:12.432845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436267285323134:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:12.432868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436267285323134:2048] VirtualId# [2:7574436267285323130:0]
2025-11-19T14:01:12.432898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:01:12.444145Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436270868147612:2346] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:12.444191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:12.446908Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436271580290441:2349] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:12.459415Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436271580290441:2349] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:12.459550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:12.459573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:12.459587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436271580290444:2048]
2025-11-19T14:01:12.459597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436271580290444:2048] [node 1] ICS09 handshake done sender: [2:7574436271580290441:2349] self: [2:7574436271580290440:0] peer: [1:7574436270868147613:0] socket: 16 qp: 5435
2025-11-19T14:01:12.459606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436271580290444:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:12.459618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:12.459305Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436270868147612:2346] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:12.459936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:12.459947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:12.459953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436270868147615:2048]
2025-11-19T14:01:12.459959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436270868147615:2048] [node 2] ICS09 handshake done sender: [1:7574436270868147612:2346] self: [1:7574436270868147613:0] peer: [2:7574436271580290440:0] socket: 17 qp: 5434
2025-11-19T14:01:12.459964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436270868147615:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:12.459971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:01:12.566894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:01:12.567018Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:12.567099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436270868147615:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:12.567104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436270868147615:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:12.567115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:12.567136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436270868147615:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:12.569041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:12.569111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436271580290444:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:12.569116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436271580290444:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:12.569141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:12.569156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436271580290444:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:12.572324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436271580290444:2048] [node 1] ICS08 incoming handshake Self# [1:7574436270868147613:0] Peer# [2:7574436271580290440:0] Counter# 194 LastInputSerial# 5
2025-11-19T14:01:12.574281Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436270868147617:2348] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:12.574414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:12.574427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436270868147615:2048] [node 2] ICS09 handshake done sender: [1:7574436270868147617:2348] self: [1:7574436270868147613:0] peer: [2:7574436271580290440:0] socket: 20 qp: 5470
2025-11-19T14:01:12.574862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436270868147615:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:12.574872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:12.578467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436270868147615:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:12.578480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436270868147615:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:12.578507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:12.578520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436270868147615:2048] [node 2] ICS08 incoming handshake Self# [2:7574436271580290440:0] Peer# [1:7574436270868147613:0] Counter# 5 LastInputSerial# 194
2025-11-19T14:01:12.578660Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:12.578734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436270868147615:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:12.578758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436270868147615:2048] VirtualId# [1:7574436270868147613:0]
2025-11-19T14:01:12.578769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:12.585158Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436271580290447:2351] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:12.585580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:12.585606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436271580290444:2048] [node 1] ICS09 handshake done sender: [2:7574436271580290447:2351] self: [2:7574436271580290440:0] peer: [1:7574436270868147613:0] socket: 21 qp: 5471
2025-11-19T14:01:12.586241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436271580290444:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:12.586254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:12.586397Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:12.586400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436271580290444:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:12.586404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436271580290444:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:12.586411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:12.586443Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:12.586494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436271580290444:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:12.586495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436271580290444:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:12.600530Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436270868147623:2351]) peer tries to resume nonexistent session Self# [2:7574436271580290440:0] Peer# [1:7574436270868147613:0]
2025-11-19T14:01:12.600552Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436270868147623:2351] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:12.607057Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436271580290452:2353] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:12.607980Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:12.608022Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:12.608026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436271580290444:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:12.608054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436271580290444:2048] VirtualId# [2:7574436271580290440:0]
2025-11-19T14:01:12.608096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 30735
TEvUndelivered Cookie# 30736
TEvUndelivered Cookie# 30737
TEvUndelivered Cookie# 30738
TEvUndelivered Cookie# 30739
TEvUndelivered Cookie# 30740
TEvUndelivered Cookie# 30741
TEvUndelivered Cookie# 30742
2025-11-19T14:01:12.616523Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436270868147624:2352] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:12.616560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:12.617584Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436271580290455:2354] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:12.635852Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436270868147624:2352] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:12.638734Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436271580290455:2354] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:12.638823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:12.638844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:12.638855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436270868147627:2048]
2025-11-19T14:01:12.638862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436270868147627:2048] [node 2] ICS09 handshake done sender: [1:7574436270868147624:2352] self: [1:7574436270868147625:0] peer: [2:7574436271580290454:0] socket: 17 qp: 5483
2025-11-19T14:01:12.638872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436270868147627:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:12.638884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:12.639181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:12.639194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:12.639200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436271580290458:2048]
2025-11-19T14:01:12.639205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436271580290458:2048] [node 1] ICS09 handshake done sender: [2:7574436271580290455:2354] self: [2:7574436271580290454:0] peer: [1:7574436270868147625:0] socket: 16 qp: 5484
2025-11-19T14:01:12.639211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436271580290458:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:12.639217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-19T14:01:12.926763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:12.926858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436270868147627:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:12.926864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436270868147627:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:12.926903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:12.926919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436270868147627:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:12.926632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7574436151321205409:2048] [node 1] ICP34 closed connection by debug command
2025-11-19T14:01:12.928261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:12.928387Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:12.928393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436271580290458:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:12.928397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436271580290458:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:12.928415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:12.928451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436271580290458:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:12.928453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436271580290458:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:12.932303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436271580290458:2048] [node 1] ICS08 incoming handshake Self# [1:7574436270868147625:0] Peer# [2:7574436271580290454:0] Counter# 1511 LastInputSerial# 36
2025-11-19T14:01:12.933609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436270868147627:2048] [node 2] ICS08 incoming handshake Self# [2:7574436271580290454:0] Peer# [1:7574436270868147625:0] Counter# 36 LastInputSerial# 1511
2025-11-19T14:01:12.934985Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436270868147629:2354] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:12.935098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:12.935114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436270868147627:2048] [node 2] ICS09 handshake done sender: [1:7574436270868147629:2354] self: [1:7574436270868147625:0] peer: [2:7574436271580290454:0] socket: 20 qp: 5611
2025-11-19T14:01:12.935686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436270868147627:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:12.935699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:12.936848Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436271580290461:2356] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:12.936927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:12.936942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436271580290458:2048] [node 1] ICS09 handshake done sender: [2:7574436271580290461:2356] self: [2:7574436271580290454:0] peer: [1:7574436270868147625:0] socket: 21 qp: 5609
2025-11-19T14:01:12.937536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436271580290458:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:12.937551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:01:13.427910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436271580290458:2048] [node 1] ICS01 socket: 21 reason# Debug
2025-11-19T14:01:13.427967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436271580290458:2048] VirtualId# [2:7574436271580290454:0]
2025-11-19T14:01:13.427976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436271580290458:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:13.428012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:13.428037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:01:13.429508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:13.429601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436270868147627:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-11-19T14:01:13.429608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436270868147627:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:13.429636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:13.429654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436270868147627:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:13.446088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436275875257761:2358]) peer tries to resume nonexistent session Self# [1:7574436270868147625:0] Peer# [2:7574436271580290454:0]
2025-11-19T14:01:13.446110Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436275875257761:2358] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:13.447400Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436275163114930:2356] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:01:13.448175Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:13.448238Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:13.448244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436270868147627:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:13.448279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436270868147627:2048] VirtualId# [1:7574436270868147625:0]
2025-11-19T14:01:13.448316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:13.450785Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436275875257762:2359] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:13.450838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:13.451048Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436275163114933:2357] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:13.475861Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436275875257762:2359] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:13.476299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:13.476319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:13.476328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436275875257765:2048]
2025-11-19T14:01:13.476335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436275875257765:2048] [node 1] ICS09 handshake done sender: [2:7574436275875257762:2359] self: [2:7574436275875257763:0] peer: [1:7574436275163114932:0] socket: 17 qp: 5745
2025-11-19T14:01:13.476344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436275875257765:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:13.476356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:13.479320Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436275163114933:2357] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:13.479578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:13.479596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:13.479604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436275163114936:2048]
2025-11-19T14:01:13.479611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436275163114936:2048] [node 2] ICS09 handshake done sender: [1:7574436275163114933:2357] self: [1:7574436275163114932:0] peer: [2:7574436275875257763:0] socket: 16 qp: 5744
2025-11-19T14:01:13.479619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436275163114936:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:13.479627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:01:13.669264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436275163114936:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-11-19T14:01:13.669350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436275163114936:2048] VirtualId# [1:7574436275163114932:0]
2025-11-19T14:01:13.669361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436275163114936:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:13.669403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:13.669426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:13.672537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:13.672626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436275875257765:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:13.672633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436275875257765:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:13.672677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:13.672706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436275875257765:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:13.678363Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436275163114938:2358]) peer tries to resume nonexistent session Self# [2:7574436275875257763:0] Peer# [1:7574436275163114932:0]
2025-11-19T14:01:13.678383Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436275163114938:2358] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:13.678999Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436275875257767:2361] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:13.679562Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:13.679593Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:13.679598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436275875257765:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:13.679667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436275875257765:2048] VirtualId# [2:7574436275875257763:0]
2025-11-19T14:01:13.679706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 31916
TEvUndelivered Cookie# 31917
TEvUndelivered Cookie# 31918
TEvUndelivered Cookie# 31919
TEvUndelivered Cookie# 31920
TEvUndelivered Cookie# 31921
TEvUndelivered Cookie# 31922
TEvUndelivered Cookie# 31923
TEvUndelivered Cookie# 31924
TEvUndelivered Cookie# 31925
2025-11-19T14:01:13.689282Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436275163114939:2359] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:13.689329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:13.693782Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436275875257770:2362] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:13.728447Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436275163114939:2359] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:13.728551Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436275875257770:2362] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:13.728682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:13.728698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:13.728706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436275875257773:2048]
2025-11-19T14:01:13.728714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436275875257773:2048] [node 1] ICS09 handshake done sender: [2:7574436275875257770:2362] self: [2:7574436275875257769:0] peer: [1:7574436275163114940:0] socket: 16 qp: 5839
2025-11-19T14:01:13.728721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436275875257773:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:13.728732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:13.728919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:13.728932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:13.728937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436275163114942:2048]
2025-11-19T14:01:13.728942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436275163114942:2048] [node 2] ICS09 handshake done sender: [1:7574436275163114939:2359] self: [1:7574436275163114940:0] peer: [2:7574436275875257769:0] socket: 17 qp: 5838
2025-11-19T14:01:13.728947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436275163114942:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:13.728953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-11-19T14:01:13.971916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7574436150609062629:2048] [node 2] ICP34 closed connection by debug command
2025-11-19T14:01:13.972654Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:13.972741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436275163114942:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:13.972748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436275163114942:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:13.972761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:13.972788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436275163114942:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:13.974771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:13.974852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436275875257773:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:13.974857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436275875257773:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:13.974891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:13.974909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436275875257773:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:13.977786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436275875257773:2048] [node 1] ICS08 incoming handshake Self# [1:7574436275163114940:0] Peer# [2:7574436275875257769:0] Counter# 1073 LastInputSerial# 27
2025-11-19T14:01:13.979518Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436275163114944:2361] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:13.979621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:13.979634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436275163114942:2048] [node 2] ICS09 handshake done sender: [1:7574436275163114944:2361] self: [1:7574436275163114940:0] peer: [2:7574436275875257769:0] socket: 20 qp: 5973
2025-11-19T14:01:13.979973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436275163114942:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:13.979985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:13.982761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436275163114942:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:13.982773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436275163114942:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:13.982799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:13.982812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436275163114942:2048] [node 2] ICS08 incoming handshake Self# [2:7574436275875257769:0] Peer# [1:7574436275163114940:0] Counter# 27 LastInputSerial# 1073
2025-11-19T14:01:13.982898Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:13.982974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436275163114942:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:13.982999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436275163114942:2048] VirtualId# [1:7574436275163114940:0]
2025-11-19T14:01:13.983009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:13.992711Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436275875257776:2364] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:13.993225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:13.993245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436275875257773:2048] [node 1] ICS09 handshake done sender: [2:7574436275875257776:2364] self: [2:7574436275875257769:0] peer: [1:7574436275163114940:0] socket: 21 qp: 5974
2025-11-19T14:01:13.993671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436275875257773:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:13.993683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:13.993823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:13.993827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436275875257773:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:13.993831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436275875257773:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:13.993838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:13.993870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:13.993920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436275875257773:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:13.993922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436275875257773:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:14.008269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436275163114950:2364]) peer tries to resume nonexistent session Self# [2:7574436275875257769:0] Peer# [1:7574436275163114940:0]
2025-11-19T14:01:14.008301Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436275163114950:2364] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:14.012374Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436275875257781:2366] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:14.013194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:14.013234Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:14.013238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436275875257773:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:14.013266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436275875257773:2048] VirtualId# [2:7574436275875257769:0]
2025-11-19T14:01:14.013299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 32198
TEvUndelivered Cookie# 32199
TEvUndelivered Cookie# 32200
TEvUndelivered Cookie# 32201
TEvUndelivered Cookie# 32202
TEvUndelivered Cookie# 32203
TEvUndelivered Cookie# 32204
TEvUndelivered Cookie# 32205
2025-11-19T14:01:14.020127Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436279458082247:2365] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:14.021105Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436280170225080:2367] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:14.020170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:14.047922Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436279458082247:2365] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:14.048077Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436280170225080:2367] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:14.048299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:14.048321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:14.048332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436280170225083:2048]
2025-11-19T14:01:14.048341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436280170225083:2048] [node 1] ICS09 handshake done sender: [2:7574436280170225080:2367] self: [2:7574436280170225079:0] peer: [1:7574436279458082248:0] socket: 16 qp: 6002
2025-11-19T14:01:14.048351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436280170225083:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:14.048362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:14.048559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:14.048574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:14.048579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436279458082250:2048]
2025-11-19T14:01:14.048584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436279458082250:2048] [node 2] ICS09 handshake done sender: [1:7574436279458082247:2365] self: [1:7574436279458082248:0] peer: [2:7574436280170225079:0] socket: 17 qp: 6004
2025-11-19T14:01:14.048589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436279458082250:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:14.048596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:01:14.428249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436279458082250:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-11-19T14:01:14.428370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436279458082250:2048] VirtualId# [1:7574436279458082248:0]
2025-11-19T14:01:14.428380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436279458082250:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:14.428418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:14.428440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:14.429717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:14.429735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436280170225083:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:14.429740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436280170225083:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:14.429748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:14.431300Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:14.431384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436280170225083:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:14.431387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436280170225083:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:14.434863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436279458082252:2367]) peer tries to resume nonexistent session Self# [2:7574436280170225079:0] Peer# [1:7574436279458082248:0]
2025-11-19T14:01:14.434884Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436279458082252:2367] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:14.435571Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436280170225085:2368] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:14.438322Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:14.438361Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:14.438366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436280170225083:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:14.438436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436280170225083:2048] VirtualId# [2:7574436280170225079:0]
2025-11-19T14:01:14.438496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 32599
TEvUndelivered Cookie# 32600
TEvUndelivered Cookie# 32601
TEvUndelivered Cookie# 32602
TEvUndelivered Cookie# 32603
TEvUndelivered Cookie# 32604
2025-11-19T14:01:14.443959Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436279458082253:2368] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:14.444021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:14.444520Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436280170225088:2369] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:14.459852Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436279458082253:2368] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:14.460406Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436280170225088:2369] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:14.460487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:14.460505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:14.460512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436279458082256:2048]
2025-11-19T14:01:14.460519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436279458082256:2048] [node 2] ICS09 handshake done sender: [1:7574436279458082253:2368] self: [1:7574436279458082254:0] peer: [2:7574436280170225087:0] socket: 17 qp: 6161
2025-11-19T14:01:14.460527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436279458082256:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:14.460537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:14.460821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:14.460831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:14.460835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436280170225091:2048]
2025-11-19T14:01:14.460841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436280170225091:2048] [node 1] ICS09 handshake done sender: [2:7574436280170225088:2369] self: [2:7574436280170225087:0] peer: [1:7574436279458082254:0] socket: 16 qp: 6163
2025-11-19T14:01:14.460845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436280170225091:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:14.460852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:01:14.541366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436280170225091:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-19T14:01:14.542084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436280170225091:2048] VirtualId# [2:7574436280170225087:0]
2025-11-19T14:01:14.542097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436280170225091:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:14.542136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:14.542161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:01:14.542892Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:14.542979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436279458082256:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-11-19T14:01:14.543046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436279458082256:2048] VirtualId# [1:7574436279458082254:0]
2025-11-19T14:01:14.543054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436279458082256:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:14.543097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:14.543113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 32605
2025-11-19T14:01:14.547778Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436279458082258:2370] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:14.547826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:14.548169Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436280170225094:2370] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:14.559114Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436279458082258:2370] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:14.559397Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436280170225094:2370] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:14.559547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:14.559562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:14.559569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436280170225097:2048]
2025-11-19T14:01:14.559576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436280170225097:2048] [node 1] ICS09 handshake done sender: [2:7574436280170225094:2370] self: [2:7574436280170225093:0] peer: [1:7574436279458082259:0] socket: 16 qp: 6210
2025-11-19T14:01:14.559584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436280170225097:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:14.559593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:14.560107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:14.560126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:14.560132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436279458082261:2048]
2025-11-19T14:01:14.560137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436279458082261:2048] [node 2] ICS09 handshake done sender: [1:7574436279458082258:2370] self: [1:7574436279458082259:0] peer: [2:7574436280170225093:0] socket: 17 qp: 6207
2025-11-19T14:01:14.560143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436279458082261:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:14.560149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:01:15.048677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436280170225097:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-19T14:01:15.048753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436280170225097:2048] VirtualId# [2:7574436280170225093:0]
2025-11-19T14:01:15.048762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436280170225097:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:15.048806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:15.048835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:01:15.055125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:15.055254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436279458082261:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:15.055261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436279458082261:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:15.055277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:15.055325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436279458082261:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:15.058394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7574436151321205409:2048] [node 1] ICP12 (actor [2:7574436284465192395:2371]) peer tries to resume nonexistent session Self# [1:7574436279458082259:0] Peer# [2:7574436280170225093:0]
2025-11-19T14:01:15.058419Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [2:7574436284465192395:2371] [node 1] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:15.063904Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [1:7574436283753049559:2372] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake
2025-11-19T14:01:15.064775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436150609062629:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:19014) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:15.064828Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:15.064832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436279458082261:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:15.064921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436279458082261:2048] VirtualId# [1:7574436279458082259:0]
2025-11-19T14:01:15.064942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:15.069285Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436284465192396:2372] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:15.069334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:15.069489Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436283753049562:2373] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:15.071809Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436284465192396:2372] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:15.071888Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436283753049562:2373] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:15.071972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:15.072003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:15.072011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436283753049565:2048]
2025-11-19T14:01:15.072017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436283753049565:2048] [node 2] ICS09 handshake done sender: [1:7574436283753049562:2373] self: [1:7574436283753049561:0] peer: [2:7574436284465192397:0] socket: 16 qp: 6362
2025-11-19T14:01:15.072023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436283753049565:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:15.072031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:15.072223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:15.072245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:15.072258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436284465192399:2048]
2025-11-19T14:01:15.072263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436284465192399:2048] [node 1] ICS09 handshake done sender: [2:7574436284465192396:2372] self: [2:7574436284465192397:0] peer: [1:7574436283753049561:0] socket: 17 qp: 6361
2025-11-19T14:01:15.072267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436284465192399:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:15.072273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:01:15.264592Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:01:15.264667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436284465192399:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-11-19T14:01:15.264673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436284465192399:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:15.264700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:15.264712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436284465192399:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:15.268293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:15.268388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436283753049565:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:15.268394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436283753049565:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:15.268446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:15.268464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436283753049565:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:15.274418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436284465192399:2048] [node 1] ICS08 incoming handshake Self# [1:7574436283753049561:0] Peer# [2:7574436284465192397:0] Counter# 685 LastInputSerial# 17
2025-11-19T14:01:15.274979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436283753049565:2048] [node 2] ICS08 incoming handshake Self# [2:7574436284465192397:0] Peer# [1:7574436283753049561:0] Counter# 17 LastInputSerial# 685
2025-11-19T14:01:15.275183Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436283753049567:2374] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:15.275279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:15.275294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436283753049565:2048] [node 2] ICS09 handshake done sender: [1:7574436283753049567:2374] self: [1:7574436283753049561:0] peer: [2:7574436284465192397:0] socket: 22 qp: 6417
2025-11-19T14:01:15.275789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436283753049565:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:15.275803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:15.283875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:13291) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7574436284465192403:2375] held: no
2025-11-19T14:01:15.286071Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436284465192403:2375] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:15.286474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:15.286493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436284465192399:2048] [node 1] ICS09 handshake done sender: [2:7574436284465192403:2375] self: [2:7574436284465192397:0] peer: [1:7574436283753049561:0] socket: 23 qp: 6419
2025-11-19T14:01:15.287109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436284465192399:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:15.287127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-11-19T14:01:15.791323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436283753049565:2048] [node 2] ICS01 socket: 22 reason# Debug
2025-11-19T14:01:15.791371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436283753049565:2048] VirtualId# [1:7574436283753049561:0]
2025-11-19T14:01:15.791378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436283753049565:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:15.791404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:15.791426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:15.794712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:15.794807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436284465192399:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-19T14:01:15.794815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436284465192399:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:15.794852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:15.794871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436284465192399:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:15.815063Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436284465192407:2378] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:15.816386Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:15.816392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436284465192399:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:15.816418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436284465192399:2048] VirtualId# [2:7574436284465192397:0]
2025-11-19T14:01:15.816444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:01:15.816454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7574436151321205409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:15.820183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436283753049575:2377]) peer tries to resume nonexistent session Self# [2:7574436284465192397:0] Peer# [1:7574436283753049561:0]
2025-11-19T14:01:15.820207Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436283753049575:2377] [node 2] ICH08 Continuation request rejected by proxy
TEvNodeDisconnected
2025-11-19T14:01:15.832919Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436284465192406:2377] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:15.833544Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436283753049573:2376] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:15.833937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [2:7574436284465192407:2378] held: no
2025-11-19T14:01:15.847737Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436284465192407:2378] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:15.847987Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436283753049573:2376] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:15.848144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:15.848161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:15.848170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436283753049577:2048]
2025-11-19T14:01:15.848188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436283753049577:2048] [node 2] ICS09 handshake done sender: [1:7574436283753049573:2376] self: [1:7574436283753049572:0] peer: [2:7574436284465192409:0] socket: 17 qp: 6603
2025-11-19T14:01:15.848196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436283753049577:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:15.848207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:15.848482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:15.848495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:15.848499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436284465192411:2048]
2025-11-19T14:01:15.848513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436284465192411:2048] [node 1] ICS09 handshake done sender: [2:7574436284465192407:2378] self: [2:7574436284465192409:0] peer: [1:7574436283753049572:0] socket: 18 qp: 6606
2025-11-19T14:01:15.848518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436284465192411:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:15.848524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-19T14:01:16.307141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7574436151321205409:2048] [node 1] ICP34 closed connection by debug command
2025-11-19T14:01:16.307883Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:16.307975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436284465192411:2048] [node 1] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-19T14:01:16.307982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436284465192411:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:16.307996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:16.308015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436284465192411:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:16.311319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:16.311410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436283753049577:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:16.311415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436283753049577:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:16.311448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:16.311467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436283753049577:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:16.314035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436283753049577:2048] [node 2] ICS08 incoming handshake Self# [2:7574436284465192409:0] Peer# [1:7574436283753049572:0] Counter# 49 LastInputSerial# 1945
2025-11-19T14:01:16.327505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436284465192411:2048] [node 1] ICS08 incoming handshake Self# [1:7574436283753049572:0] Peer# [2:7574436284465192409:0] Counter# 1945 LastInputSerial# 49
2025-11-19T14:01:16.331113Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436288048016875:2378] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:16.331662Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436288760159711:2381] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:16.331774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:16.331796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436283753049577:2048] [node 2] ICS09 handshake done sender: [1:7574436288048016875:2378] self: [1:7574436283753049572:0] peer: [2:7574436284465192409:0] socket: 22 qp: 6820
2025-11-19T14:01:16.332295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436283753049577:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:16.332308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:16.335970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:16.335992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436284465192411:2048] [node 1] ICS09 handshake done sender: [2:7574436288760159711:2381] self: [2:7574436284465192409:0] peer: [1:7574436283753049572:0] socket: 23 qp: 6817
2025-11-19T14:01:16.336786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436284465192411:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:16.336801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:01:16.652804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:01:16.652891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436283753049577:2048] [node 2] ICS07 socket disconnect 22 reason# Debug
2025-11-19T14:01:16.652896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436283753049577:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:16.652934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:16.652947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436283753049577:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:16.658915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:16.659072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436284465192411:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-19T14:01:16.659079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436284465192411:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:16.659088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:16.659109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436284465192411:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:16.661648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436284465192411:2048] [node 1] ICS08 incoming handshake Self# [1:7574436283753049572:0] Peer# [2:7574436284465192409:0] Counter# 3233 LastInputSerial# 84
2025-11-19T14:01:16.668907Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436288048016880:2380] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:16.669102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:16.669120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436283753049577:2048] [node 2] ICS09 handshake done sender: [1:7574436288048016880:2380] self: [1:7574436283753049572:0] peer: [2:7574436284465192409:0] socket: 16 qp: 6935
2025-11-19T14:01:16.669478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436283753049577:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:16.669492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:16.672117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436283753049577:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:16.672129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436283753049577:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:16.672153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:16.672168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436283753049577:2048] [node 2] ICS08 incoming handshake Self# [2:7574436284465192409:0] Peer# [1:7574436283753049572:0] Counter# 84 LastInputSerial# 3233
2025-11-19T14:01:16.672283Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:16.672362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436283753049577:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:16.672391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436283753049577:2048] VirtualId# [1:7574436283753049572:0]
2025-11-19T14:01:16.672403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:16.680708Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436288760159715:2384] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:16.680902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:16.680931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436284465192411:2048] [node 1] ICS09 handshake done sender: [2:7574436288760159715:2384] self: [2:7574436284465192409:0] peer: [1:7574436283753049572:0] socket: 17 qp: 6937
2025-11-19T14:01:16.681617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436284465192411:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:16.681633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:16.681823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:16.681828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436284465192411:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:16.681832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436284465192411:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:16.681840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:16.681875Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:16.681933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436284465192411:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:16.681935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436284465192411:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:16.695473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436288048016886:2383]) peer tries to resume nonexistent session Self# [2:7574436284465192409:0] Peer# [1:7574436283753049572:0]
2025-11-19T14:01:16.695493Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436288048016886:2383] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:16.704229Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436288760159720:2386] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:16.733121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:16.733175Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:16.733179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436284465192411:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:16.733207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436284465192411:2048] VirtualId# [2:7574436284465192409:0]
2025-11-19T14:01:16.733246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 34827
TEvUndelivered Cookie# 34828
TEvUndelivered Cookie# 34829
TEvUndelivered Cookie# 34830
TEvUndelivered Cookie# 34831
TEvUndelivered Cookie# 34832
2025-11-19T14:01:16.742904Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436288048016887:2384] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:16.742946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:16.746213Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436288760159723:2387] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:16.756066Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436288048016887:2384] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:16.756202Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436288760159723:2387] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:16.756348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:16.756365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:16.756374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436288760159726:2048]
2025-11-19T14:01:16.756380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436288760159726:2048] [node 1] ICS09 handshake done sender: [2:7574436288760159723:2387] self: [2:7574436288760159722:0] peer: [1:7574436288048016888:0] socket: 16 qp: 6960
2025-11-19T14:01:16.756390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436288760159726:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:16.756400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:16.756583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:16.756596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:16.756601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436288048016890:2048]
2025-11-19T14:01:16.756606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436288048016890:2048] [node 2] ICS09 handshake done sender: [1:7574436288048016887:2384] self: [1:7574436288048016888:0] peer: [2:7574436288760159722:0] socket: 17 qp: 6959
2025-11-19T14:01:16.756620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436288048016890:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:16.756626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-11-19T14:01:16.807812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436288760159726:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-11-19T14:01:16.809485Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:16.809581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436288048016890:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-11-19T14:01:16.808754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436288760159726:2048] VirtualId# [2:7574436288760159722:0]
2025-11-19T14:01:16.808768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436288760159726:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:16.808798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:16.808816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:01:16.811115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436288048016890:2048] VirtualId# [1:7574436288048016888:0]
2025-11-19T14:01:16.811130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436288048016890:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:16.811175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:16.811196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
Subscribe
2025-11-19T14:01:16.843855Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436288048016892:2386] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:16.843904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:16.844073Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436288760159729:2388] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:16.852028Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436288048016892:2386] [node 2] ICH04 handshake succeeded
TEvNodeConnected
2025-11-19T14:01:16.852150Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436288760159729:2388] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:16.852246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:16.852272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:16.852281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436288760159732:2048]
2025-11-19T14:01:16.852289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436288760159732:2048] [node 1] ICS09 handshake done sender: [2:7574436288760159729:2388] self: [2:7574436288760159728:0] peer: [1:7574436288048016893:0] socket: 16 qp: 6990
2025-11-19T14:01:16.852297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436288760159732:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:16.852308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:16.854665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:16.854689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:16.854696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436288048016895:2048]
2025-11-19T14:01:16.854703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436288048016895:2048] [node 2] ICS09 handshake done sender: [1:7574436288048016892:2386] self: [1:7574436288048016893:0] peer: [2:7574436288760159728:0] socket: 17 qp: 6989
2025-11-19T14:01:16.854710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436288048016895:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:16.854720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-11-19T14:01:17.310277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7574436151321205409:2048] [node 1] ICP34 closed connection by debug command
2025-11-19T14:01:17.310707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:17.310831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436288048016895:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-11-19T14:01:17.310837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436288048016895:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:17.310875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:17.310899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436288048016895:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:17.314095Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:17.314183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436288760159732:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:17.314191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436288760159732:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:17.314203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:17.314234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436288760159732:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:17.315650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436288760159732:2048] [node 1] ICS08 incoming handshake Self# [1:7574436288048016893:0] Peer# [2:7574436288760159728:0] Counter# 2576 LastInputSerial# 62
2025-11-19T14:01:17.318441Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436292342984193:2388] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:17.325732Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436293055127031:2390] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:17.324199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:17.324219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436288048016895:2048] [node 2] ICS09 handshake done sender: [1:7574436292342984193:2388] self: [1:7574436288048016893:0] peer: [2:7574436288760159728:0] socket: 20 qp: 7113
2025-11-19T14:01:17.324664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436288048016895:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:17.324678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:17.324765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436288048016895:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:17.324769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436288048016895:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:17.324782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:17.324793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436288048016895:2048] [node 2] ICS08 incoming handshake Self# [2:7574436288760159728:0] Peer# [1:7574436288048016893:0] Counter# 62 LastInputSerial# 2576
2025-11-19T14:01:17.324811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:17.324871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436288048016895:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:17.326020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:17.326040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436288760159732:2048] [node 1] ICS09 handshake done sender: [2:7574436293055127031:2390] self: [2:7574436288760159728:0] peer: [1:7574436288048016893:0] socket: 21 qp: 7114
2025-11-19T14:01:17.326463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436288760159732:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:17.326477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:17.326591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:17.326660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436288760159732:2048] [node 1] ICS01 socket: 21 reason# EndOfStream
2025-11-19T14:01:17.326690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436288760159732:2048] VirtualId# [2:7574436288760159728:0]
2025-11-19T14:01:17.326696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436288760159732:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:17.326723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:17.326746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:01:17.347936Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436292342984199:2391] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:17.347973Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [1:7574436150609062629:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:17.347982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436288048016895:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:17.348015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436288048016895:2048] VirtualId# [1:7574436288048016893:0]
2025-11-19T14:01:17.348036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:17.348048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:17.352160Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436293055127037:2392] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:17.376310Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436292342984199:2391] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:17.377340Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436293055127037:2392] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:17.377696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:17.377713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:17.377772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:17.377809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:17.377816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436292342984202:2048]
2025-11-19T14:01:17.377825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436292342984202:2048] [node 2] ICS09 handshake done sender: [1:7574436292342984199:2391] self: [1:7574436292342984200:0] peer: [2:7574436293055127036:0] socket: 18 qp: 7132
2025-11-19T14:01:17.377833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436292342984202:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:17.377843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:17.378127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436293055127040:2048]
2025-11-19T14:01:17.378135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436293055127040:2048] [node 1] ICS09 handshake done sender: [2:7574436293055127037:2392] self: [2:7574436293055127036:0] peer: [1:7574436292342984200:0] socket: 16 qp: 7130
2025-11-19T14:01:17.378139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436293055127040:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:17.378146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-11-19T14:01:17.595022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-11-19T14:01:17.595106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436293055127040:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-11-19T14:01:17.595111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436293055127040:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:17.595151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:17.595164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436293055127040:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:17.600673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:17.600780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436292342984202:2048] [node 2] ICS07 socket disconnect 18 reason# EndOfStream
2025-11-19T14:01:17.600787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436292342984202:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:17.600853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:17.600875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436292342984202:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:17.602840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436292342984202:2048] [node 2] ICS08 incoming handshake Self# [2:7574436293055127036:0] Peer# [1:7574436292342984200:0] Counter# 12 LastInputSerial# 522
2025-11-19T14:01:17.608490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436293055127040:2048] [node 1] ICS08 incoming handshake Self# [1:7574436292342984200:0] Peer# [2:7574436293055127036:0] Counter# 522 LastInputSerial# 12
2025-11-19T14:01:17.615501Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436292342984204:2393] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:17.615632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:17.615650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436292342984202:2048] [node 2] ICS09 handshake done sender: [1:7574436292342984204:2393] self: [1:7574436292342984200:0] peer: [2:7574436293055127036:0] socket: 22 qp: 7232
2025-11-19T14:01:17.616003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436292342984202:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:17.616017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:17.618050Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436293055127044:2394] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:17.623021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:17.623047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436293055127040:2048] [node 1] ICS09 handshake done sender: [2:7574436293055127044:2394] self: [2:7574436293055127036:0] peer: [1:7574436292342984200:0] socket: 23 qp: 7233
2025-11-19T14:01:17.623566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436293055127040:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:17.623581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:17.623620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7574436151321205409:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-11-19T14:01:17.936014Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-11-19T14:01:17.936097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [1:7574436292342984202:2048] [node 2] ICS07 socket disconnect 22 reason# Debug
2025-11-19T14:01:17.936103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436292342984202:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-11-19T14:01:17.936135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:17.936149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [1:7574436292342984202:2048] [node 2] ICS15 start handshake
2025-11-19T14:01:17.938542Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:17.938629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436293055127040:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-11-19T14:01:17.938634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436293055127040:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:17.938665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:17.938683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436293055127040:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:17.941326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [2:7574436293055127040:2048] [node 1] ICS08 incoming handshake Self# [1:7574436292342984200:0] Peer# [2:7574436293055127036:0] Counter# 2005 LastInputSerial# 50
2025-11-19T14:01:17.956047Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436292342984209:2395] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:17.956209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7574436150609062629:2048] [node 2] ICP20 outgoing handshake succeeded
2025-11-19T14:01:17.956232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436292342984202:2048] [node 2] ICS09 handshake done sender: [1:7574436292342984209:2395] self: [1:7574436292342984200:0] peer: [2:7574436293055127036:0] socket: 16 qp: 7337
2025-11-19T14:01:17.956661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436292342984202:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:17.956689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:17.966870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [1:7574436292342984202:2048] [node 2] ICS13 reestablish connection
2025-11-19T14:01:17.966886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436292342984202:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-11-19T14:01:17.966926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:17.966942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:255: Session [1:7574436292342984202:2048] [node 2] ICS08 incoming handshake Self# [2:7574436293055127036:0] Peer# [1:7574436292342984200:0] Counter# 50 LastInputSerial# 2005
2025-11-19T14:01:17.967078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:17.967166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436292342984202:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-11-19T14:01:17.967195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436150609062629:2048] [node 2] ICP30 unregister session Session# [1:7574436292342984202:2048] VirtualId# [1:7574436292342984200:0]
2025-11-19T14:01:17.967205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
2025-11-19T14:01:17.976988Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436293055127048:2397] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:17.977755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7574436151321205409:2048] [node 1] ICP19 incoming handshake succeeded
2025-11-19T14:01:17.977776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436293055127040:2048] [node 1] ICS09 handshake done sender: [2:7574436293055127048:2397] self: [2:7574436293055127036:0] peer: [1:7574436292342984200:0] socket: 17 qp: 7338
2025-11-19T14:01:17.978570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436293055127040:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:17.978589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:17.978748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:759: [2 <-> 1] socket: Broken pipe
2025-11-19T14:01:17.978754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:526: Session [2:7574436293055127040:2048] [node 1] ICS13 reestablish connection
2025-11-19T14:01:17.978758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436293055127040:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-11-19T14:01:17.978766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:17.978805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:17.978863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436293055127040:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-11-19T14:01:17.978865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436293055127040:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:17.998747Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7574436150609062629:2048] [node 2] ICP12 (actor [1:7574436292342984215:2398]) peer tries to resume nonexistent session Self# [2:7574436293055127036:0] Peer# [1:7574436292342984200:0]
2025-11-19T14:01:17.998770Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1111: Handshake [1:7574436292342984215:2398] [node 2] ICH08 Continuation request rejected by proxy
2025-11-19T14:01:17.999437Z :INTERCONNECT INFO: interconnect_handshake.cpp:1579: Handshake [2:7574436293055127053:2399] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake
2025-11-19T14:01:18.000241Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:13291) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-11-19T14:01:18.000298Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:480: Proxy [2:7574436151321205409:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-11-19T14:01:18.000303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436293055127040:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-11-19T14:01:18.000332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436293055127040:2048] VirtualId# [2:7574436293055127036:0]
2025-11-19T14:01:18.000367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 35954
TEvUndelivered Cookie# 35955
TEvUndelivered Cookie# 35956
TEvUndelivered Cookie# 35957
TEvUndelivered Cookie# 35958
TEvUndelivered Cookie# 35959
TEvUndelivered Cookie# 35960
TEvUndelivered Cookie# 35961
2025-11-19T14:01:18.007479Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436296637951512:2399] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:18.007519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436150609062629:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:18.008447Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436297350094352:2400] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:18.043706Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436296637951512:2399] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:18.044617Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436297350094352:2400] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:18.044911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436151321205409:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:18.044936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:18.044944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436151321205409:2048] [node 1] ICP22 created new session: [2:7574436297350094355:2048]
2025-11-19T14:01:18.044952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436297350094355:2048] [node 1] ICS09 handshake done sender: [2:7574436297350094352:2400] self: [2:7574436297350094351:0] peer: [1:7574436296637951513:0] socket: 16 qp: 7349
2025-11-19T14:01:18.044960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436297350094355:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:18.044971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:18.045223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436150609062629:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:18.045236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:18.045241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436150609062629:2048] [node 2] ICP22 created new session: [1:7574436296637951515:2048]
2025-11-19T14:01:18.045247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436296637951515:2048] [node 2] ICS09 handshake done sender: [1:7574436296637951512:2399] self: [1:7574436296637951513:0] peer: [2:7574436297350094351:0] socket: 17 qp: 7348
2025-11-19T14:01:18.045252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436296637951515:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:18.045257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
Subscribe
TEvNodeConnected
2025-11-19T14:01:18.451952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:18.452053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:543: Session [2:7574436297350094355:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-11-19T14:01:18.452059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436297350094355:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:18.452073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:18.452107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:515: Session [2:7574436297350094355:2048] [node 1] ICS15 start handshake
2025-11-19T14:01:18.452574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7574436151321205409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# ::1(::1:13291) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:01:18.452589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436297350094355:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-11-19T14:01:18.452655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436151321205409:2048] [node 1] ICP30 unregister session Session# [2:7574436297350094355:2048] VirtualId# [2:7574436297350094351:0]
2025-11-19T14:01:18.452678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:01:18.452684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7574436151321205409:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# ::1(::1:13291) Couldn't connect to any resolved address LastSessionDieTime# 2025-11-19T14:01:18.000077Z
2025-11-19T14:01:18.452686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# ::1(::1:13291) Couldn't connect to any resolved address LastSessionDieTime# 2025-11-19T14:01:18.000077Z
TEvNodeDisconnected
2025-11-19T14:01:18.468207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7574436151321205409:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:01:18.468263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7574436151321205409:2048] [node 1] ICP33 wake up from error state
2025-11-19T14:01:18.584119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436298057150423:2048] [node 2] ICP01 ready to work
Subscribe
2025-11-19T14:01:18.633966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436299299138494:2048] [node 1] ICP01 ready to work
2025-11-19T14:01:18.651343Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436298057150432:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:18.651779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436298057150423:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:18.652109Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436299299138504:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:18.667891Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436298057150432:2056] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:18.671215Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436299299138504:2056] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:18.671131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436298057150423:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:18.671154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:18.671164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436298057150423:2048] [node 2] ICP22 created new session: [1:7574436298057150435:2048]
2025-11-19T14:01:18.671173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436298057150435:2048] [node 2] ICS09 handshake done sender: [1:7574436298057150432:2056] self: [1:7574436298057150433:0] peer: [2:7574436299299138503:0] socket: 17 qp: 7553
2025-11-19T14:01:18.671182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436298057150435:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:18.671205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
TEvNodeConnected
2025-11-19T14:01:18.671405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436299299138494:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:18.671419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:18.671427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436299299138494:2048] [node 1] ICP22 created new session: [2:7574436299299138507:2048]
2025-11-19T14:01:18.671433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436299299138507:2048] [node 1] ICS09 handshake done sender: [2:7574436299299138504:2056] self: [2:7574436299299138503:0] peer: [1:7574436298057150433:0] socket: 16 qp: 7555
2025-11-19T14:01:18.671439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436299299138507:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:18.671447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:18.713467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:622: Proxy [1:7574436298057150423:2048] [node 2] ICP31 proxy http called
2025-11-19T14:01:18.715806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:622: Proxy [2:7574436299299138494:2048] [node 1] ICP31 proxy http called
2025-11-19T14:01:18.718471Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:18.718556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436299299138507:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-19T14:01:18.718616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436299299138494:2048] [node 1] ICP30 unregister session Session# [2:7574436299299138507:2048] VirtualId# [2:7574436299299138503:0]
2025-11-19T14:01:18.718624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436299299138507:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:18.718655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:18.718683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
TEvNodeDisconnected
2025-11-19T14:01:18.828469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436297411424524:2048] [node 2] ICP01 ready to work
2025-11-19T14:01:18.876244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436295771953218:2048] [node 1] ICP01 ready to work
2025-11-19T14:01:19.172717Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436301706391829:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:19.173153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436297411424524:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:19.173432Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436300066920524:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
RECEIVED TEvTest
2025-11-19T14:01:19.180477Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436300066920524:2056] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:19.180582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436295771953218:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:19.180600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:19.180609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436295771953218:2048] [node 1] ICP22 created new session: [2:7574436300066920527:2048]
2025-11-19T14:01:19.180616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436300066920527:2048] [node 1] ICS09 handshake done sender: [2:7574436300066920524:2056] self: [2:7574436300066920523:0] peer: [1:7574436301706391830:0] socket: 16 qp: 7777
2025-11-19T14:01:19.180625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436300066920527:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:19.180635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:19.180377Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436301706391829:2056] [node 2] ICH04 handshake succeeded
2025-11-19T14:01:19.181641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436297411424524:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:19.181653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:19.181660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436297411424524:2048] [node 2] ICP22 created new session: [1:7574436301706391832:2048]
2025-11-19T14:01:19.181667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436301706391832:2048] [node 2] ICS09 handshake done sender: [1:7574436301706391829:2056] self: [1:7574436301706391830:0] peer: [2:7574436300066920523:0] socket: 17 qp: 7775
2025-11-19T14:01:19.181673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436301706391832:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:19.181705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:20.027794Z :INTERCONNECT CRIT: interconnect_tcp_session.cpp:508: Session [2:7574436300066920527:2048] [node 1] ICS31 serialized event Type# 0x7ff00000 is too large
2025-11-19T14:01:20.027815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436300066920527:2048] [node 1] ICS01 socket: 16 reason# EventTooLarge
2025-11-19T14:01:20.028012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436295771953218:2048] [node 1] ICP30 unregister session Session# [2:7574436300066920527:2048] VirtualId# [2:7574436300066920523:0]
2025-11-19T14:01:20.028025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436300066920527:2048] [node 1] ICS25 shutdown socket, reason# EventTooLarge
2025-11-19T14:01:20.028051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:20.032687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
2025-11-19T14:01:20.036835Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-11-19T14:01:20.036990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [1:7574436301706391832:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-11-19T14:01:20.042381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7574436297411424524:2048] [node 2] ICP30 unregister session Session# [1:7574436301706391832:2048] VirtualId# [1:7574436301706391830:0]
2025-11-19T14:01:20.042407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [1:7574436301706391832:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:20.042454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [2] disconnected
2025-11-19T14:01:20.042476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [2] session destroyed
TEvUndelivered
2025-11-19T14:01:20.116955Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [2:7574436304361887826:2057] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:20.116118Z :INTERCONNECT WARN: interconnect_handshake.cpp:541: Handshake [1:7574436306001359130:2058] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-11-19T14:01:20.116193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7574436297411424524:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-11-19T14:01:20.135934Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [1:7574436306001359130:2058] [node 2] ICH04 handshake succeeded
RECEIVED TEvTest
2025-11-19T14:01:20.138657Z :INTERCONNECT INFO: interconnect_handshake.cpp:413: Handshake [2:7574436304361887826:2057] [node 1] ICH04 handshake succeeded
2025-11-19T14:01:20.139115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7574436295771953218:2048] [node 1] ICP20 outgoing handshake succeeded
2025-11-19T14:01:20.139135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-11-19T14:01:20.139150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7574436295771953218:2048] [node 1] ICP22 created new session: [2:7574436304361887829:2048]
2025-11-19T14:01:20.139172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [2:7574436304361887829:2048] [node 1] ICS09 handshake done sender: [2:7574436304361887826:2057] self: [2:7574436304361887825:0] peer: [1:7574436306001359131:0] socket: 16 qp: 8083
2025-11-19T14:01:20.139182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [2:7574436304361887829:2048] [node 1] ICS10 traffic start
2025-11-19T14:01:20.139198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [1] connected
2025-11-19T14:01:20.139937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7574436297411424524:2048] [node 2] ICP19 incoming handshake succeeded
2025-11-19T14:01:20.139960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-11-19T14:01:20.139971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7574436297411424524:2048] [node 2] ICP22 created new session: [1:7574436306001359133:2048]
2025-11-19T14:01:20.139979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:269: Session [1:7574436306001359133:2048] [node 2] ICS09 handshake done sender: [1:7574436306001359130:2058] self: [1:7574436306001359131:0] peer: [2:7574436304361887825:0] socket: 17 qp: 8102
2025-11-19T14:01:20.139990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:294: Session [1:7574436306001359133:2048] [node 2] ICS10 traffic start
2025-11-19T14:01:20.140004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:320: [2] connected
2025-11-19T14:01:20.179160Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-11-19T14:01:20.179262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:96: Session [2:7574436304361887829:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-11-19T14:01:20.179345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7574436295771953218:2048] [node 1] ICP30 unregister session Session# [2:7574436304361887829:2048] VirtualId# [2:7574436304361887825:0]
2025-11-19T14:01:20.179355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:562: Session [2:7574436304361887829:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-11-19T14:01:20.179394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:572: [1] disconnected
2025-11-19T14:01:20.179422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:135: [1] session destroyed
written 2560000 bytes
written 3437884 bytes
written 4133621 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4221121 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4228518 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195750 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4229419 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4196969 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4187134 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4220008 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4208568 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4196810 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
2025-11-19T14:01:51.120133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7574436440386758049:2048] [node 2] ICP01 ready to work
2025-11-19T14:01:51.144031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7574436439695025591:2048] [node 1] ICP01 ready to work
2025-11-19T14:01:51.148014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7574436440386758049:2048] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6296) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-11-19T14:01:51.148065Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [1:7574436440386758049:2048] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6296) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:01:51.148068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6296) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-11-19T14:01:51.164150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [1:7574436440386758049:2048] [node 2] ICP33 wake up from error state
